explicitly support bytes-like for signature/data in RSA sign/verify (#10259) (#10261)

this was never documented but previously worked in <42. we now also
document that this is supported to confuse ourselves less.

Author: reaperhulk

docs/hazmat/primitives/asymmetric/rsa.rst

@@ -620,7 +620,8 @@ Key interfaces
         Sign one block of data which can be verified later by others using the
         public key.
 
-        :param bytes data: The message string to sign.
+        :param data: The message string to sign.
+        :type data: :term:`bytes-like`
 
         :param padding: An instance of
             :class:`~cryptography.hazmat.primitives.asymmetric.padding.AsymmetricPadding`.
@@ -739,9 +740,11 @@ Key interfaces
         Verify one block of data was signed by the private key
         associated with this public key.
 
-        :param bytes signature: The signature to verify.
+        :param signature: The signature to verify.
+        :type signature: :term:`bytes-like`
 
-        :param bytes data: The message string that was signed.
+        :param data: The message string that was signed.
+        :type data: :term:`bytes-like`
 
         :param padding: An instance of
             :class:`~cryptography.hazmat.primitives.asymmetric.padding.AsymmetricPadding`.

src/rust/src/backend/rsa.rs

@@ -6,6 +6,7 @@ use std::collections::hash_map::DefaultHasher;
 use std::hash::{Hash, Hasher};
 
 use crate::backend::{hashes, utils};
+use crate::buf::CffiBuf;
 use crate::error::{CryptographyError, CryptographyResult};
 use crate::{exceptions, types};
 
@@ -281,11 +282,12 @@ impl RsaPrivateKey {
     fn sign<'p>(
         &self,
         py: pyo3::Python<'p>,
-        data: &[u8],
+        data: CffiBuf<'_>,
         padding: &pyo3::PyAny,
         algorithm: &pyo3::PyAny,
     ) -> CryptographyResult<&'p pyo3::PyAny> {
-        let (data, algorithm) = utils::calculate_digest_and_algorithm(py, data, algorithm)?;
+        let (data, algorithm) =
+            utils::calculate_digest_and_algorithm(py, data.as_bytes(), algorithm)?;
 
         let mut ctx = openssl::pkey_ctx::PkeyCtx::new(&self.pkey)?;
         ctx.sign_init().map_err(|_| {
@@ -419,18 +421,19 @@ impl RsaPublicKey {
     fn verify(
         &self,
         py: pyo3::Python<'_>,
-        signature: &[u8],
-        data: &[u8],
+        signature: CffiBuf<'_>,
+        data: CffiBuf<'_>,
         padding: &pyo3::PyAny,
         algorithm: &pyo3::PyAny,
     ) -> CryptographyResult<()> {
-        let (data, algorithm) = utils::calculate_digest_and_algorithm(py, data, algorithm)?;
+        let (data, algorithm) =
+            utils::calculate_digest_and_algorithm(py, data.as_bytes(), algorithm)?;
 
         let mut ctx = openssl::pkey_ctx::PkeyCtx::new(&self.pkey)?;
         ctx.verify_init()?;
         setup_signature_ctx(py, &mut ctx, padding, algorithm, self.pkey.size(), false)?;
 
-        let valid = ctx.verify(data, signature).unwrap_or(false);
+        let valid = ctx.verify(data, signature.as_bytes()).unwrap_or(false);
         if !valid {
             return Err(CryptographyError::from(
                 exceptions::InvalidSignature::new_err(()),

tests/hazmat/primitives/test_rsa.py

@@ -763,9 +763,15 @@ def test_pkcs1_minimum_key_size(self, backend):
         )
         private_key.sign(b"no failure", padding.PKCS1v15(), hashes.SHA512())
 
-    def test_sign(self, rsa_key_2048: rsa.RSAPrivateKey, backend):
+    @pytest.mark.parametrize(
+        "message",
+        [
+            b"one little message",
+            bytearray(b"one little message"),
+        ],
+    )
+    def test_sign(self, rsa_key_2048: rsa.RSAPrivateKey, message, backend):
         private_key = rsa_key_2048
-        message = b"one little message"
         pkcs = padding.PKCS1v15()
         algorithm = hashes.SHA256()
         signature = private_key.sign(message, pkcs, algorithm)
@@ -1375,9 +1381,15 @@ def test_pss_verify_salt_length_too_long(self, backend):
                 hashes.SHA1(),
             )
 
-    def test_verify(self, rsa_key_2048: rsa.RSAPrivateKey, backend):
+    @pytest.mark.parametrize(
+        "message",
+        [
+            b"one little message",
+            bytearray(b"one little message"),
+        ],
+    )
+    def test_verify(self, rsa_key_2048: rsa.RSAPrivateKey, message, backend):
         private_key = rsa_key_2048
-        message = b"one little message"
         pkcs = padding.PKCS1v15()
         algorithm = hashes.SHA256()
         signature = private_key.sign(message, pkcs, algorithm)