Fix x509_rsa_ctx_to_pss when saltlen is md_size.

x509_rsa_ctx_to_pss returns an error when trying to make an X509_ALGOR
for an arbitrary RSA-PSS salt length. This dates to the initial commit
and isn't in OpenSSL, so I imagine this was an attempt to ratchet down
on RSA-PSS parameter proliferation.

If the caller explicitly passes in md_size, rather than using the -1
convenience value, we currently fail. Allow those too and add an error
to the error queue so it is easier to diagnose.

Change-Id: Ia738142e48930ef5a916cad5326f15f64d766ba5
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/43824
Reviewed-by: Adam Langley <agl@google.com>
Commit-Queue: David Benjamin <davidben@google.com>

crypto/x509/rsa_pss.c

@@ -199,11 +199,15 @@ int x509_rsa_ctx_to_pss(EVP_MD_CTX *ctx, X509_ALGOR *algor) {
   if (saltlen == -1) {
     saltlen = EVP_MD_size(sigmd);
   } else if (saltlen == -2) {
+    // TODO(davidben): Forbid this mode. The world has largely standardized on
+    // salt length matching hash length.
     saltlen = EVP_PKEY_size(pk) - EVP_MD_size(sigmd) - 2;
     if (((EVP_PKEY_bits(pk) - 1) & 0x7) == 0) {
       saltlen--;
     }
-  } else {
+  } else if (saltlen != (int)EVP_MD_size(sigmd)) {
+    // We only allow salt length matching hash length and, for now, the -2 case.
+    OPENSSL_PUT_ERROR(X509, X509_R_INVALID_PSS_PARAMETERS);
     return 0;
   }
 

crypto/x509/x509_test.cc

@@ -1558,6 +1558,22 @@ TEST(X509Test, RSASign) {
   ASSERT_TRUE(EVP_PKEY_CTX_set_rsa_padding(pkey_ctx, RSA_PKCS1_PSS_PADDING));
   ASSERT_TRUE(EVP_PKEY_CTX_set_rsa_mgf1_md(pkey_ctx, EVP_sha512()));
   ASSERT_TRUE(SignatureRoundTrips(md_ctx.get(), pkey.get()));
+
+  // RSA-PSS with salt length matching hash length should work when passing in
+  // -1 or the value explicitly.
+  md_ctx.Reset();
+  ASSERT_TRUE(EVP_DigestSignInit(md_ctx.get(), &pkey_ctx, EVP_sha256(), NULL,
+                                 pkey.get()));
+  ASSERT_TRUE(EVP_PKEY_CTX_set_rsa_padding(pkey_ctx, RSA_PKCS1_PSS_PADDING));
+  ASSERT_TRUE(EVP_PKEY_CTX_set_rsa_pss_saltlen(pkey_ctx, -1));
+  ASSERT_TRUE(SignatureRoundTrips(md_ctx.get(), pkey.get()));
+
+  md_ctx.Reset();
+  ASSERT_TRUE(EVP_DigestSignInit(md_ctx.get(), &pkey_ctx, EVP_sha256(), NULL,
+                                 pkey.get()));
+  ASSERT_TRUE(EVP_PKEY_CTX_set_rsa_padding(pkey_ctx, RSA_PKCS1_PSS_PADDING));
+  ASSERT_TRUE(EVP_PKEY_CTX_set_rsa_pss_saltlen(pkey_ctx, 32));
+  ASSERT_TRUE(SignatureRoundTrips(md_ctx.get(), pkey.get()));
 }
 
 // Test the APIs for manually signing a certificate.