crypto: preserve RSA-PSS legacy pubkey DER

Recreate the historical rsaEncryption AlgorithmIdentifier used by
X509Certificate legacy pubkey output for restricted and unrestricted
RSA-PSS keys without relying on deprecated RSA APIs.

Signed-off-by: Filip Skokan <panva.ip@gmail.com>

Author: panva

deps/ncrypto/ncrypto.cc

@@ -93,7 +93,11 @@ const EVP_MD* GetDigestCtxMd(const EVP_MD_CTX* ctx) {
 }
 
 #if NCRYPTO_USE_OPENSSL3_PROVIDER
+using ASN1StringPointer = DeleteFnPtr<ASN1_STRING, ASN1_STRING_free>;
 using OSSLParamBldPointer = DeleteFnPtr<OSSL_PARAM_BLD, OSSL_PARAM_BLD_free>;
+using RsaPssParamsPointer = DeleteFnPtr<RSA_PSS_PARAMS, RSA_PSS_PARAMS_free>;
+using X509AlgorPointer = DeleteFnPtr<X509_ALGOR, X509_ALGOR_free>;
+using X509PubkeyPointer = DeleteFnPtr<X509_PUBKEY, X509_PUBKEY_free>;
 struct OSSLParamDeleter {
   void operator()(OSSL_PARAM* params) const {
     if (params == nullptr) return;
@@ -5684,13 +5688,74 @@ bool ReadRsaPssParams(const EVP_PKEY* pkey, Rsa::PssParams* params) {
 
   return true;
 }
+
+bool SetRsaPssHashAlgorithm(X509_ALGOR** out, const Digest& digest) {
+  if (EVP_MD_is_a(digest.get(), "SHA1")) return true;
+
+  X509AlgorPointer algorithm(X509_ALGOR_new());
+  if (!algorithm) return false;
+  X509_ALGOR_set_md(algorithm.get(), digest.get());
+  *out = algorithm.release();
+  return true;
+}
+
+bool SetRsaPssMaskGenAlgorithm(X509_ALGOR** out, const Digest& digest) {
+  if (EVP_MD_is_a(digest.get(), "SHA1")) return true;
+
+  X509AlgorPointer hash_algorithm(X509_ALGOR_new());
+  if (!hash_algorithm) return false;
+  X509_ALGOR_set_md(hash_algorithm.get(), digest.get());
+
+  ASN1StringPointer hash_algorithm_der(ASN1_item_pack(
+      hash_algorithm.get(), ASN1_ITEM_rptr(X509_ALGOR), nullptr));
+  if (!hash_algorithm_der) return false;
+
+  X509AlgorPointer algorithm(X509_ALGOR_new());
+  if (!algorithm || X509_ALGOR_set0(algorithm.get(),
+                                    OBJ_nid2obj(NID_mgf1),
+                                    V_ASN1_SEQUENCE,
+                                    hash_algorithm_der.get()) != 1) {
+    return false;
+  }
+  hash_algorithm_der.release();
+  *out = algorithm.release();
+  return true;
+}
+
+ASN1StringPointer EncodeRsaPssParams(const Rsa::PssParams& params) {
+  const Digest digest = Digest::FromName(params.digest.data());
+  if (!digest) return {};
+
+  const Digest mgf1_digest = params.mgf1_digest
+                                 ? Digest::FromName(params.mgf1_digest->data())
+                                 : digest;
+  if (!mgf1_digest) return {};
+
+  RsaPssParamsPointer pss(RSA_PSS_PARAMS_new());
+  if (!pss || !SetRsaPssHashAlgorithm(&pss->hashAlgorithm, digest) ||
+      !SetRsaPssMaskGenAlgorithm(&pss->maskGenAlgorithm, mgf1_digest)) {
+    return {};
+  }
+
+  if (params.salt_length != 20) {
+    pss->saltLength = ASN1_INTEGER_new();
+    if (pss->saltLength == nullptr ||
+        ASN1_INTEGER_set_int64(pss->saltLength, params.salt_length) != 1) {
+      return {};
+    }
+  }
+
+  return ASN1StringPointer(
+      ASN1_item_pack(pss.get(), ASN1_ITEM_rptr(RSA_PSS_PARAMS), nullptr));
+}
 }  // namespace
 
 Rsa::Rsa() : rsa_(false) {}
 
 Rsa::Rsa(const EVP_PKEY* pkey) : Rsa() {
   const int type = EVPKeyPointer::id(pkey);
   if (type != EVP_PKEY_RSA && type != EVP_PKEY_RSA_PSS) return;
+  rsa_pss_ = type == EVP_PKEY_RSA_PSS;
   if (!GetPKeyBnParam(pkey, OSSL_PKEY_PARAM_RSA_N, &n_) ||
       !GetPKeyBnParam(pkey, OSSL_PKEY_PARAM_RSA_E, &e_)) {
     return;
@@ -5786,7 +5851,35 @@ BIOPointer Rsa::derPublicKey() const {
   if (!bio) return {};
 #if NCRYPTO_USE_OPENSSL3_PROVIDER
   auto pkey = EVPKeyPointer::NewRSA(*this);
-  if (!pkey || i2d_PUBKEY_bio(bio.get(), pkey.get()) != 1) return {};
+  if (!pkey) return {};
+  if (!rsa_pss_) {
+    if (i2d_PUBKEY_bio(bio.get(), pkey.get()) != 1) return {};
+    return bio;
+  }
+
+  X509_PUBKEY* raw_pubkey = nullptr;
+  const int result = X509_PUBKEY_set(&raw_pubkey, pkey.get());
+  X509PubkeyPointer pubkey(raw_pubkey);
+  if (result != 1) return {};
+
+  int parameter_type = V_ASN1_UNDEF;
+  ASN1StringPointer parameters;
+  if (pss_params_) {
+    parameters = EncodeRsaPssParams(*pss_params_);
+    if (!parameters) return {};
+    parameter_type = V_ASN1_SEQUENCE;
+  }
+
+  if (X509_PUBKEY_set0_param(pubkey.get(),
+                             OBJ_nid2obj(NID_rsaEncryption),
+                             parameter_type,
+                             parameters.get(),
+                             nullptr,
+                             0) != 1) {
+    return {};
+  }
+  parameters.release();
+  if (i2d_X509_PUBKEY_bio(bio.get(), pubkey.get()) != 1) return {};
 #else
   if (rsa_ == nullptr || i2d_RSA_PUBKEY_bio(bio.get(), rsa_) != 1) return {};
 #endif

deps/ncrypto/ncrypto.h

@@ -639,6 +639,7 @@ class Rsa final {
  private:
 #if NCRYPTO_USE_OPENSSL3_PROVIDER
   bool rsa_ = false;
+  bool rsa_pss_ = false;
   DeleteFnPtr<BIGNUM, BN_free> n_;
   DeleteFnPtr<BIGNUM, BN_free> e_;
   DeleteFnPtr<BIGNUM, BN_clear_free> d_;

test/fixtures/keys/Makefile

@@ -71,6 +71,8 @@ all: \
   rsa_private_4096.pem \
   rsa_public_2048.pem \
   rsa_public_4096.pem \
+  rsa_pss_cert_2048.pem \
+  rsa_pss_cert_2048_sha256_sha256_16.pem \
   rsa_pss_private_2048.pem \
   rsa_pss_private_2048_sha256_sha256_16.pem \
   rsa_pss_private_2048_sha512_sha256_20.pem \
@@ -918,6 +920,12 @@ rsa_pss_private_2048_sha512_sha256_20.pem:
 rsa_pss_private_2048_sha1_sha1_20.pem:
 	openssl genpkey -algorithm RSA-PSS -pkeyopt rsa_keygen_bits:2048 -pkeyopt rsa_keygen_pubexp:65537 -pkeyopt rsa_pss_keygen_md:sha1 -pkeyopt rsa_pss_keygen_mgf1_md:sha1 -pkeyopt rsa_pss_keygen_saltlen:20 -out rsa_pss_private_2048_sha1_sha1_20.pem
 
+rsa_pss_cert_2048.pem: rsa_pss_private_2048.pem
+	openssl req -new -x509 -key rsa_pss_private_2048.pem -subj "/CN=Node.js" -days 36500 -set_serial 1 -out rsa_pss_cert_2048.pem
+
+rsa_pss_cert_2048_sha256_sha256_16.pem: rsa_pss_private_2048_sha256_sha256_16.pem
+	openssl req -new -x509 -key rsa_pss_private_2048_sha256_sha256_16.pem -subj "/CN=Node.js" -days 36500 -set_serial 1 -out rsa_pss_cert_2048_sha256_sha256_16.pem
+
 rsa_pss_public_2048.pem: rsa_pss_private_2048.pem
 	openssl pkey -in rsa_pss_private_2048.pem -pubout -out rsa_pss_public_2048.pem
 

test/fixtures/keys/rsa_pss_cert_2048.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDWjCCAg6gAwIBAgIBATBBBgkqhkiG9w0BAQowNKAPMA0GCWCGSAFlAwQCAQUA
+oRwwGgYJKoZIhvcNAQEIMA0GCWCGSAFlAwQCAQUAogMCASAwEjEQMA4GA1UEAwwH
+Tm9kZS5qczAgFw0yNjA3MTYyMjE4MzJaGA8yMTI2MDYyMjIyMTgzMlowEjEQMA4G
+A1UEAwwHTm9kZS5qczCCASAwCwYJKoZIhvcNAQEKA4IBDwAwggEKAoIBAQDLg4x1
+Lzg+WAjkJEtt1u531G7u+msWvZg01SZ1PsPdiACmK/wKPMpcuJUOdswRP6XCa1qD
+8RxmzFUIqi+y7fSlgBIGuIc6IxVi6L35lomhD3IjqU6MbVFu5QElc1KMcEnGoGqZ
+yBk5vWZ1Gv95fOkAxYneF+4uiNTb7NRInf+u96l6lwsUbar2cDT48myQns+eHcvT
+J2e/dTeF24yfk5V+b8nHUU/JmkPXzi0l9/434xjezeEhx+46cv0+XhE8Z5pMLhPW
+jjrQ1obpsbtzZaGDo05pWBoUhqI1utNM0KlWN5PJqWxNdiVBDeGIqOy3RDul8USQ
+kOjyqKotp3UyyXn3AgMBAAGjUzBRMB0GA1UdDgQWBBQDCGflRn9LTmklPpSFii4J
+fQiQjTAfBgNVHSMEGDAWgBQDCGflRn9LTmklPpSFii4JfQiQjTAPBgNVHRMBAf8E
+BTADAQH/MEEGCSqGSIb3DQEBCjA0oA8wDQYJYIZIAWUDBAIBBQChHDAaBgkqhkiG
+9w0BAQgwDQYJYIZIAWUDBAIBBQCiAwIBIAOCAQEAsXJk8iu+XNSmIheNPvzfSoNp
+7kkFqFjWRwo/a4vXb4Y7DRDsMeUM4dGNtN75Gj5bppnYtQ3ku/DWGLrVRPGs/sRx
+itOJxvJzPNCv4yYUwm8p8ua7l1Sdmrder84XTG/Dm3dOoUOxCmEQzzjghXE0mRih
+UC4aPyOGNHNlQg5YyEmfEnltlL12ls8cHE+5R56xeI34T/MukH0NVf7wNNfqrMqy
+PrrXG79+D3+Jeu3ddtwvoKQyQ/VuszJGjdplkDodttyTfu3GOgC/bbR9iz2GkcTV
+NEkjJHenMzpoAoNZukeuClOHF6NT3hBnKPmcZYlLPgIY1iOBYoGwctgb5pPxNg==
+-----END CERTIFICATE-----

test/fixtures/keys/rsa_pss_cert_2048_sha256_sha256_16.pem

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDkDCCAkSgAwIBAgIBATBBBgkqhkiG9w0BAQowNKAPMA0GCWCGSAFlAwQCAQUA
+oRwwGgYJKoZIhvcNAQEIMA0GCWCGSAFlAwQCAQUAogMCARAwEjEQMA4GA1UEAwwH
+Tm9kZS5qczAgFw0yNjA3MTYyMjAyMzVaGA8yMTI2MDYyMjIyMDIzNVowEjEQMA4G
+A1UEAwwHTm9kZS5qczCCAVYwQQYJKoZIhvcNAQEKMDSgDzANBglghkgBZQMEAgEF
+AKEcMBoGCSqGSIb3DQEBCDANBglghkgBZQMEAgEFAKIDAgEQA4IBDwAwggEKAoIB
+AQDfqNM4C+QtD73iILqOkqfV8ha3O19jpX8UujIk1Z72bbbuwEzh0+sBw0dD0N8C
+gkXnePOEEd6q7HNmbyCNqRpDK6NDvaCMDWgEaD/PlHkRntvKh81IXSMC5imjRfOc
+ZIE/Gnw7h8tanab0n75+ODvLJrmEWUG2q79Im1mWMx7Spod+Np6XEY+7I7nAUUWi
+vr35Yx5DeyxY8rxFGpsLtGsi7JNQO4aHyeBpj8tz0Fhv23uPywE2nGmPHfnkXWbr
+TcHGbzYBgEbeSH9KUkRwczqDXNOPhtfaEHEFTm0MoeKCnJe1VOjSywev77dV1KZf
+pVh3Kh0ZRQIe9YOVJhj4lMx3AgMBAAGjUzBRMB0GA1UdDgQWBBRrGC6N4gEM/Il9
+PMkKs+dhCbldkjAfBgNVHSMEGDAWgBRrGC6N4gEM/Il9PMkKs+dhCbldkjAPBgNV
+HRMBAf8EBTADAQH/MEEGCSqGSIb3DQEBCjA0oA8wDQYJYIZIAWUDBAIBBQChHDAa
+BgkqhkiG9w0BAQgwDQYJYIZIAWUDBAIBBQCiAwIBEAOCAQEA1SP4HNSgQm2OpuRb
+VjEDyVANpCyuKjUFY0FAwart4YSKHwERuR5VccugGwlbnH0vKQ2uwTB/a5BlXssg
+0KB8qjDoOoxzhgZTA7yuHuQLpgCQhd00ORpxFYI1CyB+HywrpbuV4LHJoZjO7B6O
+Z0PPRs77023WkT/O4KRrFH8uaMgPMwUIjIGXzSp+TYObLnYTWmGyJ7rJVwOPH30z
++BY++W0ZhkCuJveajuF3vGpA4o/lA0kcf6Hz2Prl28kHu0hOpZNTx2zMl+GFtlWE
+TIybjAvbA4TJW0rAURRuP4lhYsPJAnZDAnxHFvLeLKypfctv7eAc82U8rVjT+DIA
+NjayPQ==
+-----END CERTIFICATE-----

test/parallel/test-crypto-x509.js

@@ -7,6 +7,7 @@ if (!common.hasCrypto)
 
 const {
   X509Certificate,
+  createHash,
   createPrivateKey,
   generateKeyPairSync,
   createSign,
@@ -27,6 +28,29 @@ const ca = readFileSync(fixtures.path('keys', 'ca1-cert.pem'));
 
 const privateKey = createPrivateKey(key);
 
+if (!process.features.openssl_is_boringssl) {
+  for (const [name, length, digest] of [
+    [
+      'rsa_pss_cert_2048.pem',
+      292,
+      'dff998a209bfa2e6ded1208c6e57f5b6bdedfa44b631265e3e244f38e637f6e4',
+    ],
+    [
+      'rsa_pss_cert_2048_sha256_sha256_16.pem',
+      342,
+      'da0bcd53fbe3969c7cc2730f86abc34e0e1c340264bbdfa3faf01484c2eeece0',
+    ],
+  ]) {
+    const pssCert = new X509Certificate(
+      readFileSync(fixtures.path('keys', name)));
+    const pubkey = pssCert.toLegacyObject().pubkey;
+    assert.strictEqual(pubkey.length, length);
+    assert.strictEqual(
+      createHash('sha256').update(pubkey).digest('hex'),
+      digest);
+  }
+}
+
 [1, {}, false, null].forEach((i) => {
   assert.throws(() => new X509Certificate(i), {
     code: 'ERR_INVALID_ARG_TYPE'