Question

How do I get the Microsoft’s CryptoAPI CryptImportKey function to import a PEM encoded key? It actually works but CryptDecrypt returns an error.

// 1. Generate a Public/Private RSA key pair like so:

openssl genrsa -out private.pem 2048
openssl rsa -in private.pem -out public.pem -outform PEM -pubout

// 2. Create a digital signaure using OpenSSL

// Load Private key
//  -----BEGIN RSA PRIVATE KEY-----
//  BLAHBLAHBLAH
//  -----END RSA PRIVATE KEY-----

// Concat user details
std::string sUser = "John Doe | Business | [e-mail] | 1316790394 | 0 | 1 | ProductName | 1";

// Get a one-way hash of it.
SHA1((const unsigned char *) sUser.c_str(),sUser.size(), hash);

// Create the digital signature ~ PKCS #1 v2.0 format (also known as OAEP encryption)
RSA_sign(NID_sha1, hash, SHA_DIGEST_LENGTH, pbData, &iDataLen, rsa_key);

// 3. Verify the signature using Windows CryptoAPI

// Load Public key
//  -----BEGIN PUBLIC KEY-----
//  BLAHBLAHBLAH
//  -----END PUBLIC KEY-----

// Convert from PEM format to DER format - removes header and footer and decodes from base64
CryptStringToBinaryA((char*)pbPublicPEM, iPEMSize, CRYPT_STRING_ANY, pbPublicDER, &iDERSize, NULL, NULL);

// Decode from DER format to CERT_PUBLIC_KEY_INFO. This has the public key in ASN.1 encoded
// format called "SubjectPublicKeyInfo" ... szOID_RSA_RSA
// Do I need to get the "public key" and "modulus" from this format and build a PUBLICKEYBLOB manually?
CryptDecodeObjectEx( X509_ASN_ENCODING, X509_PUBLIC_KEY_INFO, pbPublicDER, iDERSize, CRYPT_ENCODE_ALLOC_FLAG, NULL, &pbPublicPBLOB, &iPBLOBSize );

// decode the RSA Public key itself to a PUBLICKEYBLOB ?
CryptDecodeObjectEx( X509_ASN_ENCODING, RSA_CSP_PUBLICKEYBLOB, pbPublicPBLOB->PublicKey.pbData, pbPublicPBLOB->PublicKey.cbData, CRYPT_ENCODE_ALLOC_FLAG, NULL, &pbPKEY, &iPKEYSize );

// Get a context
CryptAcquireContext(&hCryptProv, NULL, MS_ENHANCED_PROV, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT);

// load the key
CryptImportKey(hCryptProv, pbPKEY, iPKEYSize, 0, CRYPT_OAEP, &hKey);

// Verify the signature
CryptDecrypt(hKey, 0, TRUE, 0, pbData, &iDataLen);

// CryptDecrypt returns NTE_NO_KEY -2146893811 0x8009000D

Answer 1

You are using the wrong APIs. RSA_sign() signs a hash; use CryptVerifySignature() to verify it.