I've written a bare bones enveloping example that takes a string, seals it in an envelope, and then goes about opening it. Everything works just fine if I generate my RSA keys programatically. Unfortunately, it does not work if I encrypt the session keys with an RSA public key that was created on the command line like:
> openssl genrsa -out rsaprivatekey.pem
> openssl rsa -in rsaprivatekey.pem -out rsapublickey.pem
I would greatly appreciate if someone could explain why my programatically-created keys work, but the command-line ones do not. The code that generates usable keys looks like this:
int main() {
// generate & check keys ----
RSA* rsa = RSA_generate_key(2048, RSA_F4, NULL, 0);
int check_key = RSA_check_key(rsa);
while (check_key <= 0) {
cerr << 'error generating keys -- regenerating..';
rsa = RSA_generate_key(2048, RSA_F4, NULL, 0);
check_key = RSA_check_key(rsa);
}
RSA_blinding_on(rsa, NULL);
// write out pem-encoded public key ----
BIO* rsaPublicBio = BIO_new_file('rsapublickey.pem', 'w');
PEM_write_bio_RSAPublicKey(rsaPublicBio, rsa);
// write out pem-encoded encrypted private key ----
BIO* rsaPrivateBio = BIO_new_file('rsaprivatekey.pem', 'w');
PEM_write_bio_RSAPrivateKey(rsaPrivateBio, rsa, NULL, NULL, 0, NULL, NULL);
BIO_free(rsaPublicBio);
BIO_free(rsaPrivateBio);
RSA_free(rsa);
..
return 0;
}
The program that uses the keys is:
#include <cstdio>
#include <cstring>
#include <openssl/ssl.h>
#include <openssl/rand.h>
#include <openssl/ecdsa.h>
#define BUF_SIZE 4096
#define BLOCK_SIZE 32
int main() {
// uninitialized symmetric cipher context ----
EVP_CIPHER_CTX* ctx = new EVP_CIPHER_CTX;
// symmetric cipher ----
const EVP_CIPHER* type = EVP_aes_256_cbc();
unsigned char
message[BUF_SIZE] =
'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
printf('Unencoded string = {%s}n', message);
int npubk = 1;
unsigned char** ek = new unsigned char*[npubk];
int* ekl = new int[npubk];
EVP_PKEY** pubk = new EVP_PKEY*[npubk];
// read in pem-encoded public key ----
BIO* rsa_pub_bio = BIO_new_file('rsapublickey.pem', 'r');
RSA* rsa_pub = RSA_new();
PEM_read_bio_RSAPublicKey(rsa_pub_bio, &rsa_pub, NULL, NULL);
BIO_free(rsa_pub_bio);
// encrypt symmetric session keys ----
for (int i = 0; i < npubk; i++) {
pubk[i] = EVP_PKEY_new();
EVP_PKEY_assign_RSA(pubk[i], rsa_pub);
ek[i] = new unsigned char[EVP_PKEY_size(pubk[i])];
ekl[i] = EVP_PKEY_size(pubk[i]);
}
// random initialization vector ----
unsigned char* iv = new unsigned char[EVP_MAX_IV_LENGTH];
RAND_pseudo_bytes(iv, EVP_MAX_IV_LENGTH);
int message_len; // initialized by EVP_SealUpdate & EVP_SealFinal
unsigned char encrypt_buf[BUF_SIZE + BLOCK_SIZE];
EVP_SealInit(ctx, type, ek, &ekl[0], &iv[0], &pubk[0], npubk);
// EVP_SealUpdate changes message_len to # bytes in message ----
EVP_SealUpdate(ctx, encrypt_buf, &message_len, message, strlen((const char*) message));
printf('buf_len: %dn', message_len);
int total_len = message_len; // line must be between SealUpdate & SealFinal
// EVP_SealFinal changes message_len value to # bytes of encryption overhead ----
EVP_SealFinal(ctx, &encrypt_buf[message_len], &message_len);
int i;
printf('Encoded string = {');
for (i = 0; i < message_len; i++) {
printf('%02x', encrypt_buf[i]);
}
for (i = 0; i < message_len; i++) {
printf('%02x', encrypt_buf[i + total_len]);
}
printf('}n');
unsigned char decrypt_buf[BUF_SIZE];
int decrypt_len; // initialized by EVP_OpenUpdate & EVP_OpenFinal
// read in pem-encoded encrypted private key ----
BIO* rsa_priv_bio = BIO_new_file('rsaprivatekey.pem', 'r');
RSA* rsa_priv = RSA_new();
PEM_read_bio_RSAPrivateKey(rsa_priv_bio, &rsa_priv, NULL, NULL);
BIO_free(rsa_priv_bio);
EVP_PKEY* privk = EVP_PKEY_new();
EVP_PKEY_assign_RSA(privk, rsa_priv);
EVP_OpenInit(ctx, type, *ek, ekl[0], &iv[0], privk);
// EVP_OpenUpdate changes decrypt_len to # bytes in decrypted message ----
EVP_OpenUpdate(ctx, decrypt_buf, &decrypt_len, encrypt_buf, total_len + message_len);
total_len = decrypt_len; // line must be between OpenUpdate & OpenFinal
EVP_OpenFinal(ctx, &decrypt_buf[total_len], &decrypt_len);
// EVP_OpenFinal changes decrypt_len value to # bytes of encryption overhead ----
decrypt_buf[total_len + decrypt_len] = '0';
printf('Unencoded string = {%s}n', decrypt_buf);
delete ctx;
EVP_PKEY_free(privk);
for (int i = 0; i < npubk; i++) {
EVP_PKEY_free(pubk[i]);
delete ek[i];
}
delete[] ek;
delete[] ekl;
delete[] pubk;
delete[] iv;
return 0;
}