I am trying to perform string data encryption in C using OpenSSL with a public key shared with me. I have searched online but I haven’t had much success finding helpful information.

My encryption should use RSA PKCS1 Padding with SHA1 OAEP hash.

I created this function to encrypt

int rsaEvpEncrypt(EVP_PKEY* pkey, const char * data){
    EVP_PKEY_CTX * ctx = NULL;
    //EVP_PKEY * pkey = NULL;
    unsigned char * encrypted = NULL;
    size_t outlen = 0;
    const size_t inlen = strlen(data);

    tappa_printf("Print %d", 1);
    ctx = EVP_PKEY_CTX_new(pkey, NULL);
    tappa_printf("Print %d", 2);
    if (!ctx) {
        cleanup(ctx, NULL, NULL, NULL);
        return 1000;
    }

    // Init encryption context.
    if (EVP_PKEY_encrypt_init(ctx) <= 0) { // <-SEGFAULT LINE
        cleanup(ctx, pkey, encrypted, NULL);
        return 5;
    }

    tappa_printf("Print %d", 3);
    // Defines padding
    if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING) <= 0) {
        cleanup(ctx, pkey, encrypted, NULL);
        return 6;
    }

    // Evaluating size for encrypted buffer and encrypting
    if (EVP_PKEY_encrypt(ctx, NULL, &outlen, (const unsigned char *) data, inlen) <= 0) {
        cleanup(ctx, pkey, encrypted, NULL);
        return 7;
    }

    encrypted = (unsigned char *)OPENSSL_malloc(outlen);
    tappa_printf("Print %d", 5);
    if (!encrypted) {
        cleanup(ctx, pkey, encrypted, NULL);
        return 8;
    }

    tappa_printf("Print %d", 6);
    if (EVP_PKEY_encrypt(ctx, encrypted, &outlen, (const unsigned char *)data, inlen) <= 0) {
        cleanup(ctx, pkey, encrypted, NULL);
        return 9;
    }

    tappa_print(""ENCRYPTED: %s", (char *)encrypted);
}

and this function to generate a random key

int rsaEvpKeyGen(EVP_PKEY **outPK){
    EVP_PKEY_CTX * ctx = NULL;
    EVP_PKEY * pkey = NULL;
    EVP_PKEY * rkey = NULL;

    // const size_t inlen = strlen(argv[1]);
    size_t outlen = 0;
    size_t doutlen = 0;

    unsigned char * encrypted = NULL;
    unsigned char * decrypted = NULL;

    // Create context
     ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, NULL);
     if (!ctx) {
         cleanup(ctx, pkey, encrypted, decrypted);
         return 1;
     }
     // Init key generator
     if (EVP_PKEY_keygen_init(ctx) <= 0) {
         cleanup(ctx, pkey, encrypted, decrypted);
         return 2;
     }
     int KEY_BITS = 2048;
     // Configure key generation
     if (EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, KEY_BITS) <= 0) {
         cleanup(ctx, pkey, encrypted, decrypted);
         return 3;
     }
 
     // Create keys
     if (EVP_PKEY_keygen(ctx, &pkey) <= 0) {
         cleanup(ctx, pkey, encrypted, decrypted);
         return 4;
     }
     
    *outPK =pkey;
    return 0;
}

The code works as expected and encrypts the data as expected. However for my application to work, I need to be able to use a Public key generated externally.

In the following code, I attempt to create an EVP_PKEY from pub_key_str using EVP_PKEY_new_raw_public_key but it returns NULL

int rsaEvpKeyGen(EVP_PKEY **outPK){
    EVP_PKEY_CTX * ctx = NULL;
    EVP_PKEY * pkey = NULL;
    EVP_PKEY * rkey = NULL;

    // const size_t inlen = strlen(argv[1]);
    size_t outlen = 0;
    size_t doutlen = 0;

    unsigned char * encrypted = NULL;
    unsigned char * decrypted = NULL;
    const char *pub_key_str = "-----BEGIN PUBLIC KEY-----n"
                              "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1WRkqlsZbmaNWbOZr/M4n"
                              "ddwXKjmHUJ8zn9hy0WE15A8T4xjNuzJ/ATKbz20/2wVgFEGTWPNCUjkUvFzr3mhNn"
                              "v/YJXo5iXQDP3M6rVv/w3SXaiR9C8yhDxWUz7WX/clmj7vDvk5Iydgq07gVScx+n"
                              "5DBVbGAx10jHSDSa7f/NXMn9yNHad4hN0i1l+tsTB39CFsNeVFr349y4R8HA5Zman"
                              "EKseLc8iKzkwEQcuyMn4znaFpnOL0CmSrYB5K1E9zmmtDhMvDs540ZotcH/xpJiVn"
                              "XEdQOWGFN6rryeAKP8y+sMRfQLxoiTg37YfsxTscf0gzKBizoamTF3TjRUNM55aEn"
                              "BwIDAQABn"
                              "-----END PUBLIC KEY-----";

    pkey = EVP_PKEY_new_raw_public_key(EVP_PKEY_RSA,NULL, (const unsigned char *) pub_key_str,
                                       strlen(pub_key_str) );

    tappa_printf("PKEY is null %d",pkey == NULL);
  
    *outPK =pkey;
    
    return 0;
}