I’m working on a C program that uses the OpenSSL library to perform several cryptographic operations. However, I’m encountering an issue when attempting to decrypt a file using AES-128-ECB. The decryption process fails with the error EVP_DecryptFinal_ex:bad decrypt.

Here are the details of my implementation:

Decrypt a file (s83660-cipher.bin) using a key (s83660-source-key.bin) with AES-128-ECB.
Compute the SHA512/224 hash of the decrypted file and compare it with a provided hash (s83660-   digest.bin).
Encrypt the decrypted file with Camellia-256-OFB using another key (s83660-dest-key.bin).

#include <openssl/evp.h>
#include <openssl/aes.h>
#include <openssl/camellia.h>
#include <openssl/sha.h>
#include <openssl/err.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#define BUFFER_SIZE 1024
#define SHA512_224_DIGEST_LENGTH 28

void handleErrors(void) {
    ERR_print_errors_fp(stderr);
    abort();
}

void decrypt_aes_128_ecb(const char *cipher_file, const char *key_file, const char *output_file) {
    FILE *cf = fopen(cipher_file, "rb");
    FILE *kf = fopen(key_file, "rb");
    FILE *of = fopen(output_file, "wb");

    if (!cf || !kf || !of) {
        perror("File error");
        exit(EXIT_FAILURE);
    }

    // Read key
    unsigned char key[16];
    if (fread(key, 1, 16, kf) != 16) {
        fprintf(stderr, "Error reading keyn");
        exit(EXIT_FAILURE);
    }
    key[0] = 0;  // Salting

    // Print key for debugging
    printf("Key after modification: ");
    for (int i = 0; i < 16; i++) {
        printf("%02x", key[i]);
    }
    printf("n");

    // Initialize decryption
    EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new();
    if (!ctx) handleErrors();

    // Disable padding for manual handling
    if (EVP_CIPHER_CTX_set_padding(ctx, 0) != 1) handleErrors();

    if (EVP_DecryptInit_ex(ctx, EVP_aes_128_ecb(), NULL, key, NULL) != 1) handleErrors();

    unsigned char buffer[BUFFER_SIZE];
    unsigned char plaintext[BUFFER_SIZE + EVP_MAX_BLOCK_LENGTH];
    int len;
    int plaintext_len = 0;

    while ((len = fread(buffer, 1, BUFFER_SIZE, cf)) > 0) {
        int out_len;
        if (EVP_DecryptUpdate(ctx, plaintext, &out_len, buffer, len) != 1) handleErrors();
        fwrite(plaintext, 1, out_len, of);
        plaintext_len += out_len;
    }

    // Print intermediate plaintext length for debugging
    printf("Intermediate plaintext length: %dn", plaintext_len);

    if (EVP_DecryptFinal_ex(ctx, plaintext, &len) != 1) {
        // Print specific OpenSSL error message
        ERR_print_errors_fp(stderr);
        handleErrors();
    }
    fwrite(plaintext, 1, len, of);
    plaintext_len += len;

    printf("Final plaintext length: %dn", plaintext_len);

    // Remove padding manually
    int pad_len = plaintext[plaintext_len - 1];
    if (pad_len > 0 && pad_len <= AES_BLOCK_SIZE) {
        plaintext_len -= pad_len;
        ftruncate(fileno(of), plaintext_len);  // Adjust file length to remove padding
    }

    EVP_CIPHER_CTX_free(ctx);
    fclose(cf);
    fclose(kf);
    fclose(of);
}

void compute_sha512_224(const char *input_file, const char *digest_file) {
    FILE *inf = fopen(input_file, "rb");
    FILE *df = fopen(digest_file, "rb");

    if (!inf || !df) {
        perror("File error");
        exit(EXIT_FAILURE);
    }

    EVP_MD_CTX *mdctx = EVP_MD_CTX_new();
    if (!mdctx) handleErrors();

    if (EVP_DigestInit_ex(mdctx, EVP_sha512_224(), NULL) != 1) handleErrors();

    unsigned char buffer[BUFFER_SIZE];
    int len;
    while ((len = fread(buffer, 1, BUFFER_SIZE, inf)) > 0) {
        if (EVP_DigestUpdate(mdctx, buffer, len) != 1) handleErrors();
    }

    unsigned char hash[SHA512_224_DIGEST_LENGTH];
    unsigned int hash_len;
    if (EVP_DigestFinal_ex(mdctx, hash, &hash_len) != 1) handleErrors();
    EVP_MD_CTX_free(mdctx);

    unsigned char expected_hash[SHA512_224_DIGEST_LENGTH];
    if (fread(expected_hash, 1, SHA512_224_DIGEST_LENGTH, df) != SHA512_224_DIGEST_LENGTH) {
        fprintf(stderr, "Error reading expected hashn");
        exit(EXIT_FAILURE);
    }

    if (memcmp(hash, expected_hash, SHA512_224_DIGEST_LENGTH) == 0) {
        printf("Hashes matchn");
    } else {
        printf("Hashes do not matchn");
    }

    fclose(inf);
    fclose(df);
}

void encrypt_camellia_256_ofb(const char *input_file, const char *key_file, const char *output_file) {
    FILE *inf = fopen(input_file, "rb");
    FILE *kf = fopen(key_file, "rb");
    FILE *of = fopen(output_file, "wb");

    if (!inf || !kf || !of) {
        perror("File error");
        exit(EXIT_FAILURE);
    }

    unsigned char key[32];
    unsigned char iv[16] = {0};  // Initialize IV to 0

    if (fread(key, 1, 32, kf) != 32) {
        fprintf(stderr, "Error reading keyn");
        exit(EXIT_FAILURE);
    }

    EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new();
    if (!ctx) handleErrors();

    if (EVP_EncryptInit_ex(ctx, EVP_camellia_256_ofb(), NULL, key, iv) != 1) handleErrors();

    unsigned char buffer[BUFFER_SIZE];
    unsigned char ciphertext[BUFFER_SIZE + EVP_MAX_BLOCK_LENGTH];
    int len;
    int ciphertext_len = 0;

    while ((len = fread(buffer, 1, BUFFER_SIZE, inf)) > 0) {
        int out_len;
        if (EVP_EncryptUpdate(ctx, ciphertext, &out_len, buffer, len) != 1) handleErrors();
        fwrite(ciphertext, 1, out_len, of);
        ciphertext_len += out_len;
    }

    if (EVP_EncryptFinal_ex(ctx, ciphertext, &len) != 1) handleErrors();
    fwrite(ciphertext, 1, len, of);
    ciphertext_len += len;

    EVP_CIPHER_CTX_free(ctx);
    fclose(inf);
    fclose(kf);
    fclose(of);
}

int main(void) {
    decrypt_aes_128_ecb("s83660-cipher.bin", "s83660-source-key.bin", "decrypted.jpg");
    compute_sha512_224("decrypted.jpg", "s83660-digest.bin");
    encrypt_camellia_256_ofb("decrypted.jpg", "s83660-dest-key.bin", "s83660-dest-cipher.bin");

    return 0;
}

Output

Key after modification: 000db08f3b6c505d76e9e88962e94c57
Intermediate plaintext length: 102224
139956396554048:error:06065064:digital envelope routines:EVP_DecryptFinal_ex:bad decrypt:crypto/evp/evp_enc.c:646:
Aborted (core dumped)

Steps I’ve Taken:

Verified the key length is 16 bytes.
Corrected the first byte of the key as instructed.
Attempted to disable automatic padding and handle it manually.

My Environment:

OpenSSL version: OpenSSL 1.1.1
GCC version: 9.3.0
OS: Ubuntu 20.04 LTS

My Questions:

Why am I getting the EVP_DecryptFinal_ex:bad decrypt error even though the key and data seem correct?
Is there an issue with the way I'm handling the padding manually?
Are there additional steps I can take to debug this issue?

Any help or insights would be greatly appreciated. Thank you!