I have two files: ca.crt and ca.key, generated using this commands:

openssl genrsa -out ca.key

openssl req -x509 -new -key ca.key -days 1825 -out ca.crt -subj “/CN=Root CA

Then I installed the ca.crt certificate in the “Trusted Root Certificate Authorities” on Windows

And then I used this code to setup openssl:

SSL_library_init();
SSLeay_add_ssl_algorithms();
SSL_load_error_strings();

SSL_CTX* ctx = SSL_CTX_new(SSLv3_method());
SSL* ssl = SSL_new(ctx);
if (!ssl) {
    printf("ssl errorn");
    return -1;
}

SSL_set_fd(ssl, s);

if (SSL_CTX_load_verify_locations(ctx, R"(mypathtoca.crt)", nullptr) <= 0) {
    ERR_print_errors_fp(stderr);
    printf("verify errn");
}

if (!SSL_CTX_use_PrivateKey_file(ctx, R"(mypathtoca.key)", SSL_FILETYPE_PEM))
    printf("cert err 2n");

int err = SSL_connect(ssl);
if (err <= 0) {
    printf("errno: %Xn", errno);
    printf("error creating ssl connection: %dn", err);
    printf("%Xn", SSL_get_error(ssl, err));
    return -1;
}

Output:

errno: 0
error creating ssl connection: -1
1