C - Openssl (d2i_RSA_PUBKEY, d2i_RSAPrivateKey and d2i_RSAPublicKey)

Question

I created a private RSA key via the following command:

 openssl genrsa -out keypair.pem 2048

I need these keys to be stored in DER format (PKCS#1). Thus, I converted this private key, which is in PEM format, into two files in DER format: one for the private key, the other for the public key:

openssl rsa -inform PEM -in keypair.pem -outform DER -pubout -out public.der

openssl rsa -inform PEM -in keypair.pem -outform DER -out private.der

In my code, I loaded the contents of public.der and private.der into two char variables*.

None of the calls below work:

d2i_RSAPublicKey(NULL, &public_key_bytes, public_key_length);

d2i_RSAPublicKey(NULL, &public_key_bytes, public_key_length);

d2i_RSAPrivateKey(NULL, &private_key_bytes, private_key_length);

I know the calls don’t work because the return of all of them is null.

I also tried the following:

RSA * rsa = RSA_new();
d2i_RSAPublicKey(&rsa, &public_key_bytes, public_key_length);

RSA * rsa = RSA_new();
d2i_RSAPublicKey(&rsa, &public_key_bytes, public_key_length);

RSA * rsa = RSA_new();
d2i_RSAPrivateKey(&rsa, &private_key_bytes, private_key_length);

The same result, all return null.

My test code is as follows::

#include <stdio.h>
#include <stdlib.h>

#include <openssl/rsa.h>
#include <openssl/bio.h>
#include <openssl/pem.h>

typedef struct
{
    int len;
    char * bytes;
} FileData;

static FileData readFileBytes(const char * name, int zero_ended)
{
    FILE * fl = fopen(name, "r");
    if (fl == NULL) return (FileData) { .len = 0, .bytes = NULL };
    fseek(fl, 0, SEEK_END);
    long len = ftell(fl);
    char * ret = malloc(len + (zero_ended ? 1 : 0));
    fseek(fl, 0, SEEK_SET);
    fread(ret, 1, len, fl);
    if (zero_ended) ret[len] = 0;
    fclose(fl);
    return (FileData) { .len = len, .bytes = ret };
}

int main()
{
    FileData private_key = readFileBytes("../private.der", 0);
    FileData public_key = readFileBytes("../public.der", 0);

    char* public_key_bytes = public_key.bytes;
    int public_key_length = public_key.len;

    char* private_key_bytes = private_key.bytes;
    int private_key_length = private_key.len;

    RSA * rsa;

    public_key_bytes = public_key.bytes;
    public_key_length = public_key.len;
    rsa = d2i_RSA_PUBKEY(NULL, &public_key_bytes, public_key_length);
    printf("d2i_RSA_PUBKEY(NULL, &public_key_bytes, public_key_length) != NULL -> %s\n", (rsa != NULL) ? "true" : "false");

    public_key_bytes = public_key.bytes;
    public_key_length = public_key.len;
    rsa = d2i_RSAPublicKey(NULL, &public_key_bytes, public_key_length);
    printf("d2i_RSAPublicKey(NULL, &public_key_bytes, public_key_length) != NULL -> %s\n", (rsa != NULL) ? "true" : "false");

    private_key_bytes = private_key.bytes;
    private_key_length = private_key.len;
    rsa = d2i_RSAPrivateKey(NULL, &private_key_bytes, private_key_length);
    printf("d2i_RSAPrivateKey(NULL, &private_key_bytes, private_key_length) != NULL -> %s\n", (rsa != NULL) ? "true" : "false");

    public_key_bytes = public_key.bytes;
    public_key_length = public_key.len;
    rsa = RSA_new();
    rsa = d2i_RSA_PUBKEY(&rsa, &public_key_bytes, public_key_length);
    printf("d2i_RSA_PUBKEY(&rsa, &public_key_bytes, public_key_length) != NULL -> %s\n", (rsa != NULL) ? "true" : "false");

    public_key_bytes = public_key.bytes;
    public_key_length = public_key.len;
    rsa = RSA_new();
    rsa = d2i_RSAPublicKey(&rsa, &public_key_bytes, public_key_length);
    printf("d2i_RSAPublicKey(&rsa, &public_key_bytes, public_key_length) != NULL -> %s\n", (rsa != NULL) ? "true" : "false");

    private_key_bytes = private_key.bytes;
    private_key_length = private_key.len;
    rsa = RSA_new();
    rsa = d2i_RSAPrivateKey(&rsa, &private_key_bytes, private_key_length);
    printf("d2i_RSAPrivateKey(&rsa, &private_key_bytes, private_key_length) != NULL -> %s\n", (rsa != NULL) ? "true" : "false");

    getchar();

    return 0;
}

What am I doing wrong?

Answer 1

The first parameter of d2i_RSAPublicKey cannot be null, it has to be an RSA pointer.

RSA *rsa = RSA_new() ;
d2i_RSAPublicKey(rsa, &public_key_bytes, public_key_length);

https://www.openssl.org/docs/manmaster/crypto/i2d_RSA_PUBKEY.html

Answer 2

In Stackoverflow I got the answer (https://stackoverflow.com/a/36805530/6242158).

I was running this code on Windows, and for Windows to handle binary files properly, just use the "Rb".