From 402cffdeb29606eb1310d3dab12e95453b139322 Mon Sep 17 00:00:00 2001 From: akallabeth Date: Tue, 25 Apr 2023 09:46:00 +0200 Subject: [PATCH] [emu,scard] openssl 3 support --- libfreerdp/emu/scard/smartcard_virtual_gids.c | 69 ++++++++----------- 1 file changed, 28 insertions(+), 41 deletions(-) diff --git a/libfreerdp/emu/scard/smartcard_virtual_gids.c b/libfreerdp/emu/scard/smartcard_virtual_gids.c index 592a771dc..bbfdee4ed 100644 --- a/libfreerdp/emu/scard/smartcard_virtual_gids.c +++ b/libfreerdp/emu/scard/smartcard_virtual_gids.c @@ -503,13 +503,8 @@ handle_error: static int get_rsa_key_size(const rdpPrivateKey* privateKey) { WINPR_ASSERT(privateKey); - RSA* rsa = freerdp_key_get_RSA(privateKey); - if (!rsa) - return -1; - const int size = RSA_size(rsa); - RSA_free(rsa); - return size; + return freerdp_key_get_bits(privateKey); } static BYTE vgids_get_algid(vgidsContext* p_Ctx) @@ -801,28 +796,20 @@ static UINT16 vgids_handle_chained_response(vgidsContext* context, const BYTE** static BOOL vgids_get_public_key(vgidsContext* context, UINT16 doTag) { BOOL rc = FALSE; - BYTE* buf = NULL; wStream* pubKey = NULL; wStream* response = NULL; WINPR_ASSERT(context); /* Get key components */ - RSA* rsa = freerdp_certificate_get_RSA(context->certificate); - if (!rsa) - return FALSE; - const BIGNUM *n, *e; - RSA_get0_key(rsa, &n, &e, NULL); + size_t nSize = 0; + size_t eSize = 0; - const size_t nSize = BN_num_bytes(n); - const size_t eSize = BN_num_bytes(e); + char* n = freerdp_certificate_get_param(context->certificate, FREERDP_CERT_RSA_N, &nSize); + char* e = freerdp_certificate_get_param(context->certificate, FREERDP_CERT_RSA_E, &eSize); - buf = malloc(nSize > eSize ? nSize : eSize); - if (!buf) - { - WLog_ERR(TAG, "Failed to allocate buffer for public key"); + if (!n || !e) goto handle_error; - } pubKey = Stream_New(NULL, nSize + eSize + 0x10); if (!pubKey) @@ -839,12 +826,10 @@ static BOOL vgids_get_public_key(vgidsContext* context, UINT16 doTag) } /* write modulus and exponent DOs */ - BN_bn2bin(n, buf); - if (!vgids_write_tlv(pubKey, 0x81, buf, nSize)) + if (!vgids_write_tlv(pubKey, 0x81, n, nSize)) goto handle_error; - BN_bn2bin(e, buf); - if (!vgids_write_tlv(pubKey, 0x82, buf, eSize)) + if (!vgids_write_tlv(pubKey, 0x82, e, eSize)) goto handle_error; /* write ISO public key template */ @@ -857,8 +842,8 @@ static BOOL vgids_get_public_key(vgidsContext* context, UINT16 doTag) rc = TRUE; handle_error: - free(buf); - RSA_free(rsa); + free(n); + free(e); Stream_Free(pubKey, TRUE); if (!rc) Stream_Free(response, TRUE); @@ -1043,7 +1028,7 @@ static BOOL vgids_perform_digital_signature(vgidsContext* context) { size_t sigSize, msgSize; EVP_PKEY_CTX* ctx = NULL; - EVP_PKEY* pk = EVP_PKEY_new(); + EVP_PKEY* pk = freerdp_key_get_evp_pkey(context->privateKey); const vgidsDigestInfoMap gidsDigestInfo[VGIDS_MAX_DIGEST_INFO] = { { g_PKCS1_SHA1, sizeof(g_PKCS1_SHA1), EVP_sha1() }, { g_PKCS1_SHA224, sizeof(g_PKCS1_SHA224), EVP_sha224() }, @@ -1060,11 +1045,6 @@ static BOOL vgids_perform_digital_signature(vgidsContext* context) return FALSE; } - RSA* rsa = freerdp_key_get_RSA(context->privateKey); - if (!rsa) - return FALSE; - - EVP_PKEY_set1_RSA(pk, rsa); vgids_reset_context_response(context); /* for each digest info */ @@ -1142,7 +1122,6 @@ static BOOL vgids_perform_digital_signature(vgidsContext* context) } EVP_PKEY_free(pk); - RSA_free(rsa); vgids_reset_context_command_data(context); return TRUE; @@ -1151,7 +1130,6 @@ sign_failed: vgids_reset_context_response(context); EVP_PKEY_CTX_free(ctx); EVP_PKEY_free(pk); - RSA_free(rsa); return FALSE; } @@ -1170,11 +1148,18 @@ static BOOL vgids_perform_decrypt(vgidsContext* context) padding = RSA_PKCS1_OAEP_PADDING; /* init response buffer */ - RSA* rsa = freerdp_key_get_RSA(context->privateKey); - if (!rsa) + EVP_PKEY* pkey = freerdp_key_get_evp_pkey(context->privateKey); + if (!pkey) + goto decrypt_failed; + EVP_PKEY_CTX* ctx = EVP_PKEY_CTX_new(pkey, NULL); + if (!ctx) + goto decrypt_failed; + if (EVP_PKEY_decrypt_init(ctx) <= 0) + goto decrypt_failed; + if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING) <= 0) goto decrypt_failed; - context->responseData = Stream_New(NULL, RSA_size(rsa)); + context->responseData = Stream_New(NULL, freerdp_key_get_bits(context->privateKey)); if (!context->responseData) { WLog_ERR(TAG, "Failed to create decryption buffer"); @@ -1182,9 +1167,10 @@ static BOOL vgids_perform_decrypt(vgidsContext* context) } /* Determine buffer length */ - res = RSA_private_decrypt((int)Stream_Length(context->commandData), - Stream_Buffer(context->commandData), - Stream_Buffer(context->responseData), rsa, padding); + size_t outlen = Stream_Capacity(context->responseData); + const size_t inlen = Stream_Length(context->commandData); + res = EVP_PKEY_decrypt(ctx, Stream_Buffer(context->responseData), &outlen, + Stream_Buffer(context->commandData), inlen); if (res < 0) { @@ -1192,11 +1178,12 @@ static BOOL vgids_perform_decrypt(vgidsContext* context) goto decrypt_failed; } - Stream_SetLength(context->responseData, res); + Stream_SetLength(context->responseData, outlen); rc = TRUE; decrypt_failed: - RSA_free(rsa); + EVP_PKEY_CTX_free(ctx); + EVP_PKEY_free(pkey); vgids_reset_context_command_data(context); if (!rc) vgids_reset_context_response(context);