hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-28 10:15:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

Crypto: Fix openssl padding to propery link async padding to hashing configs.

Browse Source
This commit is contained in:
REDMOND\brodes
2025-06-26 09:18:30 -04:00
parent 7559c06fdb
commit 6e5734adca
10 changed files with 188 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cpp/ql/lib/experimental/quantum/OpenSSL/AlgorithmInstances/AlgToAVCFlow.qll
View File

@@ -49,7 +49,9 @@ module KnownOpenSslAlgorithmToAlgorithmValueConsumerFlow =
DataFlow::Global<KnownOpenSslAlgorithmToAlgorithmValueConsumerConfig>;
module RSAPaddingAlgorithmToPaddingAlgorithmValueConsumerConfig implements DataFlow::ConfigSig {
predicate isSource(DataFlow::Node source) { source.asExpr() instanceof OpenSslPaddingLiteral }
predicate isSource(DataFlow::Node source) {
source.asExpr() instanceof OpenSslSpecialPaddingLiteral
}
predicate isSink(DataFlow::Node sink) {
exists(PaddingAlgorithmValueConsumer c | c.getInputNode() = sink)

14
cpp/ql/lib/experimental/quantum/OpenSSL/AlgorithmInstances/CipherAlgorithmInstance.qll
View File

@@ -4,8 +4,9 @@ private import KnownAlgorithmConstants
private import Crypto::KeyOpAlg as KeyOpAlg
private import OpenSSLAlgorithmInstanceBase
private import PaddingAlgorithmInstance
private import experimental.quantum.OpenSSL.AlgorithmValueConsumers.OpenSSLAlgorithmValueConsumerBase
private import experimental.quantum.OpenSSL.AlgorithmValueConsumers.DirectAlgorithmValueConsumer
private import experimental.quantum.OpenSSL.Operations.OpenSSLOperationBase
private import experimental.quantum.OpenSSL.AlgorithmValueConsumers.OpenSSLAlgorithmValueConsumers
private import OpenSSLAlgorithmInstances
private import AlgToAVCFlow
private import BlockAlgorithmInstance
@@ -97,10 +98,13 @@ class KnownOpenSslCipherConstantAlgorithmInstance extends OpenSslAlgorithmInstan
}
override Crypto::PaddingAlgorithmInstance getPaddingAlgorithm() {
//TODO: the padding is either self, or it flows through getter ctx to a set padding call
// like EVP_PKEY_CTX_set_rsa_padding
result = this
// TODO or trace through getter ctx to set padding
or
exists(OperationStep s |
this.getAvc().(AvcContextCreationStep).flowsToOperationStep(s) and
s.getAlgorithmValueConsumerForInput(PaddingAlgorithmIO()) =
result.(OpenSslAlgorithmInstance).getAvc()
)
}
override string getRawAlgorithmName() {

61
cpp/ql/lib/experimental/quantum/OpenSSL/AlgorithmInstances/PaddingAlgorithmInstance.qll
View File

@@ -1,6 +1,7 @@
import cpp
private import experimental.quantum.Language
private import OpenSSLAlgorithmInstanceBase
private import experimental.quantum.OpenSSL.Operations.OpenSSLOperationBase
private import experimental.quantum.OpenSSL.AlgorithmInstances.KnownAlgorithmConstants
private import AlgToAVCFlow
private import experimental.quantum.OpenSSL.AlgorithmValueConsumers.DirectAlgorithmValueConsumer
@@ -17,13 +18,14 @@ private import experimental.quantum.OpenSSL.AlgorithmValueConsumers.OpenSSLAlgor
* # define RSA_PKCS1_WITH_TLS_PADDING 7
* # define RSA_PKCS1_NO_IMPLICIT_REJECT_PADDING 8
*/
class OpenSslPaddingLiteral extends Literal {
class OpenSslSpecialPaddingLiteral extends Literal {
// TODO: we can be more specific about where the literal is in a larger expression
// to avoid literals that are clealy not representing an algorithm, e.g., array indices.
OpenSslPaddingLiteral() { this.getValue().toInt() in [0, 1, 3, 4, 5, 6, 7, 8] }
OpenSslSpecialPaddingLiteral() { this.getValue().toInt() in [0, 1, 3, 4, 5, 6, 7, 8] }
}
/**
* Holds if `e` has the given `type`.
* Given a `KnownOpenSslPaddingAlgorithmExpr`, converts this to a padding family type.
* Does not bind if there is no mapping (no mapping to 'unknown' or 'other').
*/
@@ -44,9 +46,6 @@ predicate knownOpenSslConstantToPaddingFamilyType(
)
}
//abstract class OpenSslPaddingAlgorithmInstance extends OpenSslAlgorithmInstance, Crypto::PaddingAlgorithmInstance{}
// TODO: need to alter this to include known padding constants which don't have the
// same mechanics as those with known nids
class KnownOpenSslPaddingConstantAlgorithmInstance extends OpenSslAlgorithmInstance,
Crypto::PaddingAlgorithmInstance instanceof Expr
{
@@ -78,7 +77,7 @@ class KnownOpenSslPaddingConstantAlgorithmInstance extends OpenSslAlgorithmInsta
isPaddingSpecificConsumer = false
or
// Possibility 3: padding-specific literal
this instanceof OpenSslPaddingLiteral and
this instanceof OpenSslSpecialPaddingLiteral and
exists(DataFlow::Node src, DataFlow::Node sink |
// Sink is an argument to a CipherGetterCall
sink = getterCall.getInputNode() and
@@ -123,44 +122,6 @@ class KnownOpenSslPaddingConstantAlgorithmInstance extends OpenSslAlgorithmInsta
}
}
// // Values used for EVP_PKEY_CTX_set_rsa_padding, these are
// // not the same as 'typical' constants found in the set of known algorithm constants
// // they do not have an NID
// // TODO: what about setting the padding directly?
// class KnownRSAPaddingConstant extends OpenSslPaddingAlgorithmInstance, Crypto::PaddingAlgorithmInstance instanceof Literal
// {
// KnownRSAPaddingConstant() {
// // from rsa.h in openssl:
// // # define RSA_PKCS1_PADDING 1
// // # define RSA_NO_PADDING 3
// // # define RSA_PKCS1_OAEP_PADDING 4
// // # define RSA_X931_PADDING 5
// // /* EVP_PKEY_ only */
// // # define RSA_PKCS1_PSS_PADDING 6
// // # define RSA_PKCS1_WITH_TLS_PADDING 7
// // /* internal RSA_ only */
// // # define RSA_PKCS1_NO_IMPLICIT_REJECT_PADDING 8
// this instanceof Literal and
// this.getValue().toInt() in [0, 1, 3, 4, 5, 6, 7, 8]
// // TODO: trace to padding-specific consumers
// RSAPaddingAlgorithmToPaddingAlgorithmValueConsumerFlow
// }
// override string getRawPaddingAlgorithmName() { result = this.(Literal).getValue().toString() }
// override Crypto::TPaddingType getPaddingType() {
// if this.(Literal).getValue().toInt() in [1, 6, 7, 8]
// then result = Crypto::PKCS1_v1_5()
// else
// if this.(Literal).getValue().toInt() = 3
// then result = Crypto::NoPadding()
// else
// if this.(Literal).getValue().toInt() = 4
// then result = Crypto::OAEP()
// else
// if this.(Literal).getValue().toInt() = 5
// then result = Crypto::ANSI_X9_23()
// else result = Crypto::OtherPadding()
// }
// }
class OAEPPaddingAlgorithmInstance extends Crypto::OAEPPaddingAlgorithmInstance,
KnownOpenSslPaddingConstantAlgorithmInstance
{
@@ -169,10 +130,18 @@ class OAEPPaddingAlgorithmInstance extends Crypto::OAEPPaddingAlgorithmInstance,
}
override Crypto::HashAlgorithmInstance getOAEPEncodingHashAlgorithm() {
none() //TODO
exists(OperationStep s |
this.getAvc().(AvcContextCreationStep).flowsToOperationStep(s) and
s.getAlgorithmValueConsumerForInput(HashAlgorithmOaepIO()) =
result.(OpenSslAlgorithmInstance).getAvc()
)
}
override Crypto::HashAlgorithmInstance getMGF1HashAlgorithm() {
none() //TODO
exists(OperationStep s |
this.getAvc().(AvcContextCreationStep).flowsToOperationStep(s) and
s.getAlgorithmValueConsumerForInput(HashAlgorithmMgf1IO()) =
result.(OpenSslAlgorithmInstance).getAvc()
)
}
}

26
cpp/ql/lib/experimental/quantum/OpenSSL/Operations/CipherOperation.qll
View File

@@ -214,7 +214,14 @@ class EvpCipherFinalCall extends EvpCipherOperationFinalStep {
*/
class EvpPKeyCipherOperation extends EvpCipherOperationFinalStep {
EvpPKeyCipherOperation() {
this.getTarget().getName() in ["EVP_PKEY_encrypt", "EVP_PKEY_decrypt"]
this.getTarget().getName() in ["EVP_PKEY_encrypt", "EVP_PKEY_decrypt"] and
// TODO: for now ignore this operation entirely if it is setting the cipher text to null
// this needs to be re-evalauted if this scenario sets other values worth tracking
(
exists(this.(Call).getArgument(1).getValue())
implies
this.(Call).getArgument(1).getValue().toInt() != 0
)
}
override DataFlow::Node getInput(IOType type) {
@@ -226,9 +233,24 @@ class EvpPKeyCipherOperation extends EvpCipherOperationFinalStep {
override DataFlow::Node getOutput(IOType type) {
super.getOutput(type) = result
or
result.asExpr() = this.getArgument(1) and type = CiphertextIO()
result.asExpr() = this.getArgument(1) and
type = CiphertextIO() and
this.getStepType() = FinalStep()
// TODO: could indicate text lengths here, as well
}
override OperationStepType getStepType() {
// When the output buffer is null, the step is not a final step
// it is used to get the buffer size, if 0 consider it an initialization step
// NOTE/TODO: not tracing 0 to the arg, just looking for 0 directly in param
// the assumption is this is the common case, but we may want to make this more
// robust and support a dataflow.
result = FinalStep() and
(exists(super.getArgument(1).getValue()) implies super.getArgument(1).getValue().toInt() != 0)
or
result = InitializerStep() and
super.getArgument(1).getValue().toInt() = 0
}
}
/**

33
cpp/ql/lib/experimental/quantum/OpenSSL/Operations/EVPPKeyCtxInitializer.qll
View File

@@ -71,19 +71,42 @@ class EvpCtxSetEcParamgenCurveNidInitializer extends OperationStep {
* - `EVP_PKEY_CTX_set_ecdh_kdf_md`
*/
class EvpCtxSetHashInitializer extends OperationStep {
boolean isOaep;
boolean isMgf1;
EvpCtxSetHashInitializer() {
this.getTarget().getName() in [
"EVP_PKEY_CTX_set_signature_md", "EVP_PKEY_CTX_set_rsa_mgf1_md_name",
"EVP_PKEY_CTX_set_rsa_mgf1_md", "EVP_PKEY_CTX_set_rsa_oaep_md_name",
"EVP_PKEY_CTX_set_rsa_oaep_md", "EVP_PKEY_CTX_set_dsa_paramgen_md",
"EVP_PKEY_CTX_set_signature_md", "EVP_PKEY_CTX_set_dsa_paramgen_md",
"EVP_PKEY_CTX_set_dh_kdf_md", "EVP_PKEY_CTX_set_ecdh_kdf_md"
]
] and
isOaep = false and
isMgf1 = false
or
this.getTarget().getName() in [
"EVP_PKEY_CTX_set_rsa_mgf1_md_name", "EVP_PKEY_CTX_set_rsa_mgf1_md"
] and
isOaep = false and
isMgf1 = true
or
this.getTarget().getName() in [
"EVP_PKEY_CTX_set_rsa_oaep_md_name",
"EVP_PKEY_CTX_set_rsa_oaep_md"
] and
isOaep = true and
isMgf1 = false
}
override DataFlow::Node getInput(IOType type) {
result.asExpr() = this.getArgument(0) and type = ContextIO()
or
result.asExpr() = this.getArgument(1) and type = HashAlgorithmIO()
result.asExpr() = this.getArgument(1) and
type = HashAlgorithmIO() and
isOaep = false and
isMgf1 = false
or
result.asExpr() = this.getArgument(1) and type = HashAlgorithmOaepIO() and isOaep = true
or
result.asExpr() = this.getArgument(1) and type = HashAlgorithmMgf1IO() and isMgf1 = true
}
override DataFlow::Node getOutput(IOType type) {

18
cpp/ql/lib/experimental/quantum/OpenSSL/Operations/OpenSSLOperationBase.qll
View File

@@ -58,7 +58,11 @@ newtype TIOType =
// For OSSL_PARAM and OSSL_LIB_CTX use of OsslParamIO and OsslLibContextIO
ContextIO() or
DigestIO() or
// For OAEP and MGF1 hashes, there is a special IO type for these hashes
// it is recommended to set the most explicit type known, not both
HashAlgorithmIO() or
HashAlgorithmOaepIO() or
HashAlgorithmMgf1IO() or
IVorNonceIO() or
KeyIO() or
KeyOperationSubtypeIO() or
@@ -254,18 +258,18 @@ abstract class OperationStep extends Call {
* operation step (dominating operation step, see `getDominatingInitializersToStep`).
*/
Crypto::AlgorithmValueConsumer getPrimaryAlgorithmValueConsumer() {
exists(DataFlow::Node src, DataFlow::Node sink, IOType t, OperationStep avcSucc |
exists(DataFlow::Node src, DataFlow::Node sink, IOType t, OperationStep avcConsumingPred |
(t = PrimaryAlgorithmIO() or t = ContextIO()) and
avcSucc.flowsToOperationStep(this) and
avcConsumingPred.flowsToOperationStep(this) and
src.asExpr() = result and
sink = avcSucc.getInput(t) and
sink = avcConsumingPred.getInput(t) and
AvcToOperationStepFlow::flow(src, sink) and
(
// Case 1: the avcSucc step is a dominating initialization step
// Case 1: the avcConsumingPred step is a dominating initialization step
t = PrimaryAlgorithmIO() and
avcSucc = this.getDominatingInitializersToStep(PrimaryAlgorithmIO())
avcConsumingPred = this.getDominatingInitializersToStep(PrimaryAlgorithmIO())
or
// Case 2: the succ is a context input (any avcSucc is valid)
// Case 2: the pred is a context input
t = ContextIO()
)
)
@@ -277,6 +281,8 @@ abstract class OperationStep extends Call {
* TODO: generalize to use this for `getPrimaryAlgorithmValueConsumer`
*/
Crypto::AlgorithmValueConsumer getAlgorithmValueConsumerForInput(IOType type) {
result = this and this.setsValue(type)
or
exists(DataFlow::Node src, DataFlow::Node sink |
AvcToOperationStepFlow::flow(src, sink) and
src.asExpr() = result and

14
cpp/ql/test/experimental/library-tests/quantum/node_edges.expected
View File

@@ -41,6 +41,20 @@
| openssl_basic.c:167:9:167:27 | SignOperation | Input | openssl_basic.c:163:35:163:41 | Message |
| openssl_basic.c:167:9:167:27 | SignOperation | Key | openssl_basic.c:160:59:160:62 | Key |
| openssl_basic.c:167:9:167:27 | SignOperation | Output | openssl_basic.c:167:34:167:36 | SignatureOutput |
| openssl_basic.c:235:51:235:55 | KeyOperationAlgorithm | Mode | openssl_basic.c:235:51:235:55 | KeyOperationAlgorithm |
| openssl_basic.c:235:51:235:55 | KeyOperationAlgorithm | Padding | openssl_basic.c:249:51:249:72 | PaddingAlgorithm |
| openssl_basic.c:238:9:238:25 | KeyGeneration | Algorithm | openssl_basic.c:235:51:235:55 | KeyOperationAlgorithm |
| openssl_basic.c:238:9:238:25 | KeyGeneration | Output | openssl_basic.c:238:39:238:43 | Key |
| openssl_basic.c:238:39:238:43 | Key | Algorithm | openssl_basic.c:235:51:235:55 | KeyOperationAlgorithm |
| openssl_basic.c:243:52:243:55 | Key | Source | openssl_basic.c:238:39:238:43 | Key |
| openssl_basic.c:249:51:249:72 | PaddingAlgorithm | MD | openssl_basic.c:250:51:250:60 | HashAlgorithm |
| openssl_basic.c:249:51:249:72 | PaddingAlgorithm | MGF1Hash | openssl_basic.c:251:51:251:60 | HashAlgorithm |
| openssl_basic.c:262:24:262:39 | EncryptOperation | Algorithm | openssl_basic.c:235:51:235:55 | KeyOperationAlgorithm |
| openssl_basic.c:262:24:262:39 | EncryptOperation | Input | openssl_basic.c:263:64:263:70 | Message |
| openssl_basic.c:262:24:262:39 | EncryptOperation | Key | openssl_basic.c:243:52:243:55 | Key |
| openssl_basic.c:262:24:262:39 | EncryptOperation | Nonce | openssl_basic.c:262:24:262:39 | EncryptOperation |
| openssl_basic.c:262:24:262:39 | EncryptOperation | Output | openssl_basic.c:262:54:262:63 | KeyOperationOutput |
| openssl_basic.c:263:64:263:70 | Message | Source | openssl_basic.c:231:27:231:49 | Constant |
| openssl_pkey.c:21:10:21:28 | KeyGeneration | Algorithm | openssl_pkey.c:21:10:21:28 | KeyGeneration |
| openssl_pkey.c:21:10:21:28 | KeyGeneration | Output | openssl_pkey.c:21:30:21:32 | Key |
| openssl_pkey.c:21:10:21:28 | KeyOperationAlgorithm | Mode | openssl_pkey.c:21:10:21:28 | KeyOperationAlgorithm |

15
cpp/ql/test/experimental/library-tests/quantum/node_properties.expected
View File

@@ -32,6 +32,21 @@
| openssl_basic.c:180:42:180:59 | Constant | Description | 0123456789012345 | openssl_basic.c:180:42:180:59 | openssl_basic.c:180:42:180:59 |
| openssl_basic.c:181:49:181:87 | Constant | Description | This is a test message for encryption | openssl_basic.c:181:49:181:87 | openssl_basic.c:181:49:181:87 |
| openssl_basic.c:218:32:218:33 | Constant | Description | 32 | openssl_basic.c:218:32:218:33 | openssl_basic.c:218:32:218:33 |
| openssl_basic.c:231:27:231:49 | Constant | Description | Encrypt me with OAEP! | openssl_basic.c:231:27:231:49 | openssl_basic.c:231:27:231:49 |
| openssl_basic.c:235:51:235:55 | KeyOperationAlgorithm | Name | RSA | openssl_basic.c:235:51:235:55 | openssl_basic.c:235:51:235:55 |
| openssl_basic.c:235:51:235:55 | KeyOperationAlgorithm | RawName | RSA | openssl_basic.c:235:51:235:55 | openssl_basic.c:235:51:235:55 |
| openssl_basic.c:237:54:237:57 | Constant | Description | 2048 | openssl_basic.c:237:54:237:57 | openssl_basic.c:237:54:237:57 |
| openssl_basic.c:238:39:238:43 | Key | KeyType | Asymmetric | openssl_basic.c:238:39:238:43 | openssl_basic.c:238:39:238:43 |
| openssl_basic.c:243:52:243:55 | Key | KeyType | Unknown | openssl_basic.c:243:52:243:55 | openssl_basic.c:243:52:243:55 |
| openssl_basic.c:249:51:249:72 | PaddingAlgorithm | Name | OAEP | openssl_basic.c:249:51:249:72 | openssl_basic.c:249:51:249:72 |
| openssl_basic.c:249:51:249:72 | PaddingAlgorithm | RawName | 4 | openssl_basic.c:249:51:249:72 | openssl_basic.c:249:51:249:72 |
| openssl_basic.c:250:51:250:60 | HashAlgorithm | DigestSize | 256 | openssl_basic.c:250:51:250:60 | openssl_basic.c:250:51:250:60 |
| openssl_basic.c:250:51:250:60 | HashAlgorithm | Name | SHA2 | openssl_basic.c:250:51:250:60 | openssl_basic.c:250:51:250:60 |
| openssl_basic.c:250:51:250:60 | HashAlgorithm | RawName | EVP_sha256 | openssl_basic.c:250:51:250:60 | openssl_basic.c:250:51:250:60 |
| openssl_basic.c:251:51:251:60 | HashAlgorithm | DigestSize | 256 | openssl_basic.c:251:51:251:60 | openssl_basic.c:251:51:251:60 |
| openssl_basic.c:251:51:251:60 | HashAlgorithm | Name | SHA2 | openssl_basic.c:251:51:251:60 | openssl_basic.c:251:51:251:60 |
| openssl_basic.c:251:51:251:60 | HashAlgorithm | RawName | EVP_sha256 | openssl_basic.c:251:51:251:60 | openssl_basic.c:251:51:251:60 |
| openssl_basic.c:262:24:262:39 | EncryptOperation | KeyOperationSubtype | Encrypt | openssl_basic.c:262:24:262:39 | openssl_basic.c:262:24:262:39 |
| openssl_pkey.c:21:10:21:28 | KeyOperationAlgorithm | Name | RSA | openssl_pkey.c:21:10:21:28 | openssl_pkey.c:21:10:21:28 |
| openssl_pkey.c:21:10:21:28 | KeyOperationAlgorithm | RawName | RSA_generate_key_ex | openssl_pkey.c:21:10:21:28 | openssl_pkey.c:21:10:21:28 |
| openssl_pkey.c:21:30:21:32 | Key | KeyType | Asymmetric | openssl_pkey.c:21:30:21:32 | openssl_pkey.c:21:30:21:32 |

12
cpp/ql/test/experimental/library-tests/quantum/nodes.expected
View File

@@ -34,6 +34,18 @@
| openssl_basic.c:180:42:180:59 | Constant |
| openssl_basic.c:181:49:181:87 | Constant |
| openssl_basic.c:218:32:218:33 | Constant |
| openssl_basic.c:231:27:231:49 | Constant |
| openssl_basic.c:235:51:235:55 | KeyOperationAlgorithm |
| openssl_basic.c:237:54:237:57 | Constant |
| openssl_basic.c:238:9:238:25 | KeyGeneration |
| openssl_basic.c:238:39:238:43 | Key |
| openssl_basic.c:243:52:243:55 | Key |
| openssl_basic.c:249:51:249:72 | PaddingAlgorithm |
| openssl_basic.c:250:51:250:60 | HashAlgorithm |
| openssl_basic.c:251:51:251:60 | HashAlgorithm |
| openssl_basic.c:262:24:262:39 | EncryptOperation |
| openssl_basic.c:262:54:262:63 | KeyOperationOutput |
| openssl_basic.c:263:64:263:70 | Message |
| openssl_pkey.c:21:10:21:28 | KeyGeneration |
| openssl_pkey.c:21:10:21:28 | KeyOperationAlgorithm |
| openssl_pkey.c:21:30:21:32 | Key |

58
cpp/ql/test/experimental/library-tests/quantum/openssl_basic.c
View File

@@ -1,7 +1,7 @@
#include "openssl/evp.h"
#include "openssl/obj_mac.h"
#include "openssl/rand.h"
#include "openssl/rsa.h"
size_t strlen(const char* str);
// Sample OpenSSL code that demonstrates various cryptographic operations
@@ -218,4 +218,58 @@ int test_main() {
calculate_hmac_sha256(key, 32, plaintext, plaintext_len, hmac);
return 0;
}
}
/**
* Simplified signature test
*/
int test_rsa_oaep_basic(void) {
EVP_PKEY_CTX *keygen_ctx = NULL, *encrypt_ctx = NULL;
EVP_PKEY *pkey = NULL;
unsigned char *ciphertext = NULL;
size_t ciphertext_len = 0;
const char *message = "Encrypt me with OAEP!";
int ret = 1;
// Generate RSA key
keygen_ctx = EVP_PKEY_CTX_new_from_name(NULL, "RSA", NULL);
if (!keygen_ctx || EVP_PKEY_keygen_init(keygen_ctx) <= 0 ||
EVP_PKEY_CTX_set_rsa_keygen_bits(keygen_ctx, 2048) <= 0 ||
EVP_PKEY_generate(keygen_ctx, &pkey) <= 0) {
goto cleanup;
}
// Create encryption context
encrypt_ctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL);
if (!encrypt_ctx || EVP_PKEY_encrypt_init(encrypt_ctx) <= 0) {
goto cleanup;
}
// Set OAEP padding
if (EVP_PKEY_CTX_set_rsa_padding(encrypt_ctx, RSA_PKCS1_OAEP_PADDING) <= 0 ||
EVP_PKEY_CTX_set_rsa_oaep_md(encrypt_ctx, EVP_sha256()) <= 0 ||
EVP_PKEY_CTX_set_rsa_mgf1_md(encrypt_ctx, EVP_sha256()) <= 0) {
goto cleanup;
}
// Determine buffer size
if (EVP_PKEY_encrypt(encrypt_ctx, NULL, &ciphertext_len,
(const unsigned char *)message, strlen(message)) <= 0) {
goto cleanup;
}
ciphertext = OPENSSL_malloc(ciphertext_len);
if (!ciphertext || EVP_PKEY_encrypt(encrypt_ctx, ciphertext, &ciphertext_len,
(const unsigned char *)message, strlen(message)) <= 0) {
goto cleanup;
}
ret = 0;
cleanup:
EVP_PKEY_CTX_free(keygen_ctx);
EVP_PKEY_CTX_free(encrypt_ctx);
EVP_PKEY_free(pkey);
OPENSSL_free(ciphertext);
return ret;
}