summaryrefslogtreecommitdiff
path: root/src/openssl/kt_rsa.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/openssl/kt_rsa.c')
-rw-r--r--src/openssl/kt_rsa.c388
1 files changed, 171 insertions, 217 deletions
diff --git a/src/openssl/kt_rsa.c b/src/openssl/kt_rsa.c
index 8d47e427..8fcb93b6 100644
--- a/src/openssl/kt_rsa.c
+++ b/src/openssl/kt_rsa.c
@@ -1,14 +1,19 @@
-/**
- *
- * XMLSec library
+/*
+ * XML Security Library (http://www.aleksey.com/xmlsec).
*
- * RSA Algorithms support
*
* This is free software; see Copyright file in the source
* distribution for preciese wording.
*
* Copyright (C) 2002-2016 Aleksey Sanin <aleksey@aleksey.com>. All Rights Reserved.
*/
+/**
+ * SECTION:kt_rsa
+ * @Short_description: RSA Key Transport transforms implementation for OpenSSL.
+ * @Stability: Private
+ *
+ */
+
#include "globals.h"
#ifndef XMLSEC_NO_RSA
@@ -34,6 +39,41 @@
#include <xmlsec/openssl/crypto.h>
#include <xmlsec/openssl/evp.h>
#include <xmlsec/openssl/bn.h>
+#include "openssl_compat.h"
+
+#ifdef OPENSSL_IS_BORINGSSL
+
+/* defined in boringssl/crypto/fipsmodule/rsa/internal.h */
+int RSA_padding_check_PKCS1_OAEP_mgf1(uint8_t *out, size_t *out_len, size_t max_out,
+ const uint8_t *from, size_t from_len,
+ const uint8_t *param, size_t param_len,
+ const EVP_MD *md, const EVP_MD *mgf1md);
+
+static int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int to_len,
+ unsigned char *from, int from_len,
+ int rsa_len,
+ unsigned char *param, int param_len) {
+ size_t out_len = 0;
+ int ret;
+
+ ret = RSA_padding_check_PKCS1_OAEP_mgf1(to, &out_len, to_len, from, from_len, param, param_len, NULL, NULL);
+ if(!ret) {
+ return(-1);
+ }
+ return((int)out_len);
+}
+
+
+int RSA_padding_add_PKCS1_OAEP(uint8_t *to, size_t to_len,
+ const uint8_t *from,
+ size_t from_len,
+ const uint8_t *param,
+ size_t param_len) {
+ return RSA_padding_add_PKCS1_OAEP_mgf1(to, to_len, from, from_len, param, param_len, NULL, NULL);
+}
+#endif /* OPENSSL_IS_BORINGSSL */
+
+
/**************************************************************************
*
@@ -166,6 +206,7 @@ static int
xmlSecOpenSSLRsaPkcs1SetKey(xmlSecTransformPtr transform, xmlSecKeyPtr key) {
xmlSecOpenSSLRsaPkcs1CtxPtr ctx;
EVP_PKEY* pKey;
+ RSA *rsa;
xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaPkcs1Id), -1);
xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
@@ -179,23 +220,18 @@ xmlSecOpenSSLRsaPkcs1SetKey(xmlSecTransformPtr transform, xmlSecKeyPtr key) {
pKey = xmlSecOpenSSLKeyDataRsaGetEvp(xmlSecKeyGetValue(key));
if(pKey == NULL) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "xmlSecOpenSSLKeyDataRsaGetEvp",
- XMLSEC_ERRORS_R_XMLSEC_FAILED,
- XMLSEC_ERRORS_NO_MESSAGE);
+ xmlSecInternalError("xmlSecOpenSSLKeyDataRsaGetEvp",
+ xmlSecTransformGetName(transform));
return(-1);
}
- xmlSecAssert2(pKey->type == EVP_PKEY_RSA, -1);
- xmlSecAssert2(pKey->pkey.rsa != NULL, -1);
+ xmlSecAssert2(EVP_PKEY_base_id(pKey) == EVP_PKEY_RSA, -1);
+ rsa = EVP_PKEY_get0_RSA(pKey);
+ xmlSecAssert2(rsa != NULL, -1);
ctx->pKey = xmlSecOpenSSLEvpKeyDup(pKey);
if(ctx->pKey == NULL) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "xmlSecOpenSSLEvpKeyDup",
- XMLSEC_ERRORS_R_XMLSEC_FAILED,
- XMLSEC_ERRORS_NO_MESSAGE);
+ xmlSecInternalError("xmlSecOpenSSLEvpKeyDup",
+ xmlSecTransformGetName(transform));
return(-1);
}
@@ -225,11 +261,8 @@ xmlSecOpenSSLRsaPkcs1Execute(xmlSecTransformPtr transform, int last, xmlSecTrans
} else if((transform->status == xmlSecTransformStatusWorking) && (last != 0)) {
ret = xmlSecOpenSSLRsaPkcs1Process(transform, transformCtx);
if(ret < 0) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "xmlSecOpenSSLRsaPkcs1Process",
- XMLSEC_ERRORS_R_XMLSEC_FAILED,
- XMLSEC_ERRORS_NO_MESSAGE);
+ xmlSecInternalError("xmlSecOpenSSLRsaPkcs1Process",
+ xmlSecTransformGetName(transform));
return(-1);
}
transform->status = xmlSecTransformStatusFinished;
@@ -237,11 +270,7 @@ xmlSecOpenSSLRsaPkcs1Execute(xmlSecTransformPtr transform, int last, xmlSecTrans
/* the only way we can get here is if there is no input */
xmlSecAssert2(xmlSecBufferGetSize(&(transform->inBuf)) == 0, -1);
} else {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- NULL,
- XMLSEC_ERRORS_R_INVALID_STATUS,
- "status=%d", transform->status);
+ xmlSecInvalidTransfromStatusError(transform);
return(-1);
}
return(0);
@@ -253,6 +282,7 @@ xmlSecOpenSSLRsaPkcs1Process(xmlSecTransformPtr transform, xmlSecTransformCtxPtr
xmlSecBufferPtr in, out;
xmlSecSize inSize, outSize;
xmlSecSize keySize;
+ RSA *rsa;
int ret;
xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaPkcs1Id), -1);
@@ -263,10 +293,11 @@ xmlSecOpenSSLRsaPkcs1Process(xmlSecTransformPtr transform, xmlSecTransformCtxPtr
ctx = xmlSecOpenSSLRsaPkcs1GetCtx(transform);
xmlSecAssert2(ctx != NULL, -1);
xmlSecAssert2(ctx->pKey != NULL, -1);
- xmlSecAssert2(ctx->pKey->type == EVP_PKEY_RSA, -1);
- xmlSecAssert2(ctx->pKey->pkey.rsa != NULL, -1);
+ xmlSecAssert2(EVP_PKEY_base_id(ctx->pKey) == EVP_PKEY_RSA, -1);
+ rsa = EVP_PKEY_get0_RSA(ctx->pKey);
+ xmlSecAssert2(rsa != NULL, -1);
- keySize = RSA_size(ctx->pKey->pkey.rsa);
+ keySize = RSA_size(rsa);
xmlSecAssert2(keySize > 0, -1);
in = &(transform->inBuf);
@@ -279,55 +310,43 @@ xmlSecOpenSSLRsaPkcs1Process(xmlSecTransformPtr transform, xmlSecTransformCtxPtr
/* the encoded size is equal to the keys size so we could not
* process more than that */
if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- NULL,
- XMLSEC_ERRORS_R_INVALID_SIZE,
- "%d when expected less than %d", inSize, keySize);
+ xmlSecInvalidSizeLessThanError("Input data", inSize, keySize,
+ xmlSecTransformGetName(transform));
return(-1);
} else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- NULL,
- XMLSEC_ERRORS_R_INVALID_SIZE,
- "%d when expected %d", inSize, keySize);
+ xmlSecInvalidSizeError("Input data", inSize, keySize,
+ xmlSecTransformGetName(transform));
return(-1);
}
outSize = keySize;
ret = xmlSecBufferSetMaxSize(out, outSize);
if(ret < 0) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "xmlSecBufferSetMaxSize",
- XMLSEC_ERRORS_R_XMLSEC_FAILED,
- "size=%d", outSize);
+ xmlSecInternalError2("xmlSecBufferSetMaxSize",
+ xmlSecTransformGetName(transform),
+ "size=%d", outSize);
return(-1);
}
if(transform->operation == xmlSecTransformOperationEncrypt) {
ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in),
- xmlSecBufferGetData(out),
- ctx->pKey->pkey.rsa, RSA_PKCS1_PADDING);
+ xmlSecBufferGetData(out),
+ rsa, RSA_PKCS1_PADDING);
if(ret <= 0) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "RSA_public_encrypt",
- XMLSEC_ERRORS_R_CRYPTO_FAILED,
- "size=%d", inSize);
+ xmlSecOpenSSLError2("RSA_public_encrypt",
+ xmlSecTransformGetName(transform),
+ "size=%lu", (unsigned long)inSize);
return(-1);
}
outSize = ret;
} else {
ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in),
- xmlSecBufferGetData(out),
- ctx->pKey->pkey.rsa, RSA_PKCS1_PADDING);
+ xmlSecBufferGetData(out),
+ rsa, RSA_PKCS1_PADDING);
if(ret <= 0) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "RSA_private_decrypt",
- XMLSEC_ERRORS_R_CRYPTO_FAILED,
- "size=%d", inSize);
+ xmlSecOpenSSLError2("RSA_private_decrypt",
+ xmlSecTransformGetName(transform),
+ "size=%lu", (unsigned long)inSize);
return(-1);
}
outSize = ret;
@@ -335,21 +354,17 @@ xmlSecOpenSSLRsaPkcs1Process(xmlSecTransformPtr transform, xmlSecTransformCtxPtr
ret = xmlSecBufferSetSize(out, outSize);
if(ret < 0) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "xmlSecBufferSetSize",
- XMLSEC_ERRORS_R_XMLSEC_FAILED,
- "size=%d", outSize);
+ xmlSecInternalError2("xmlSecBufferSetSize",
+ xmlSecTransformGetName(transform),
+ "size=%d", outSize);
return(-1);
}
ret = xmlSecBufferRemoveHead(in, inSize);
if(ret < 0) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "xmlSecBufferRemoveHead",
- XMLSEC_ERRORS_R_XMLSEC_FAILED,
- "size=%d", inSize);
+ xmlSecInternalError2("xmlSecBufferRemoveHead",
+ xmlSecTransformGetName(transform),
+ "size=%d", inSize);
return(-1);
}
@@ -449,11 +464,8 @@ xmlSecOpenSSLRsaOaepInitialize(xmlSecTransformPtr transform) {
ret = xmlSecBufferInitialize(&(ctx->oaepParams), 0);
if(ret < 0) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "xmlSecBufferInitialize",
- XMLSEC_ERRORS_R_XMLSEC_FAILED,
- XMLSEC_ERRORS_NO_MESSAGE);
+ xmlSecInternalError("xmlSecBufferInitialize",
+ xmlSecTransformGetName(transform));
return(-1);
}
return(0);
@@ -496,11 +508,8 @@ xmlSecOpenSSLRsaOaepNodeRead(xmlSecTransformPtr transform, xmlNodePtr node, xmlS
if(xmlSecCheckNodeName(cur, xmlSecNodeRsaOAEPparams, xmlSecEncNs)) {
ret = xmlSecBufferBase64NodeContentRead(&(ctx->oaepParams), cur);
if(ret < 0) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "xmlSecBufferBase64NodeContentRead",
- XMLSEC_ERRORS_R_XMLSEC_FAILED,
- XMLSEC_ERRORS_NO_MESSAGE);
+ xmlSecInternalError("xmlSecBufferBase64NodeContentRead",
+ xmlSecTransformGetName(transform));
return(-1);
}
} else if(xmlSecCheckNodeName(cur, xmlSecNodeDigestMethod, xmlSecDSigNs)) {
@@ -509,33 +518,24 @@ xmlSecOpenSSLRsaOaepNodeRead(xmlSecTransformPtr transform, xmlNodePtr node, xmlS
/* Algorithm attribute is required */
algorithm = xmlGetProp(cur, xmlSecAttrAlgorithm);
if(algorithm == NULL) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- xmlSecErrorsSafeString(xmlSecAttrAlgorithm),
- XMLSEC_ERRORS_R_INVALID_NODE_ATTRIBUTE,
- "node=%s",
- xmlSecErrorsSafeString(xmlSecNodeGetName(cur)));
+ xmlSecInvalidNodeAttributeError(cur, xmlSecAttrAlgorithm,
+ xmlSecTransformGetName(transform),
+ "empty");
return(-1);
}
/* for now we support only sha1 */
if(xmlStrcmp(algorithm, xmlSecHrefSha1) != 0) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- xmlSecErrorsSafeString(algorithm),
- XMLSEC_ERRORS_R_INVALID_TRANSFORM,
- "digest algorithm is not supported for rsa/oaep");
+ xmlSecInvalidTransfromError2(transform,
+ "digest algorithm=\"%s\" is not supported for rsa/oaep",
+ xmlSecErrorsSafeString(algorithm));
xmlFree(algorithm);
return(-1);
}
xmlFree(algorithm);
} else {
/* not found */
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- xmlSecErrorsSafeString(xmlSecNodeGetName(cur)),
- XMLSEC_ERRORS_R_UNEXPECTED_NODE,
- XMLSEC_ERRORS_NO_MESSAGE);
+ xmlSecUnexpectedNodeError(cur, xmlSecTransformGetName(transform));
return(-1);
}
@@ -574,6 +574,7 @@ static int
xmlSecOpenSSLRsaOaepSetKey(xmlSecTransformPtr transform, xmlSecKeyPtr key) {
xmlSecOpenSSLRsaOaepCtxPtr ctx;
EVP_PKEY* pKey;
+ RSA *rsa;
xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaOaepId), -1);
xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
@@ -587,23 +588,18 @@ xmlSecOpenSSLRsaOaepSetKey(xmlSecTransformPtr transform, xmlSecKeyPtr key) {
pKey = xmlSecOpenSSLKeyDataRsaGetEvp(xmlSecKeyGetValue(key));
if(pKey == NULL) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "xmlSecOpenSSLKeyDataRsaGetEvp",
- XMLSEC_ERRORS_R_XMLSEC_FAILED,
- XMLSEC_ERRORS_NO_MESSAGE);
+ xmlSecInternalError("xmlSecOpenSSLKeyDataRsaGetEvp",
+ xmlSecTransformGetName(transform));
return(-1);
}
- xmlSecAssert2(pKey->type == EVP_PKEY_RSA, -1);
- xmlSecAssert2(pKey->pkey.rsa != NULL, -1);
+ xmlSecAssert2(EVP_PKEY_base_id(pKey) == EVP_PKEY_RSA, -1);
+ rsa = EVP_PKEY_get0_RSA(pKey);
+ xmlSecAssert2(rsa != NULL, -1);
ctx->pKey = xmlSecOpenSSLEvpKeyDup(pKey);
if(ctx->pKey == NULL) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "xmlSecOpenSSLEvpKeyDup",
- XMLSEC_ERRORS_R_XMLSEC_FAILED,
- XMLSEC_ERRORS_NO_MESSAGE);
+ xmlSecInternalError("xmlSecOpenSSLEvpKeyDup",
+ xmlSecTransformGetName(transform));
return(-1);
}
@@ -633,11 +629,8 @@ xmlSecOpenSSLRsaOaepExecute(xmlSecTransformPtr transform, int last, xmlSecTransf
} else if((transform->status == xmlSecTransformStatusWorking) && (last != 0)) {
ret = xmlSecOpenSSLRsaOaepProcess(transform, transformCtx);
if(ret < 0) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "xmlSecOpenSSLRsaOaepProcess",
- XMLSEC_ERRORS_R_XMLSEC_FAILED,
- XMLSEC_ERRORS_NO_MESSAGE);
+ xmlSecInternalError("xmlSecOpenSSLRsaOaepProcess",
+ xmlSecTransformGetName(transform));
return(-1);
}
transform->status = xmlSecTransformStatusFinished;
@@ -645,11 +638,7 @@ xmlSecOpenSSLRsaOaepExecute(xmlSecTransformPtr transform, int last, xmlSecTransf
/* the only way we can get here is if there is no input */
xmlSecAssert2(xmlSecBufferGetSize(&(transform->inBuf)) == 0, -1);
} else {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- NULL,
- XMLSEC_ERRORS_R_INVALID_STATUS,
- "status=%d", transform->status);
+ xmlSecInvalidTransfromStatusError(transform);
return(-1);
}
return(0);
@@ -662,6 +651,7 @@ xmlSecOpenSSLRsaOaepProcess(xmlSecTransformPtr transform, xmlSecTransformCtxPtr
xmlSecBufferPtr in, out;
xmlSecSize inSize, outSize;
xmlSecSize keySize;
+ RSA *rsa;
int ret;
xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaOaepId), -1);
@@ -672,10 +662,11 @@ xmlSecOpenSSLRsaOaepProcess(xmlSecTransformPtr transform, xmlSecTransformCtxPtr
ctx = xmlSecOpenSSLRsaOaepGetCtx(transform);
xmlSecAssert2(ctx != NULL, -1);
xmlSecAssert2(ctx->pKey != NULL, -1);
- xmlSecAssert2(ctx->pKey->type == EVP_PKEY_RSA, -1);
- xmlSecAssert2(ctx->pKey->pkey.rsa != NULL, -1);
+ xmlSecAssert2(EVP_PKEY_base_id(ctx->pKey) == EVP_PKEY_RSA, -1);
+ rsa = EVP_PKEY_get0_RSA(ctx->pKey);
+ xmlSecAssert2(rsa != NULL, -1);
- keySize = RSA_size(ctx->pKey->pkey.rsa);
+ keySize = RSA_size(rsa);
xmlSecAssert2(keySize > 0, -1);
in = &(transform->inBuf);
@@ -688,29 +679,21 @@ xmlSecOpenSSLRsaOaepProcess(xmlSecTransformPtr transform, xmlSecTransformCtxPtr
/* the encoded size is equal to the keys size so we could not
* process more than that */
if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- NULL,
- XMLSEC_ERRORS_R_INVALID_SIZE,
- "%d when expected less than %d", inSize, keySize);
+ xmlSecInvalidSizeLessThanError("Input data", inSize, keySize,
+ xmlSecTransformGetName(transform));
return(-1);
} else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- NULL,
- XMLSEC_ERRORS_R_INVALID_SIZE,
- "%d when expected %d", inSize, keySize);
+ xmlSecInvalidSizeError("Input data", inSize, keySize,
+ xmlSecTransformGetName(transform));
return(-1);
}
outSize = keySize;
ret = xmlSecBufferSetMaxSize(out, outSize);
if(ret < 0) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "xmlSecBufferSetMaxSize",
- XMLSEC_ERRORS_R_XMLSEC_FAILED,
- "size=%d", outSize);
+ xmlSecInternalError2("xmlSecBufferSetMaxSize",
+ xmlSecTransformGetName(transform),
+ "size=%d", outSize);
return(-1);
}
@@ -719,125 +702,105 @@ xmlSecOpenSSLRsaOaepProcess(xmlSecTransformPtr transform, xmlSecTransformCtxPtr
/* encode w/o OAEPParams --> simple */
ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
- ctx->pKey->pkey.rsa, RSA_PKCS1_OAEP_PADDING);
+ rsa, RSA_PKCS1_OAEP_PADDING);
if(ret <= 0) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "RSA_public_encrypt(RSA_PKCS1_OAEP_PADDING)",
- XMLSEC_ERRORS_R_CRYPTO_FAILED,
- XMLSEC_ERRORS_NO_MESSAGE);
+ xmlSecOpenSSLError("RSA_public_encrypt(RSA_PKCS1_OAEP_PADDING)",
+ xmlSecTransformGetName(transform));
return(-1);
}
outSize = ret;
} else if((transform->operation == xmlSecTransformOperationEncrypt) && (paramsSize > 0)) {
+ xmlSecBuffer tmp;
+
xmlSecAssert2(xmlSecBufferGetData(&(ctx->oaepParams)) != NULL, -1);
- /* add space for padding */
- ret = xmlSecBufferSetMaxSize(in, keySize);
+ /* allocate space for temp buffer */
+ ret = xmlSecBufferInitialize(&tmp, keySize);
if(ret < 0) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "xmlSecBufferSetMaxSize",
- XMLSEC_ERRORS_R_XMLSEC_FAILED,
- "size=%d", keySize);
+ xmlSecInternalError2("xmlSecBufferInitialize",
+ xmlSecTransformGetName(transform),
+ "size=%d", keySize);
return(-1);
}
/* add padding */
- ret = RSA_padding_add_PKCS1_OAEP(xmlSecBufferGetData(in), keySize,
+ ret = RSA_padding_add_PKCS1_OAEP(xmlSecBufferGetData(&tmp), keySize,
xmlSecBufferGetData(in), inSize,
- xmlSecBufferGetData(&(ctx->oaepParams)),
- paramsSize);
+ xmlSecBufferGetData(&(ctx->oaepParams)), paramsSize);
if(ret != 1) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "RSA_padding_add_PKCS1_OAEP",
- XMLSEC_ERRORS_R_CRYPTO_FAILED,
- XMLSEC_ERRORS_NO_MESSAGE);
+ xmlSecOpenSSLError("RSA_padding_add_PKCS1_OAEP",
+ xmlSecTransformGetName(transform));
+ xmlSecBufferFinalize(&tmp);
return(-1);
}
- inSize = keySize;
/* encode with OAEPParams */
- ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in),
+ ret = RSA_public_encrypt(keySize, xmlSecBufferGetData(&tmp),
xmlSecBufferGetData(out),
- ctx->pKey->pkey.rsa, RSA_NO_PADDING);
+ rsa, RSA_NO_PADDING);
if(ret <= 0) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "RSA_public_encrypt(RSA_NO_PADDING)",
- XMLSEC_ERRORS_R_CRYPTO_FAILED,
- XMLSEC_ERRORS_NO_MESSAGE);
+ xmlSecOpenSSLError("RSA_public_encrypt(RSA_NO_PADDING)",
+ xmlSecTransformGetName(transform));
+ xmlSecBufferFinalize(&tmp);
return(-1);
}
outSize = ret;
+ xmlSecBufferFinalize(&tmp);
} else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize == 0)) {
ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
- ctx->pKey->pkey.rsa, RSA_PKCS1_OAEP_PADDING);
+ rsa, RSA_PKCS1_OAEP_PADDING);
if(ret <= 0) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "RSA_private_decrypt(RSA_PKCS1_OAEP_PADDING)",
- XMLSEC_ERRORS_R_CRYPTO_FAILED,
- XMLSEC_ERRORS_NO_MESSAGE);
+ xmlSecOpenSSLError("RSA_private_decrypt(RSA_PKCS1_OAEP_PADDING)",
+ xmlSecTransformGetName(transform));
return(-1);
}
outSize = ret;
} else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize != 0)) {
BIGNUM * bn;
- bn = BN_new();
- if(bn == NULL) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "BN_new()",
- XMLSEC_ERRORS_R_CRYPTO_FAILED,
- XMLSEC_ERRORS_NO_MESSAGE);
- return(-1);
- }
ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
- ctx->pKey->pkey.rsa, RSA_NO_PADDING);
+ rsa, RSA_NO_PADDING);
if(ret <= 0) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "RSA_private_decrypt(RSA_NO_PADDING)",
- XMLSEC_ERRORS_R_CRYPTO_FAILED,
- XMLSEC_ERRORS_NO_MESSAGE);
- BN_free(bn);
+ xmlSecOpenSSLError("RSA_private_decrypt(RSA_NO_PADDING)",
+ xmlSecTransformGetName(transform));
return(-1);
}
outSize = ret;
+#ifndef OPENSSL_IS_BORINGSSL
/*
- * the private decrypt w/o padding adds '0's at the begginning.
+ * the private decrypt w/o padding adds '0's at the beginning.
* it's not clear for me can I simply skip all '0's from the
* beggining so I have to do decode it back to BIGNUM and dump
* buffer again
*/
+ bn = BN_new();
+ if(bn == NULL) {
+ xmlSecOpenSSLError("BN_new()",
+ xmlSecTransformGetName(transform));
+ return(-1);
+ }
+
if(BN_bin2bn(xmlSecBufferGetData(out), outSize, bn) == NULL) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "BN_bin2bn",
- XMLSEC_ERRORS_R_CRYPTO_FAILED,
- "size=%d", outSize);
+ xmlSecOpenSSLError2("BN_bin2bn",
+ xmlSecTransformGetName(transform),
+ "size=%lu", (unsigned long)outSize);
BN_free(bn);
return(-1);
}
ret = BN_bn2bin(bn, xmlSecBufferGetData(out));
if(ret <= 0) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "BN_bn2bin",
- XMLSEC_ERRORS_R_CRYPTO_FAILED,
- XMLSEC_ERRORS_NO_MESSAGE);
+ xmlSecOpenSSLError("BN_bn2bin",
+ xmlSecTransformGetName(transform));
BN_free(bn);
return(-1);
}
BN_free(bn);
outSize = ret;
+#endif /* OPENSSL_IS_BORINGSSL */
ret = RSA_padding_check_PKCS1_OAEP(xmlSecBufferGetData(out), outSize,
xmlSecBufferGetData(out), outSize,
@@ -845,41 +808,32 @@ xmlSecOpenSSLRsaOaepProcess(xmlSecTransformPtr transform, xmlSecTransformCtxPtr
xmlSecBufferGetData(&(ctx->oaepParams)),
paramsSize);
if(ret < 0) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "RSA_padding_check_PKCS1_OAEP",
- XMLSEC_ERRORS_R_CRYPTO_FAILED,
- XMLSEC_ERRORS_NO_MESSAGE);
+ xmlSecOpenSSLError("RSA_padding_check_PKCS1_OAEP",
+ xmlSecTransformGetName(transform));
return(-1);
}
outSize = ret;
} else {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "",
- XMLSEC_ERRORS_R_XMLSEC_FAILED,
- "Unexpected trasnform operation: %d; paramsSize: %d",
- (int)transform->operation, (int)paramsSize);
+ xmlSecOtherError3(XMLSEC_ERRORS_R_INVALID_OPERATION,
+ xmlSecTransformGetName(transform),
+ "Unexpected transform operation: %ld; paramsSize: %ld",
+ (long int)transform->operation, (long int)paramsSize);
return(-1);
}
ret = xmlSecBufferSetSize(out, outSize);
if(ret < 0) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "xmlSecBufferSetSize",
- XMLSEC_ERRORS_R_XMLSEC_FAILED,
- "size=%d", outSize);
+ xmlSecInternalError2("xmlSecBufferSetSize",
+ xmlSecTransformGetName(transform),
+ "size=%d", outSize);
return(-1);
}
ret = xmlSecBufferRemoveHead(in, inSize);
if(ret < 0) {
- xmlSecError(XMLSEC_ERRORS_HERE,
- xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
- "xmlSecBufferRemoveHead",
- XMLSEC_ERRORS_R_XMLSEC_FAILED,
- "size=%d", inSize);
+ xmlSecInternalError2("xmlSecBufferRemoveHead",
+ xmlSecTransformGetName(transform),
+ "size=%d", inSize);
return(-1);
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-05-29 08:35:51 +0000