diff options
author | Anas Nashif <anas.nashif@intel.com> | 2013-09-20 04:45:41 -0400 |
---|---|---|
committer | Anas Nashif <anas.nashif@intel.com> | 2013-09-20 04:45:41 -0400 |
commit | 6b6383d52bc147134bb6b60b07e924b176c67e3a (patch) | |
tree | 9753a1ec40b1fbe2acfaa881af46e3c0f7da6401 /src/openssl/kt_rsa.c | |
parent | 07bb297329b9e9754d09dcb6d70417272a626619 (diff) | |
download | xmlsec1-6b6383d52bc147134bb6b60b07e924b176c67e3a.tar.gz xmlsec1-6b6383d52bc147134bb6b60b07e924b176c67e3a.tar.bz2 xmlsec1-6b6383d52bc147134bb6b60b07e924b176c67e3a.zip |
Imported Upstream version 1.2.19
Diffstat (limited to 'src/openssl/kt_rsa.c')
-rw-r--r-- | src/openssl/kt_rsa.c | 973 |
1 files changed, 486 insertions, 487 deletions
diff --git a/src/openssl/kt_rsa.c b/src/openssl/kt_rsa.c index 1ed3685e..1cf1aba1 100644 --- a/src/openssl/kt_rsa.c +++ b/src/openssl/kt_rsa.c @@ -1,12 +1,12 @@ -/** +/** * * XMLSec library - * + * * RSA Algorithms support - * + * * This is free software; see Copyright file in the source * distribution for preciese wording. - * + * * Copyright (C) 2002-2003 Aleksey Sanin <aleksey@aleksey.com> */ #include "globals.h" @@ -40,11 +40,11 @@ * Internal OpenSSL RSA PKCS1 CTX * *************************************************************************/ -typedef struct _xmlSecOpenSSLRsaPkcs1Ctx xmlSecOpenSSLRsaPkcs1Ctx, - *xmlSecOpenSSLRsaPkcs1CtxPtr; +typedef struct _xmlSecOpenSSLRsaPkcs1Ctx xmlSecOpenSSLRsaPkcs1Ctx, + *xmlSecOpenSSLRsaPkcs1CtxPtr; struct _xmlSecOpenSSLRsaPkcs1Ctx { - EVP_PKEY* pKey; -}; + EVP_PKEY* pKey; +}; /********************************************************************* * @@ -53,77 +53,77 @@ struct _xmlSecOpenSSLRsaPkcs1Ctx { * xmlSecOpenSSLRsaPkcs1Ctx is located after xmlSecTransform * ********************************************************************/ -#define xmlSecOpenSSLRsaPkcs1Size \ - (sizeof(xmlSecTransform) + sizeof(xmlSecOpenSSLRsaPkcs1Ctx)) +#define xmlSecOpenSSLRsaPkcs1Size \ + (sizeof(xmlSecTransform) + sizeof(xmlSecOpenSSLRsaPkcs1Ctx)) #define xmlSecOpenSSLRsaPkcs1GetCtx(transform) \ ((xmlSecOpenSSLRsaPkcs1CtxPtr)(((xmlSecByte*)(transform)) + sizeof(xmlSecTransform))) -static int xmlSecOpenSSLRsaPkcs1Initialize (xmlSecTransformPtr transform); -static void xmlSecOpenSSLRsaPkcs1Finalize (xmlSecTransformPtr transform); -static int xmlSecOpenSSLRsaPkcs1SetKeyReq (xmlSecTransformPtr transform, - xmlSecKeyReqPtr keyReq); -static int xmlSecOpenSSLRsaPkcs1SetKey (xmlSecTransformPtr transform, - xmlSecKeyPtr key); -static int xmlSecOpenSSLRsaPkcs1Execute (xmlSecTransformPtr transform, - int last, - xmlSecTransformCtxPtr transformCtx); -static int xmlSecOpenSSLRsaPkcs1Process (xmlSecTransformPtr transform, - xmlSecTransformCtxPtr transformCtx); +static int xmlSecOpenSSLRsaPkcs1Initialize (xmlSecTransformPtr transform); +static void xmlSecOpenSSLRsaPkcs1Finalize (xmlSecTransformPtr transform); +static int xmlSecOpenSSLRsaPkcs1SetKeyReq (xmlSecTransformPtr transform, + xmlSecKeyReqPtr keyReq); +static int xmlSecOpenSSLRsaPkcs1SetKey (xmlSecTransformPtr transform, + xmlSecKeyPtr key); +static int xmlSecOpenSSLRsaPkcs1Execute (xmlSecTransformPtr transform, + int last, + xmlSecTransformCtxPtr transformCtx); +static int xmlSecOpenSSLRsaPkcs1Process (xmlSecTransformPtr transform, + xmlSecTransformCtxPtr transformCtx); static xmlSecTransformKlass xmlSecOpenSSLRsaPkcs1Klass = { /* klass/object sizes */ - sizeof(xmlSecTransformKlass), /* xmlSecSize klassSize */ - xmlSecOpenSSLRsaPkcs1Size, /* xmlSecSize objSize */ - - xmlSecNameRsaPkcs1, /* const xmlChar* name; */ - xmlSecHrefRsaPkcs1, /* const xmlChar* href; */ - xmlSecTransformUsageEncryptionMethod, /* xmlSecAlgorithmUsage usage; */ - - xmlSecOpenSSLRsaPkcs1Initialize, /* xmlSecTransformInitializeMethod initialize; */ - xmlSecOpenSSLRsaPkcs1Finalize, /* xmlSecTransformFinalizeMethod finalize; */ - NULL, /* xmlSecTransformNodeReadMethod readNode; */ - NULL, /* xmlSecTransformNodeWriteMethod writeNode; */ - xmlSecOpenSSLRsaPkcs1SetKeyReq, /* xmlSecTransformSetKeyMethod setKeyReq; */ - xmlSecOpenSSLRsaPkcs1SetKey, /* xmlSecTransformSetKeyMethod setKey; */ - NULL, /* xmlSecTransformValidateMethod validate; */ - xmlSecTransformDefaultGetDataType, /* xmlSecTransformGetDataTypeMethod getDataType; */ - xmlSecTransformDefaultPushBin, /* xmlSecTransformPushBinMethod pushBin; */ - xmlSecTransformDefaultPopBin, /* xmlSecTransformPopBinMethod popBin; */ - NULL, /* xmlSecTransformPushXmlMethod pushXml; */ - NULL, /* xmlSecTransformPopXmlMethod popXml; */ - xmlSecOpenSSLRsaPkcs1Execute, /* xmlSecTransformExecuteMethod execute; */ - - NULL, /* void* reserved0; */ - NULL, /* void* reserved1; */ + sizeof(xmlSecTransformKlass), /* xmlSecSize klassSize */ + xmlSecOpenSSLRsaPkcs1Size, /* xmlSecSize objSize */ + + xmlSecNameRsaPkcs1, /* const xmlChar* name; */ + xmlSecHrefRsaPkcs1, /* const xmlChar* href; */ + xmlSecTransformUsageEncryptionMethod, /* xmlSecAlgorithmUsage usage; */ + + xmlSecOpenSSLRsaPkcs1Initialize, /* xmlSecTransformInitializeMethod initialize; */ + xmlSecOpenSSLRsaPkcs1Finalize, /* xmlSecTransformFinalizeMethod finalize; */ + NULL, /* xmlSecTransformNodeReadMethod readNode; */ + NULL, /* xmlSecTransformNodeWriteMethod writeNode; */ + xmlSecOpenSSLRsaPkcs1SetKeyReq, /* xmlSecTransformSetKeyMethod setKeyReq; */ + xmlSecOpenSSLRsaPkcs1SetKey, /* xmlSecTransformSetKeyMethod setKey; */ + NULL, /* xmlSecTransformValidateMethod validate; */ + xmlSecTransformDefaultGetDataType, /* xmlSecTransformGetDataTypeMethod getDataType; */ + xmlSecTransformDefaultPushBin, /* xmlSecTransformPushBinMethod pushBin; */ + xmlSecTransformDefaultPopBin, /* xmlSecTransformPopBinMethod popBin; */ + NULL, /* xmlSecTransformPushXmlMethod pushXml; */ + NULL, /* xmlSecTransformPopXmlMethod popXml; */ + xmlSecOpenSSLRsaPkcs1Execute, /* xmlSecTransformExecuteMethod execute; */ + + NULL, /* void* reserved0; */ + NULL, /* void* reserved1; */ }; -/** +/** * xmlSecOpenSSLTransformRsaPkcs1GetKlass: * * The RSA-PKCS1 key transport transform klass. * * Returns: RSA-PKCS1 key transport transform klass. */ -xmlSecTransformId +xmlSecTransformId xmlSecOpenSSLTransformRsaPkcs1GetKlass(void) { return(&xmlSecOpenSSLRsaPkcs1Klass); } -static int +static int xmlSecOpenSSLRsaPkcs1Initialize(xmlSecTransformPtr transform) { xmlSecOpenSSLRsaPkcs1CtxPtr ctx; - + xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaPkcs1Id), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaPkcs1Size), -1); ctx = xmlSecOpenSSLRsaPkcs1GetCtx(transform); xmlSecAssert2(ctx != NULL, -1); - + memset(ctx, 0, sizeof(xmlSecOpenSSLRsaPkcs1Ctx)); return(0); } -static void +static void xmlSecOpenSSLRsaPkcs1Finalize(xmlSecTransformPtr transform) { xmlSecOpenSSLRsaPkcs1CtxPtr ctx; @@ -132,14 +132,14 @@ xmlSecOpenSSLRsaPkcs1Finalize(xmlSecTransformPtr transform) { ctx = xmlSecOpenSSLRsaPkcs1GetCtx(transform); xmlSecAssert(ctx != NULL); - + if(ctx->pKey != NULL) { - EVP_PKEY_free(ctx->pKey); + EVP_PKEY_free(ctx->pKey); } memset(ctx, 0, sizeof(xmlSecOpenSSLRsaPkcs1Ctx)); } -static int +static int xmlSecOpenSSLRsaPkcs1SetKeyReq(xmlSecTransformPtr transform, xmlSecKeyReqPtr keyReq) { xmlSecOpenSSLRsaPkcs1CtxPtr ctx; @@ -151,22 +151,22 @@ xmlSecOpenSSLRsaPkcs1SetKeyReq(xmlSecTransformPtr transform, xmlSecKeyReqPtr ke ctx = xmlSecOpenSSLRsaPkcs1GetCtx(transform); xmlSecAssert2(ctx != NULL, -1); - keyReq->keyId = xmlSecOpenSSLKeyDataRsaId; + keyReq->keyId = xmlSecOpenSSLKeyDataRsaId; if(transform->operation == xmlSecTransformOperationEncrypt) { keyReq->keyType = xmlSecKeyDataTypePublic; - keyReq->keyUsage = xmlSecKeyUsageEncrypt; + keyReq->keyUsage = xmlSecKeyUsageEncrypt; } else { keyReq->keyType = xmlSecKeyDataTypePrivate; - keyReq->keyUsage = xmlSecKeyUsageDecrypt; - } + keyReq->keyUsage = xmlSecKeyUsageDecrypt; + } return(0); } -static int +static int xmlSecOpenSSLRsaPkcs1SetKey(xmlSecTransformPtr transform, xmlSecKeyPtr key) { xmlSecOpenSSLRsaPkcs1CtxPtr ctx; EVP_PKEY* pKey; - + xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaPkcs1Id), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaPkcs1Size), -1); @@ -179,30 +179,30 @@ xmlSecOpenSSLRsaPkcs1SetKey(xmlSecTransformPtr transform, xmlSecKeyPtr key) { pKey = xmlSecOpenSSLKeyDataRsaGetEvp(xmlSecKeyGetValue(key)); if(pKey == NULL) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "xmlSecOpenSSLKeyDataRsaGetEvp", - XMLSEC_ERRORS_R_XMLSEC_FAILED, - XMLSEC_ERRORS_NO_MESSAGE); - return(-1); + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "xmlSecOpenSSLKeyDataRsaGetEvp", + XMLSEC_ERRORS_R_XMLSEC_FAILED, + XMLSEC_ERRORS_NO_MESSAGE); + return(-1); } - xmlSecAssert2(pKey->type == EVP_PKEY_RSA, -1); - xmlSecAssert2(pKey->pkey.rsa != NULL, -1); - - ctx->pKey = xmlSecOpenSSLEvpKeyDup(pKey); + xmlSecAssert2(pKey->type == EVP_PKEY_RSA, -1); + xmlSecAssert2(pKey->pkey.rsa != NULL, -1); + + ctx->pKey = xmlSecOpenSSLEvpKeyDup(pKey); if(ctx->pKey == NULL) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "xmlSecOpenSSLEvpKeyDup", - XMLSEC_ERRORS_R_XMLSEC_FAILED, - XMLSEC_ERRORS_NO_MESSAGE); - return(-1); + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "xmlSecOpenSSLEvpKeyDup", + XMLSEC_ERRORS_R_XMLSEC_FAILED, + XMLSEC_ERRORS_NO_MESSAGE); + return(-1); } return(0); } -static int +static int xmlSecOpenSSLRsaPkcs1Execute(xmlSecTransformPtr transform, int last, xmlSecTransformCtxPtr transformCtx) { xmlSecOpenSSLRsaPkcs1CtxPtr ctx; int ret; @@ -217,37 +217,37 @@ xmlSecOpenSSLRsaPkcs1Execute(xmlSecTransformPtr transform, int last, xmlSecTrans xmlSecAssert2(ctx->pKey != NULL, -1); if(transform->status == xmlSecTransformStatusNone) { - transform->status = xmlSecTransformStatusWorking; - } - + transform->status = xmlSecTransformStatusWorking; + } + if((transform->status == xmlSecTransformStatusWorking) && (last == 0)) { - /* just do nothing */ + /* just do nothing */ } else if((transform->status == xmlSecTransformStatusWorking) && (last != 0)) { - ret = xmlSecOpenSSLRsaPkcs1Process(transform, transformCtx); - if(ret < 0) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "xmlSecOpenSSLRsaPkcs1Process", - XMLSEC_ERRORS_R_XMLSEC_FAILED, - XMLSEC_ERRORS_NO_MESSAGE); - return(-1); - } - transform->status = xmlSecTransformStatusFinished; + ret = xmlSecOpenSSLRsaPkcs1Process(transform, transformCtx); + if(ret < 0) { + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "xmlSecOpenSSLRsaPkcs1Process", + XMLSEC_ERRORS_R_XMLSEC_FAILED, + XMLSEC_ERRORS_NO_MESSAGE); + return(-1); + } + transform->status = xmlSecTransformStatusFinished; } else if(transform->status == xmlSecTransformStatusFinished) { - /* the only way we can get here is if there is no input */ - xmlSecAssert2(xmlSecBufferGetSize(&(transform->inBuf)) == 0, -1); + /* the only way we can get here is if there is no input */ + xmlSecAssert2(xmlSecBufferGetSize(&(transform->inBuf)) == 0, -1); } else { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - NULL, - XMLSEC_ERRORS_R_INVALID_STATUS, - "status=%d", transform->status); - return(-1); + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + NULL, + XMLSEC_ERRORS_R_INVALID_STATUS, + "status=%d", transform->status); + return(-1); } return(0); } -static int +static int xmlSecOpenSSLRsaPkcs1Process(xmlSecTransformPtr transform, xmlSecTransformCtxPtr transformCtx) { xmlSecOpenSSLRsaPkcs1CtxPtr ctx; xmlSecBufferPtr in, out; @@ -263,110 +263,110 @@ xmlSecOpenSSLRsaPkcs1Process(xmlSecTransformPtr transform, xmlSecTransformCtxPtr ctx = xmlSecOpenSSLRsaPkcs1GetCtx(transform); xmlSecAssert2(ctx != NULL, -1); xmlSecAssert2(ctx->pKey != NULL, -1); - xmlSecAssert2(ctx->pKey->type == EVP_PKEY_RSA, -1); - xmlSecAssert2(ctx->pKey->pkey.rsa != NULL, -1); - + xmlSecAssert2(ctx->pKey->type == EVP_PKEY_RSA, -1); + xmlSecAssert2(ctx->pKey->pkey.rsa != NULL, -1); + keySize = RSA_size(ctx->pKey->pkey.rsa); xmlSecAssert2(keySize > 0, -1); - + in = &(transform->inBuf); out = &(transform->outBuf); - + inSize = xmlSecBufferGetSize(in); - outSize = xmlSecBufferGetSize(out); + outSize = xmlSecBufferGetSize(out); xmlSecAssert2(outSize == 0, -1); /* the encoded size is equal to the keys size so we could not * process more than that */ if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - NULL, - XMLSEC_ERRORS_R_INVALID_SIZE, - "%d when expected less than %d", inSize, keySize); - return(-1); + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + NULL, + XMLSEC_ERRORS_R_INVALID_SIZE, + "%d when expected less than %d", inSize, keySize); + return(-1); } else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - NULL, - XMLSEC_ERRORS_R_INVALID_SIZE, - "%d when expected %d", inSize, keySize); - return(-1); + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + NULL, + XMLSEC_ERRORS_R_INVALID_SIZE, + "%d when expected %d", inSize, keySize); + return(-1); } - - outSize = keySize; + + outSize = keySize; ret = xmlSecBufferSetMaxSize(out, outSize); if(ret < 0) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "xmlSecBufferSetMaxSize", - XMLSEC_ERRORS_R_XMLSEC_FAILED, - "size=%d", outSize); - return(-1); + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "xmlSecBufferSetMaxSize", + XMLSEC_ERRORS_R_XMLSEC_FAILED, + "size=%d", outSize); + return(-1); } if(transform->operation == xmlSecTransformOperationEncrypt) { - ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in), - xmlSecBufferGetData(out), - ctx->pKey->pkey.rsa, RSA_PKCS1_PADDING); - if(ret <= 0) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "RSA_public_encrypt", - XMLSEC_ERRORS_R_CRYPTO_FAILED, - "size=%d", inSize); - return(-1); - } - outSize = ret; + ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in), + xmlSecBufferGetData(out), + ctx->pKey->pkey.rsa, RSA_PKCS1_PADDING); + if(ret <= 0) { + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "RSA_public_encrypt", + XMLSEC_ERRORS_R_CRYPTO_FAILED, + "size=%d", inSize); + return(-1); + } + outSize = ret; } else { - ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in), - xmlSecBufferGetData(out), - ctx->pKey->pkey.rsa, RSA_PKCS1_PADDING); - if(ret <= 0) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "RSA_private_decrypt", - XMLSEC_ERRORS_R_CRYPTO_FAILED, - "size=%d", inSize); - return(-1); - } - outSize = ret; + ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in), + xmlSecBufferGetData(out), + ctx->pKey->pkey.rsa, RSA_PKCS1_PADDING); + if(ret <= 0) { + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "RSA_private_decrypt", + XMLSEC_ERRORS_R_CRYPTO_FAILED, + "size=%d", inSize); + return(-1); + } + outSize = ret; } ret = xmlSecBufferSetSize(out, outSize); if(ret < 0) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "xmlSecBufferSetSize", - XMLSEC_ERRORS_R_XMLSEC_FAILED, - "size=%d", outSize); - return(-1); + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "xmlSecBufferSetSize", + XMLSEC_ERRORS_R_XMLSEC_FAILED, + "size=%d", outSize); + return(-1); } - + ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "xmlSecBufferRemoveHead", - XMLSEC_ERRORS_R_XMLSEC_FAILED, - "size=%d", inSize); - return(-1); + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "xmlSecBufferRemoveHead", + XMLSEC_ERRORS_R_XMLSEC_FAILED, + "size=%d", inSize); + return(-1); } - + return(0); } /************************************************************************** * - * Internal OpenSSL RSA OAPE CTX + * Internal OpenSSL RSA OAEP CTX * *************************************************************************/ -typedef struct _xmlSecOpenSSLRsaOaepCtx xmlSecOpenSSLRsaOaepCtx, - *xmlSecOpenSSLRsaOaepCtxPtr; +typedef struct _xmlSecOpenSSLRsaOaepCtx xmlSecOpenSSLRsaOaepCtx, + *xmlSecOpenSSLRsaOaepCtxPtr; struct _xmlSecOpenSSLRsaOaepCtx { - EVP_PKEY* pKey; - xmlSecBuffer oaepParams; -}; + EVP_PKEY* pKey; + xmlSecBuffer oaepParams; +}; /********************************************************************* * @@ -375,91 +375,91 @@ struct _xmlSecOpenSSLRsaOaepCtx { * xmlSecOpenSSLRsaOaepCtx is located after xmlSecTransform * ********************************************************************/ -#define xmlSecOpenSSLRsaOaepSize \ - (sizeof(xmlSecTransform) + sizeof(xmlSecOpenSSLRsaOaepCtx)) +#define xmlSecOpenSSLRsaOaepSize \ + (sizeof(xmlSecTransform) + sizeof(xmlSecOpenSSLRsaOaepCtx)) #define xmlSecOpenSSLRsaOaepGetCtx(transform) \ ((xmlSecOpenSSLRsaOaepCtxPtr)(((xmlSecByte*)(transform)) + sizeof(xmlSecTransform))) -static int xmlSecOpenSSLRsaOaepInitialize (xmlSecTransformPtr transform); -static void xmlSecOpenSSLRsaOaepFinalize (xmlSecTransformPtr transform); -static int xmlSecOpenSSLRsaOaepNodeRead (xmlSecTransformPtr transform, - xmlNodePtr node, - xmlSecTransformCtxPtr transformCtx); -static int xmlSecOpenSSLRsaOaepSetKeyReq (xmlSecTransformPtr transform, - xmlSecKeyReqPtr keyReq); -static int xmlSecOpenSSLRsaOaepSetKey (xmlSecTransformPtr transform, - xmlSecKeyPtr key); -static int xmlSecOpenSSLRsaOaepExecute (xmlSecTransformPtr transform, - int last, - xmlSecTransformCtxPtr transformCtx); -static int xmlSecOpenSSLRsaOaepProcess (xmlSecTransformPtr transform, - xmlSecTransformCtxPtr transformCtx); +static int xmlSecOpenSSLRsaOaepInitialize (xmlSecTransformPtr transform); +static void xmlSecOpenSSLRsaOaepFinalize (xmlSecTransformPtr transform); +static int xmlSecOpenSSLRsaOaepNodeRead (xmlSecTransformPtr transform, + xmlNodePtr node, + xmlSecTransformCtxPtr transformCtx); +static int xmlSecOpenSSLRsaOaepSetKeyReq (xmlSecTransformPtr transform, + xmlSecKeyReqPtr keyReq); +static int xmlSecOpenSSLRsaOaepSetKey (xmlSecTransformPtr transform, + xmlSecKeyPtr key); +static int xmlSecOpenSSLRsaOaepExecute (xmlSecTransformPtr transform, + int last, + xmlSecTransformCtxPtr transformCtx); +static int xmlSecOpenSSLRsaOaepProcess (xmlSecTransformPtr transform, + xmlSecTransformCtxPtr transformCtx); static xmlSecTransformKlass xmlSecOpenSSLRsaOaepKlass = { /* klass/object sizes */ - sizeof(xmlSecTransformKlass), /* xmlSecSize klassSize */ - xmlSecOpenSSLRsaOaepSize, /* xmlSecSize objSize */ - - xmlSecNameRsaOaep, /* const xmlChar* name; */ - xmlSecHrefRsaOaep, /* const xmlChar* href; */ - xmlSecTransformUsageEncryptionMethod, /* xmlSecAlgorithmUsage usage; */ - - xmlSecOpenSSLRsaOaepInitialize, /* xmlSecTransformInitializeMethod initialize; */ - xmlSecOpenSSLRsaOaepFinalize, /* xmlSecTransformFinalizeMethod finalize; */ - xmlSecOpenSSLRsaOaepNodeRead, /* xmlSecTransformNodeReadMethod readNode; */ - NULL, /* xmlSecTransformNodeWriteMethod writeNode; */ - xmlSecOpenSSLRsaOaepSetKeyReq, /* xmlSecTransformSetKeyMethod setKeyReq; */ - xmlSecOpenSSLRsaOaepSetKey, /* xmlSecTransformSetKeyMethod setKey; */ - NULL, /* xmlSecTransformValidateMethod validate; */ - xmlSecTransformDefaultGetDataType, /* xmlSecTransformGetDataTypeMethod getDataType; */ - xmlSecTransformDefaultPushBin, /* xmlSecTransformPushBinMethod pushBin; */ - xmlSecTransformDefaultPopBin, /* xmlSecTransformPopBinMethod popBin; */ - NULL, /* xmlSecTransformPushXmlMethod pushXml; */ - NULL, /* xmlSecTransformPopXmlMethod popXml; */ - xmlSecOpenSSLRsaOaepExecute, /* xmlSecTransformExecuteMethod execute; */ - - NULL, /* void* reserved0; */ - NULL, /* void* reserved1; */ + sizeof(xmlSecTransformKlass), /* xmlSecSize klassSize */ + xmlSecOpenSSLRsaOaepSize, /* xmlSecSize objSize */ + + xmlSecNameRsaOaep, /* const xmlChar* name; */ + xmlSecHrefRsaOaep, /* const xmlChar* href; */ + xmlSecTransformUsageEncryptionMethod, /* xmlSecAlgorithmUsage usage; */ + + xmlSecOpenSSLRsaOaepInitialize, /* xmlSecTransformInitializeMethod initialize; */ + xmlSecOpenSSLRsaOaepFinalize, /* xmlSecTransformFinalizeMethod finalize; */ + xmlSecOpenSSLRsaOaepNodeRead, /* xmlSecTransformNodeReadMethod readNode; */ + NULL, /* xmlSecTransformNodeWriteMethod writeNode; */ + xmlSecOpenSSLRsaOaepSetKeyReq, /* xmlSecTransformSetKeyMethod setKeyReq; */ + xmlSecOpenSSLRsaOaepSetKey, /* xmlSecTransformSetKeyMethod setKey; */ + NULL, /* xmlSecTransformValidateMethod validate; */ + xmlSecTransformDefaultGetDataType, /* xmlSecTransformGetDataTypeMethod getDataType; */ + xmlSecTransformDefaultPushBin, /* xmlSecTransformPushBinMethod pushBin; */ + xmlSecTransformDefaultPopBin, /* xmlSecTransformPopBinMethod popBin; */ + NULL, /* xmlSecTransformPushXmlMethod pushXml; */ + NULL, /* xmlSecTransformPopXmlMethod popXml; */ + xmlSecOpenSSLRsaOaepExecute, /* xmlSecTransformExecuteMethod execute; */ + + NULL, /* void* reserved0; */ + NULL, /* void* reserved1; */ }; -/** +/** * xmlSecOpenSSLTransformRsaOaepGetKlass: * * The RSA-OAEP key transport transform klass. * * Returns: RSA-OAEP key transport transform klass. */ -xmlSecTransformId +xmlSecTransformId xmlSecOpenSSLTransformRsaOaepGetKlass(void) { return(&xmlSecOpenSSLRsaOaepKlass); } -static int +static int xmlSecOpenSSLRsaOaepInitialize(xmlSecTransformPtr transform) { xmlSecOpenSSLRsaOaepCtxPtr ctx; int ret; - + xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaOaepId), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaOaepSize), -1); ctx = xmlSecOpenSSLRsaOaepGetCtx(transform); xmlSecAssert2(ctx != NULL, -1); - memset(ctx, 0, sizeof(xmlSecOpenSSLRsaOaepCtx)); + memset(ctx, 0, sizeof(xmlSecOpenSSLRsaOaepCtx)); ret = xmlSecBufferInitialize(&(ctx->oaepParams), 0); if(ret < 0) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "xmlSecBufferInitialize", - XMLSEC_ERRORS_R_XMLSEC_FAILED, - XMLSEC_ERRORS_NO_MESSAGE); - return(-1); + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "xmlSecBufferInitialize", + XMLSEC_ERRORS_R_XMLSEC_FAILED, + XMLSEC_ERRORS_NO_MESSAGE); + return(-1); } return(0); } -static void +static void xmlSecOpenSSLRsaOaepFinalize(xmlSecTransformPtr transform) { xmlSecOpenSSLRsaOaepCtxPtr ctx; @@ -468,20 +468,20 @@ xmlSecOpenSSLRsaOaepFinalize(xmlSecTransformPtr transform) { ctx = xmlSecOpenSSLRsaOaepGetCtx(transform); xmlSecAssert(ctx != NULL); - + if(ctx->pKey != NULL) { - EVP_PKEY_free(ctx->pKey); + EVP_PKEY_free(ctx->pKey); } xmlSecBufferFinalize(&(ctx->oaepParams)); - memset(ctx, 0, sizeof(xmlSecOpenSSLRsaOaepCtx)); + memset(ctx, 0, sizeof(xmlSecOpenSSLRsaOaepCtx)); } -static int +static int xmlSecOpenSSLRsaOaepNodeRead(xmlSecTransformPtr transform, xmlNodePtr node, xmlSecTransformCtxPtr transformCtx) { xmlSecOpenSSLRsaOaepCtxPtr ctx; xmlNodePtr cur; int ret; - + xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaOaepId), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaOaepSize), -1); xmlSecAssert2(node != NULL, -1); @@ -490,64 +490,63 @@ xmlSecOpenSSLRsaOaepNodeRead(xmlSecTransformPtr transform, xmlNodePtr node, xmlS ctx = xmlSecOpenSSLRsaOaepGetCtx(transform); xmlSecAssert2(ctx != NULL, -1); xmlSecAssert2(xmlSecBufferGetSize(&(ctx->oaepParams)) == 0, -1); - + cur = xmlSecGetNextElementNode(node->children); - if((cur != NULL) && xmlSecCheckNodeName(cur, xmlSecNodeRsaOAEPparams, xmlSecEncNs)) { - ret = xmlSecBufferBase64NodeContentRead(&(ctx->oaepParams), cur); - if(ret < 0) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "xmlSecBufferBase64NodeContentRead", - XMLSEC_ERRORS_R_XMLSEC_FAILED, - XMLSEC_ERRORS_NO_MESSAGE); - return(-1); - } - cur = xmlSecGetNextElementNode(cur->next); - } - - if((cur != NULL) && xmlSecCheckNodeName(cur, xmlSecNodeDigestMethod, xmlSecDSigNs)) { - xmlChar* algorithm; - - /* Algorithm attribute is required */ - algorithm = xmlGetProp(cur, xmlSecAttrAlgorithm); - if(algorithm == NULL) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - xmlSecErrorsSafeString(xmlSecAttrAlgorithm), - XMLSEC_ERRORS_R_INVALID_NODE_ATTRIBUTE, - "node=%s", - xmlSecErrorsSafeString(xmlSecNodeGetName(cur))); - return(-1); + while(cur != NULL) { + if(xmlSecCheckNodeName(cur, xmlSecNodeRsaOAEPparams, xmlSecEncNs)) { + ret = xmlSecBufferBase64NodeContentRead(&(ctx->oaepParams), cur); + if(ret < 0) { + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "xmlSecBufferBase64NodeContentRead", + XMLSEC_ERRORS_R_XMLSEC_FAILED, + XMLSEC_ERRORS_NO_MESSAGE); + return(-1); + } + } else if(xmlSecCheckNodeName(cur, xmlSecNodeDigestMethod, xmlSecDSigNs)) { + xmlChar* algorithm; + + /* Algorithm attribute is required */ + algorithm = xmlGetProp(cur, xmlSecAttrAlgorithm); + if(algorithm == NULL) { + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + xmlSecErrorsSafeString(xmlSecAttrAlgorithm), + XMLSEC_ERRORS_R_INVALID_NODE_ATTRIBUTE, + "node=%s", + xmlSecErrorsSafeString(xmlSecNodeGetName(cur))); + return(-1); + } + + /* for now we support only sha1 */ + if(xmlStrcmp(algorithm, xmlSecHrefSha1) != 0) { + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + xmlSecErrorsSafeString(algorithm), + XMLSEC_ERRORS_R_INVALID_TRANSFORM, + "digest algorithm is not supported for rsa/oaep"); + xmlFree(algorithm); + return(-1); + } + xmlFree(algorithm); + } else { + /* not found */ + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + xmlSecErrorsSafeString(xmlSecNodeGetName(cur)), + XMLSEC_ERRORS_R_UNEXPECTED_NODE, + XMLSEC_ERRORS_NO_MESSAGE); + return(-1); } - /* for now we support only sha1 */ - if(xmlStrcmp(algorithm, xmlSecHrefSha1) != 0) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - xmlSecErrorsSafeString(algorithm), - XMLSEC_ERRORS_R_INVALID_TRANSFORM, - "digest algorithm is not supported for rsa/oaep"); - xmlFree(algorithm); - return(-1); - } - xmlFree(algorithm); - - cur = xmlSecGetNextElementNode(cur->next); + /* next node */ + cur = xmlSecGetNextElementNode(cur->next); } - if(cur != NULL) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - xmlSecErrorsSafeString(xmlSecNodeGetName(cur)), - XMLSEC_ERRORS_R_UNEXPECTED_NODE, - XMLSEC_ERRORS_NO_MESSAGE); - return(-1); - } - return(0); } -static int +static int xmlSecOpenSSLRsaOaepSetKeyReq(xmlSecTransformPtr transform, xmlSecKeyReqPtr keyReq) { xmlSecOpenSSLRsaOaepCtxPtr ctx; @@ -559,23 +558,23 @@ xmlSecOpenSSLRsaOaepSetKeyReq(xmlSecTransformPtr transform, xmlSecKeyReqPtr key ctx = xmlSecOpenSSLRsaOaepGetCtx(transform); xmlSecAssert2(ctx != NULL, -1); - keyReq->keyId = xmlSecOpenSSLKeyDataRsaId; + keyReq->keyId = xmlSecOpenSSLKeyDataRsaId; if(transform->operation == xmlSecTransformOperationEncrypt) { keyReq->keyType = xmlSecKeyDataTypePublic; - keyReq->keyUsage = xmlSecKeyUsageEncrypt; + keyReq->keyUsage = xmlSecKeyUsageEncrypt; } else { keyReq->keyType = xmlSecKeyDataTypePrivate; - keyReq->keyUsage = xmlSecKeyUsageDecrypt; + keyReq->keyUsage = xmlSecKeyUsageDecrypt; } - + return(0); } -static int +static int xmlSecOpenSSLRsaOaepSetKey(xmlSecTransformPtr transform, xmlSecKeyPtr key) { xmlSecOpenSSLRsaOaepCtxPtr ctx; EVP_PKEY* pKey; - + xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaOaepId), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaOaepSize), -1); @@ -588,30 +587,30 @@ xmlSecOpenSSLRsaOaepSetKey(xmlSecTransformPtr transform, xmlSecKeyPtr key) { pKey = xmlSecOpenSSLKeyDataRsaGetEvp(xmlSecKeyGetValue(key)); if(pKey == NULL) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "xmlSecOpenSSLKeyDataRsaGetEvp", - XMLSEC_ERRORS_R_XMLSEC_FAILED, - XMLSEC_ERRORS_NO_MESSAGE); - return(-1); + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "xmlSecOpenSSLKeyDataRsaGetEvp", + XMLSEC_ERRORS_R_XMLSEC_FAILED, + XMLSEC_ERRORS_NO_MESSAGE); + return(-1); } - xmlSecAssert2(pKey->type == EVP_PKEY_RSA, -1); - xmlSecAssert2(pKey->pkey.rsa != NULL, -1); - - ctx->pKey = xmlSecOpenSSLEvpKeyDup(pKey); + xmlSecAssert2(pKey->type == EVP_PKEY_RSA, -1); + xmlSecAssert2(pKey->pkey.rsa != NULL, -1); + + ctx->pKey = xmlSecOpenSSLEvpKeyDup(pKey); if(ctx->pKey == NULL) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "xmlSecOpenSSLEvpKeyDup", - XMLSEC_ERRORS_R_XMLSEC_FAILED, - XMLSEC_ERRORS_NO_MESSAGE); - return(-1); + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "xmlSecOpenSSLEvpKeyDup", + XMLSEC_ERRORS_R_XMLSEC_FAILED, + XMLSEC_ERRORS_NO_MESSAGE); + return(-1); } return(0); } -static int +static int xmlSecOpenSSLRsaOaepExecute(xmlSecTransformPtr transform, int last, xmlSecTransformCtxPtr transformCtx) { xmlSecOpenSSLRsaOaepCtxPtr ctx; int ret; @@ -626,37 +625,37 @@ xmlSecOpenSSLRsaOaepExecute(xmlSecTransformPtr transform, int last, xmlSecTransf xmlSecAssert2(ctx->pKey != NULL, -1); if(transform->status == xmlSecTransformStatusNone) { - transform->status = xmlSecTransformStatusWorking; - } - + transform->status = xmlSecTransformStatusWorking; + } + if((transform->status == xmlSecTransformStatusWorking) && (last == 0)) { - /* just do nothing */ + /* just do nothing */ } else if((transform->status == xmlSecTransformStatusWorking) && (last != 0)) { - ret = xmlSecOpenSSLRsaOaepProcess(transform, transformCtx); - if(ret < 0) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "xmlSecOpenSSLRsaOaepProcess", - XMLSEC_ERRORS_R_XMLSEC_FAILED, - XMLSEC_ERRORS_NO_MESSAGE); - return(-1); - } - transform->status = xmlSecTransformStatusFinished; + ret = xmlSecOpenSSLRsaOaepProcess(transform, transformCtx); + if(ret < 0) { + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "xmlSecOpenSSLRsaOaepProcess", + XMLSEC_ERRORS_R_XMLSEC_FAILED, + XMLSEC_ERRORS_NO_MESSAGE); + return(-1); + } + transform->status = xmlSecTransformStatusFinished; } else if(transform->status == xmlSecTransformStatusFinished) { - /* the only way we can get here is if there is no input */ - xmlSecAssert2(xmlSecBufferGetSize(&(transform->inBuf)) == 0, -1); + /* the only way we can get here is if there is no input */ + xmlSecAssert2(xmlSecBufferGetSize(&(transform->inBuf)) == 0, -1); } else { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - NULL, - XMLSEC_ERRORS_R_INVALID_STATUS, - "status=%d", transform->status); - return(-1); + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + NULL, + XMLSEC_ERRORS_R_INVALID_STATUS, + "status=%d", transform->status); + return(-1); } return(0); } -static int +static int xmlSecOpenSSLRsaOaepProcess(xmlSecTransformPtr transform, xmlSecTransformCtxPtr transformCtx) { xmlSecOpenSSLRsaOaepCtxPtr ctx; xmlSecSize paramsSize; @@ -673,203 +672,203 @@ xmlSecOpenSSLRsaOaepProcess(xmlSecTransformPtr transform, xmlSecTransformCtxPtr ctx = xmlSecOpenSSLRsaOaepGetCtx(transform); xmlSecAssert2(ctx != NULL, -1); xmlSecAssert2(ctx->pKey != NULL, -1); - xmlSecAssert2(ctx->pKey->type == EVP_PKEY_RSA, -1); - xmlSecAssert2(ctx->pKey->pkey.rsa != NULL, -1); - + xmlSecAssert2(ctx->pKey->type == EVP_PKEY_RSA, -1); + xmlSecAssert2(ctx->pKey->pkey.rsa != NULL, -1); + keySize = RSA_size(ctx->pKey->pkey.rsa); xmlSecAssert2(keySize > 0, -1); - + in = &(transform->inBuf); out = &(transform->outBuf); - + inSize = xmlSecBufferGetSize(in); - outSize = xmlSecBufferGetSize(out); + outSize = xmlSecBufferGetSize(out); xmlSecAssert2(outSize == 0, -1); /* the encoded size is equal to the keys size so we could not * process more than that */ if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - NULL, - XMLSEC_ERRORS_R_INVALID_SIZE, - "%d when expected less than %d", inSize, keySize); - return(-1); + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + NULL, + XMLSEC_ERRORS_R_INVALID_SIZE, + "%d when expected less than %d", inSize, keySize); + return(-1); } else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - NULL, - XMLSEC_ERRORS_R_INVALID_SIZE, - "%d when expected %d", inSize, keySize); - return(-1); + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + NULL, + XMLSEC_ERRORS_R_INVALID_SIZE, + "%d when expected %d", inSize, keySize); + return(-1); } - - outSize = keySize; + + outSize = keySize; ret = xmlSecBufferSetMaxSize(out, outSize); if(ret < 0) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "xmlSecBufferSetMaxSize", - XMLSEC_ERRORS_R_XMLSEC_FAILED, - "size=%d", outSize); - return(-1); + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "xmlSecBufferSetMaxSize", + XMLSEC_ERRORS_R_XMLSEC_FAILED, + "size=%d", outSize); + return(-1); } paramsSize = xmlSecBufferGetSize(&(ctx->oaepParams)); if((transform->operation == xmlSecTransformOperationEncrypt) && (paramsSize == 0)) { - /* encode w/o OAEPParams --> simple */ - ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in), - xmlSecBufferGetData(out), - ctx->pKey->pkey.rsa, RSA_PKCS1_OAEP_PADDING); - if(ret <= 0) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "RSA_public_encrypt(RSA_PKCS1_OAEP_PADDING)", - XMLSEC_ERRORS_R_CRYPTO_FAILED, - XMLSEC_ERRORS_NO_MESSAGE); - return(-1); - } - outSize = ret; + /* encode w/o OAEPParams --> simple */ + ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in), + xmlSecBufferGetData(out), + ctx->pKey->pkey.rsa, RSA_PKCS1_OAEP_PADDING); + if(ret <= 0) { + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "RSA_public_encrypt(RSA_PKCS1_OAEP_PADDING)", + XMLSEC_ERRORS_R_CRYPTO_FAILED, + XMLSEC_ERRORS_NO_MESSAGE); + return(-1); + } + outSize = ret; } else if((transform->operation == xmlSecTransformOperationEncrypt) && (paramsSize > 0)) { - xmlSecAssert2(xmlSecBufferGetData(&(ctx->oaepParams)) != NULL, -1); - - /* add space for padding */ - ret = xmlSecBufferSetMaxSize(in, keySize); + xmlSecAssert2(xmlSecBufferGetData(&(ctx->oaepParams)) != NULL, -1); + + /* add space for padding */ + ret = xmlSecBufferSetMaxSize(in, keySize); if(ret < 0) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "xmlSecBufferSetMaxSize", - XMLSEC_ERRORS_R_XMLSEC_FAILED, - "size=%d", keySize); - return(-1); - } - - /* add padding */ - ret = RSA_padding_add_PKCS1_OAEP(xmlSecBufferGetData(in), keySize, - xmlSecBufferGetData(in), inSize, - xmlSecBufferGetData(&(ctx->oaepParams)), - paramsSize); - if(ret != 1) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "RSA_padding_add_PKCS1_OAEP", - XMLSEC_ERRORS_R_CRYPTO_FAILED, - XMLSEC_ERRORS_NO_MESSAGE); - return(-1); - } - inSize = keySize; - - /* encode with OAEPParams */ - ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in), - xmlSecBufferGetData(out), - ctx->pKey->pkey.rsa, RSA_NO_PADDING); - if(ret <= 0) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "RSA_public_encrypt(RSA_NO_PADDING)", - XMLSEC_ERRORS_R_CRYPTO_FAILED, - XMLSEC_ERRORS_NO_MESSAGE); - return(-1); - } - outSize = ret; + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "xmlSecBufferSetMaxSize", + XMLSEC_ERRORS_R_XMLSEC_FAILED, + "size=%d", keySize); + return(-1); + } + + /* add padding */ + ret = RSA_padding_add_PKCS1_OAEP(xmlSecBufferGetData(in), keySize, + xmlSecBufferGetData(in), inSize, + xmlSecBufferGetData(&(ctx->oaepParams)), + paramsSize); + if(ret != 1) { + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "RSA_padding_add_PKCS1_OAEP", + XMLSEC_ERRORS_R_CRYPTO_FAILED, + XMLSEC_ERRORS_NO_MESSAGE); + return(-1); + } + inSize = keySize; + + /* encode with OAEPParams */ + ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in), + xmlSecBufferGetData(out), + ctx->pKey->pkey.rsa, RSA_NO_PADDING); + if(ret <= 0) { + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "RSA_public_encrypt(RSA_NO_PADDING)", + XMLSEC_ERRORS_R_CRYPTO_FAILED, + XMLSEC_ERRORS_NO_MESSAGE); + return(-1); + } + outSize = ret; } else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize == 0)) { - ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in), - xmlSecBufferGetData(out), - ctx->pKey->pkey.rsa, RSA_PKCS1_OAEP_PADDING); - if(ret <= 0) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "RSA_private_decrypt(RSA_PKCS1_OAEP_PADDING)", - XMLSEC_ERRORS_R_CRYPTO_FAILED, - XMLSEC_ERRORS_NO_MESSAGE); - return(-1); - } - outSize = ret; + ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in), + xmlSecBufferGetData(out), + ctx->pKey->pkey.rsa, RSA_PKCS1_OAEP_PADDING); + if(ret <= 0) { + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "RSA_private_decrypt(RSA_PKCS1_OAEP_PADDING)", + XMLSEC_ERRORS_R_CRYPTO_FAILED, + XMLSEC_ERRORS_NO_MESSAGE); + return(-1); + } + outSize = ret; } else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize != 0)) { - BIGNUM bn; - - ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in), - xmlSecBufferGetData(out), - ctx->pKey->pkey.rsa, RSA_NO_PADDING); - if(ret <= 0) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "RSA_private_decrypt(RSA_NO_PADDING)", - XMLSEC_ERRORS_R_CRYPTO_FAILED, - XMLSEC_ERRORS_NO_MESSAGE); - return(-1); - } - outSize = ret; - - /* - * the private decrypt w/o padding adds '0's at the begginning. - * it's not clear for me can I simply skip all '0's from the - * beggining so I have to do decode it back to BIGNUM and dump - * buffer again - */ - BN_init(&bn); - if(BN_bin2bn(xmlSecBufferGetData(out), outSize, &bn) == NULL) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "BN_bin2bn", - XMLSEC_ERRORS_R_CRYPTO_FAILED, - "size=%d", outSize); - BN_clear_free(&bn); - return(-1); - } - - ret = BN_bn2bin(&bn, xmlSecBufferGetData(out)); - if(ret <= 0) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "BN_bn2bin", - XMLSEC_ERRORS_R_CRYPTO_FAILED, - XMLSEC_ERRORS_NO_MESSAGE); - BN_clear_free(&bn); - return(-1); - } - BN_clear_free(&bn); - outSize = ret; - - ret = RSA_padding_check_PKCS1_OAEP(xmlSecBufferGetData(out), outSize, - xmlSecBufferGetData(out), outSize, - keySize, - xmlSecBufferGetData(&(ctx->oaepParams)), - paramsSize); - if(ret < 0) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "RSA_padding_check_PKCS1_OAEP", - XMLSEC_ERRORS_R_CRYPTO_FAILED, - XMLSEC_ERRORS_NO_MESSAGE); - return(-1); - } - outSize = ret; + BIGNUM bn; + + ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in), + xmlSecBufferGetData(out), + ctx->pKey->pkey.rsa, RSA_NO_PADDING); + if(ret <= 0) { + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "RSA_private_decrypt(RSA_NO_PADDING)", + XMLSEC_ERRORS_R_CRYPTO_FAILED, + XMLSEC_ERRORS_NO_MESSAGE); + return(-1); + } + outSize = ret; + + /* + * the private decrypt w/o padding adds '0's at the begginning. + * it's not clear for me can I simply skip all '0's from the + * beggining so I have to do decode it back to BIGNUM and dump + * buffer again + */ + BN_init(&bn); + if(BN_bin2bn(xmlSecBufferGetData(out), outSize, &bn) == NULL) { + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "BN_bin2bn", + XMLSEC_ERRORS_R_CRYPTO_FAILED, + "size=%d", outSize); + BN_clear_free(&bn); + return(-1); + } + + ret = BN_bn2bin(&bn, xmlSecBufferGetData(out)); + if(ret <= 0) { + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "BN_bn2bin", + XMLSEC_ERRORS_R_CRYPTO_FAILED, + XMLSEC_ERRORS_NO_MESSAGE); + BN_clear_free(&bn); + return(-1); + } + BN_clear_free(&bn); + outSize = ret; + + ret = RSA_padding_check_PKCS1_OAEP(xmlSecBufferGetData(out), outSize, + xmlSecBufferGetData(out), outSize, + keySize, + xmlSecBufferGetData(&(ctx->oaepParams)), + paramsSize); + if(ret < 0) { + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "RSA_padding_check_PKCS1_OAEP", + XMLSEC_ERRORS_R_CRYPTO_FAILED, + XMLSEC_ERRORS_NO_MESSAGE); + return(-1); + } + outSize = ret; } else { - xmlSecAssert2("we could not be here" == NULL, -1); - return(-1); + xmlSecAssert2("we could not be here" == NULL, -1); + return(-1); } ret = xmlSecBufferSetSize(out, outSize); if(ret < 0) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "xmlSecBufferSetSize", - XMLSEC_ERRORS_R_XMLSEC_FAILED, - "size=%d", outSize); - return(-1); + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "xmlSecBufferSetSize", + XMLSEC_ERRORS_R_XMLSEC_FAILED, + "size=%d", outSize); + return(-1); } - + ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { - xmlSecError(XMLSEC_ERRORS_HERE, - xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), - "xmlSecBufferRemoveHead", - XMLSEC_ERRORS_R_XMLSEC_FAILED, - "size=%d", inSize); - return(-1); + xmlSecError(XMLSEC_ERRORS_HERE, + xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), + "xmlSecBufferRemoveHead", + XMLSEC_ERRORS_R_XMLSEC_FAILED, + "size=%d", inSize); + return(-1); } - + return(0); } |