[CVE-2020-8623] A flaw in native PKCS#11 code can lead to a remotely triggerable assertion failure in pk11.csubmit/tizen_6.0/20230621.060633accepted/tizen/6.0/unified/20230626.074933tizen_6.0accepted/tizen_6.0_unified

Change-Id: Idb5f01dbc5d4f4827a2d420e5c4720f36e81540e

3 files changed, 97 insertions, 45 deletions

diff --git a/lib/dns/pkcs11rsa_link.c b/lib/dns/pkcs11rsa_link.c
index 0473acd7..480a24ab 100644
--- a/lib/dns/pkcs11rsa_link.c
+++ b/lib/dns/pkcs11rsa_link.c
@@ -309,7 +309,7 @@ pkcs11rsa_createctx_verify(dst_key_t *key, unsigned int maxbits,
 		key->key_alg == DST_ALG_NSEC3RSASHA1 ||
 		key->key_alg == DST_ALG_RSASHA256 ||
 		key->key_alg == DST_ALG_RSASHA512);
-
+	REQUIRE(maxbits <= RSA_MAX_PUBEXP_BITS);
 	/*
 	 * Reject incorrect RSA key lengths.
 	 */
@@ -352,6 +352,7 @@ pkcs11rsa_createctx_verify(dst_key_t *key, unsigned int maxbits,
 	for (attr = pk11_attribute_first(rsa);
 	     attr != NULL;
 	     attr = pk11_attribute_next(rsa, attr))
+	{
 		switch (attr->type) {
 		case CKA_MODULUS:
 			INSIST(keyTemplate[5].type == attr->type);
@@ -372,12 +373,16 @@ pkcs11rsa_createctx_verify(dst_key_t *key, unsigned int maxbits,
 			memmove(keyTemplate[6].pValue, attr->pValue,
 				attr->ulValueLen);
 			keyTemplate[6].ulValueLen = attr->ulValueLen;
-			if (pk11_numbits(attr->pValue,
-					 attr->ulValueLen) > maxbits &&
-			    maxbits != 0)
+			unsigned int bits;
+			ret = pk11_numbits(attr->pValue, attr->ulValueLen,
+					   &bits);
+			if (ret != ISC_R_SUCCESS ||
+			    (bits > maxbits && maxbits != 0)) {			
 				DST_RET(DST_R_VERIFYFAILURE);
+			}
 			break;
 		}
+	}
 	pk11_ctx->object = CK_INVALID_HANDLE;
 	pk11_ctx->ontoken = false;
 	PK11_RET(pkcs_C_CreateObject,
@@ -1386,6 +1391,8 @@ pkcs11rsa_fromdns(dst_key_t *key, isc_buffer_t *data) {
 	CK_BYTE *exponent = NULL, *modulus = NULL;
 	CK_ATTRIBUTE *attr;
 	unsigned int length;
+	unsigned int bits;
+	isc_result_t ret = ISC_R_SUCCESS;	
 
 	isc_buffer_remainingregion(data, &r);
 	if (r.length == 0)
@@ -1403,9 +1410,7 @@ pkcs11rsa_fromdns(dst_key_t *key, isc_buffer_t *data) {
 
 	if (e_bytes == 0) {
 		if (r.length < 2) {
-			isc_safe_memwipe(rsa, sizeof(*rsa));
-			isc_mem_put(key->mctx, rsa, sizeof(*rsa));
-			return (DST_R_INVALIDPUBLICKEY);
+			DST_RET(DST_R_INVALIDPUBLICKEY);
 		}
 		e_bytes = (*r.base) << 8;
 		isc_region_consume(&r, 1);
@@ -1414,17 +1419,18 @@ pkcs11rsa_fromdns(dst_key_t *key, isc_buffer_t *data) {
 	}
 
 	if (r.length < e_bytes) {
-		isc_safe_memwipe(rsa, sizeof(*rsa));
-		isc_mem_put(key->mctx, rsa, sizeof(*rsa));
-		return (DST_R_INVALIDPUBLICKEY);
+		DST_RET(DST_R_INVALIDPUBLICKEY);
 	}
 	exponent = r.base;
 	isc_region_consume(&r, e_bytes);
 	modulus = r.base;
 	mod_bytes = r.length;
 
-	key->key_size = pk11_numbits(modulus, mod_bytes);
-
+	ret = pk11_numbits(modulus, mod_bytes, &bits);
+	if (ret != ISC_R_SUCCESS) {
+		goto err;
+	}
+	key->key_size = bits;
 	isc_buffer_forward(data, length);
 
 	rsa->repr = (CK_ATTRIBUTE *) isc_mem_get(key->mctx, sizeof(*attr) * 2);
@@ -1450,7 +1456,7 @@ pkcs11rsa_fromdns(dst_key_t *key, isc_buffer_t *data) {
 
 	return (ISC_R_SUCCESS);
 
-    nomemory:
+	nomemory:
 	for (attr = pk11_attribute_first(rsa);
 	     attr != NULL;
 	     attr = pk11_attribute_next(rsa, attr))
@@ -1476,6 +1482,11 @@ pkcs11rsa_fromdns(dst_key_t *key, isc_buffer_t *data) {
 	isc_safe_memwipe(rsa, sizeof(*rsa));
 	isc_mem_put(key->mctx, rsa, sizeof(*rsa));
 	return (ISC_R_NOMEMORY);
+
+	err:
+	isc_safe_memwipe(rsa, sizeof(*rsa));
+	isc_mem_put(key->mctx, rsa, sizeof(*rsa));
+	return (ret)
 }
 
 static isc_result_t
@@ -1654,6 +1665,7 @@ pkcs11rsa_fetch(dst_key_t *key, const char *engine, const char *label,
 	pk11_object_t *pubrsa;
 	pk11_context_t *pk11_ctx = NULL;
 	isc_result_t ret;
+	unsigned int bits;
 
 	if (label == NULL)
 		return (DST_R_NOENGINE);
@@ -1740,7 +1752,11 @@ pkcs11rsa_fetch(dst_key_t *key, const char *engine, const char *label,
 
 	attr = pk11_attribute_bytype(rsa, CKA_MODULUS);
 	INSIST(attr != NULL);
-	key->key_size = pk11_numbits(attr->pValue, attr->ulValueLen);
+	ret = pk11_numbits(attr->pValue, attr->ulValueLen, &bits);
+	if (ret != ISC_R_SUCCESS) {
+		goto err;
+	}
+	key->key_size = bits;
 
 	return (ISC_R_SUCCESS);
 
@@ -1826,6 +1842,7 @@ pkcs11rsa_parse(dst_key_t *key, isc_lex_t *lexer, dst_key_t *pub) {
 	CK_ATTRIBUTE *attr;
 	isc_mem_t *mctx = key->mctx;
 	const char *engine = NULL, *label = NULL;
+	unsigned int bits;
 
 	/* read private key file */
 	ret = dst__privstruct_parse(key, DST_ALG_RSA, lexer, mctx, &priv);
@@ -1969,13 +1986,21 @@ pkcs11rsa_parse(dst_key_t *key, isc_lex_t *lexer, dst_key_t *pub) {
 
 	attr = pk11_attribute_bytype(rsa, CKA_MODULUS);
 	INSIST(attr != NULL);
-	key->key_size = pk11_numbits(attr->pValue, attr->ulValueLen);
-
+	ret = pk11_numbits(attr->pValue, attr->ulValueLen, &bits);
+	if (ret != ISC_R_SUCCESS) {
+		goto err;
+	}
+	key->key_size = bits;
 	attr = pk11_attribute_bytype(rsa, CKA_PUBLIC_EXPONENT);
 	INSIST(attr != NULL);
-	if (pk11_numbits(attr->pValue, attr->ulValueLen) > RSA_MAX_PUBEXP_BITS)
-		DST_RET(ISC_R_RANGE);
 
+	ret = pk11_numbits(attr->pValue, attr->ulValueLen, &bits);
+	if (ret != ISC_R_SUCCESS) {
+		goto err;
+	}
+	if (bits > RSA_MAX_PUBEXP_BITS) {
+		DST_RET(ISC_R_RANGE);
+	}
 	dst__privstruct_free(&priv, mctx);
 	isc_safe_memwipe(&priv, sizeof(priv));
 
@@ -2009,6 +2034,7 @@ pkcs11rsa_fromlabel(dst_key_t *key, const char *engine, const char *label,
 	pk11_context_t *pk11_ctx = NULL;
 	isc_result_t ret;
 	unsigned int i;
+	unsigned int bits;
 
 	UNUSED(pin);
 
@@ -2103,13 +2129,22 @@ pkcs11rsa_fromlabel(dst_key_t *key, const char *engine, const char *label,
 
 	attr = pk11_attribute_bytype(rsa, CKA_PUBLIC_EXPONENT);
 	INSIST(attr != NULL);
-	if (pk11_numbits(attr->pValue, attr->ulValueLen) > RSA_MAX_PUBEXP_BITS)
-		DST_RET(ISC_R_RANGE);
+	ret = pk11_numbits(attr->pValue, attr->ulValueLen, &bits);
+	if (ret != ISC_R_SUCCESS) {
+		goto err;
+	}
+	if (bits > RSA_MAX_PUBEXP_BITS) {
+ 		DST_RET(ISC_R_RANGE);
+ 	}
 
 	attr = pk11_attribute_bytype(rsa, CKA_MODULUS);
 	INSIST(attr != NULL);
-	key->key_size = pk11_numbits(attr->pValue, attr->ulValueLen);
-
+	ret = pk11_numbits(attr->pValue, attr->ulValueLen, &bits);
+	if (ret != ISC_R_SUCCESS) {
+		goto err;
+	}
+	key->key_size = bits;
+	
 	pk11_return_session(pk11_ctx);
 	isc_safe_memwipe(pk11_ctx, sizeof(*pk11_ctx));
 	isc_mem_put(key->mctx, pk11_ctx, sizeof(*pk11_ctx));
diff --git a/lib/isc/include/pk11/internal.h b/lib/isc/include/pk11/internal.h
index aa8907ab..563bf91d 100644
--- a/lib/isc/include/pk11/internal.h
+++ b/lib/isc/include/pk11/internal.h
@@ -25,7 +25,7 @@ void pk11_mem_put(void *ptr, size_t size);
 
 CK_SLOT_ID pk11_get_best_token(pk11_optype_t optype);
 
-unsigned int pk11_numbits(CK_BYTE_PTR data, unsigned int bytecnt);
+isc_result_t pk11_numbits(CK_BYTE_PTR data, unsigned int bytecnt, unsigned int *bits);
 
 CK_ATTRIBUTE *pk11_attribute_first(const pk11_object_t *obj);
 
diff --git a/lib/isc/pk11.c b/lib/isc/pk11.c
index 49861f2e..f95bcd75 100644
--- a/lib/isc/pk11.c
+++ b/lib/isc/pk11.c
@@ -636,13 +636,15 @@ pk11_get_best_token(pk11_optype_t optype) {
 	return (token->slotid);
 }
 
-unsigned int
-pk11_numbits(CK_BYTE_PTR data, unsigned int bytecnt) {
+isc_result_t
+pk11_numbits(CK_BYTE_PTR data, unsigned int bytecnt, unsigned int *bits) {
 	unsigned int bitcnt, i;
 	CK_BYTE top;
 
-	if (bytecnt == 0)
-		return (0);
+	if (bytecnt == 0) {
+		*bits = 0;
+		return (ISC_R_SUCCESS);
+	}
 	bitcnt = bytecnt * 8;
 	for (i = 0; i < bytecnt; i++) {
 		top = data[i];
@@ -650,26 +652,41 @@ pk11_numbits(CK_BYTE_PTR data, unsigned int bytecnt) {
 			bitcnt -= 8;
 			continue;
 		}
-		if (top & 0x80)
-			return (bitcnt);
-		if (top & 0x40)
-			return (bitcnt - 1);
-		if (top & 0x20)
-			return (bitcnt - 2);
-		if (top & 0x10)
-			return (bitcnt - 3);
-		if (top & 0x08)
-			return (bitcnt - 4);
-		if (top & 0x04)
-			return (bitcnt - 5);
-		if (top & 0x02)
-			return (bitcnt - 6);
-		if (top & 0x01)
-			return (bitcnt - 7);
+		if (top & 0x80) {
+			*bits = bitcnt;
+			return (ISC_R_SUCCESS);
+		}
+		if (top & 0x40) {
+			*bits = bitcnt - 1;
+			return (ISC_R_SUCCESS);
+		}
+		if (top & 0x20) {
+			*bits = bitcnt - 2;
+			return (ISC_R_SUCCESS);
+		}
+		if (top & 0x10) {
+			*bits = bitcnt - 3;
+			return (ISC_R_SUCCESS);
+		}
+		if (top & 0x08) {
+			*bits = bitcnt - 4;
+			return (ISC_R_SUCCESS);
+		}
+		if (top & 0x04) {
+			*bits = bitcnt - 5;
+			return (ISC_R_SUCCESS);
+		}
+		if (top & 0x02) {
+			*bits = bitcnt - 6;
+			return (ISC_R_SUCCESS);
+		}
+		if (top & 0x01) {
+			*bits = bitcnt - 7;
+			return (ISC_R_SUCCESS);
+		}
 		break;
 	}
-	INSIST(0);
-	ISC_UNREACHABLE();
+	return (ISC_R_RANGE);
 }
 
 CK_ATTRIBUTE *