Use NSS instead of beecrypt for encryption (Tomas Mraz)

6 files changed, 39 insertions, 76 deletions

diff --git a/lib/Makefile.am b/lib/Makefile.am
index 89b0f7b5f..0fbfcd379 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -4,7 +4,7 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)
 AM_CPPFLAGS += -I$(top_srcdir)/lib
 AM_CPPFLAGS += -I$(top_builddir)/rpmdb -I$(top_srcdir)/rpmdb
 AM_CPPFLAGS += -I$(top_srcdir)/rpmio
-AM_CPPFLAGS += @WITH_BEECRYPT_INCLUDE@
+AM_CPPFLAGS += @WITH_NSS_INCLUDE@
 AM_CPPFLAGS += @WITH_POPT_INCLUDE@
 AM_CPPFLAGS += -I$(top_srcdir)/misc
 AM_CPPFLAGS += -DLOCALEDIR="\"$(localedir)\""
@@ -33,7 +33,7 @@ librpm_la_LDFLAGS = -release 4.4
 librpm_la_LIBADD = \
 	$(top_builddir)/rpmdb/librpmdb.la \
 	$(top_builddir)/rpmio/librpmio.la \
-	@WITH_BEECRYPT_LIB@ \
+	@WITH_NSS_LIB@ \
 	@WITH_POPT_LIB@ \
 	@WITH_SELINUX_LIB@ \
 	@LIBINTL@
diff --git a/lib/formats.c b/lib/formats.c
index 76594cf95..3f93dfebe 100644
--- a/lib/formats.c
+++ b/lib/formats.c
@@ -190,20 +190,16 @@ static char * base64Format(int32_t type, const void * data,
 	int lc;
 	/* XXX HACK ALERT: element field abused as no. bytes of binary data. */
 	size_t ns = element;
-	size_t nt = ((ns + 2) / 3) * 4;
-
-	/* Add additional bytes necessary for eol string(s). */
-	if (b64encode_chars_per_line > 0 && b64encode_eolstr != NULL) {
-	    lc = (nt + b64encode_chars_per_line - 1) / b64encode_chars_per_line;
-        if (((nt + b64encode_chars_per_line - 1) % b64encode_chars_per_line) != 0)
-            ++lc;
-	    nt += lc * strlen(b64encode_eolstr);
+	size_t nt = 0;
+
+	if ((enc = b64encode(data, ns, -1)) != NULL) {
+		nt = strlen(enc);
 	}
 
 	val = t = xmalloc(nt + padding + 1);
 
 	*t = '\0';
-	if ((enc = b64encode(data, ns)) != NULL) {
+	if (enc != NULL) {
 	    t = stpcpy(t, enc);
 	    enc = _free(enc);
 	}
@@ -277,10 +273,13 @@ static char * xmlFormat(int32_t type, const void * data,
 	xtag = "string";
 	break;
     case RPM_BIN_TYPE:
-    {	int cpl = b64encode_chars_per_line;
-	b64encode_chars_per_line = 0;
-	s = base64Format(type, data, formatPrefix, padding, element);
-	b64encode_chars_per_line = cpl;
+    {	
+	/* XXX HACK ALERT: element field abused as no. bytes of binary data. */
+	size_t ns = element;
+    	if ((s = b64encode(data, ns, 0)) == NULL) {
+    		/* XXX proper error handling would be better. */
+    		s = xcalloc(1, padding + (ns / 3) * 4 + 1);
+    	}
 	xtag = "base64";
     }	break;
     case RPM_CHAR_TYPE:
diff --git a/lib/package.c b/lib/package.c
index 9deeb80bc..525c5d7a5 100644
--- a/lib/package.c
+++ b/lib/package.c
@@ -929,11 +929,9 @@ rpmRC rpmReadPackageFile(rpmts ts, FD_t fd, const char * fn, Header * hdrp)
 		fddig->hashctx = NULL;
 		break;
 	    case PGPHASHALGO_SHA1:
-#if HAVE_BEECRYPT_API_H
 	    case PGPHASHALGO_SHA256:
 	    case PGPHASHALGO_SHA384:
 	    case PGPHASHALGO_SHA512:
-#endif
 		dig->sha1ctx = fddig->hashctx;
 		fddig->hashctx = NULL;
 		break;
diff --git a/lib/rpmchecksig.c b/lib/rpmchecksig.c
index 46abd8297..57b2440b9 100644
--- a/lib/rpmchecksig.c
+++ b/lib/rpmchecksig.c
@@ -518,11 +518,9 @@ assert(dig->md5ctx == NULL);
 	    fddig->hashctx = NULL;
 	    break;
 	case PGPHASHALGO_SHA1:
-#if HAVE_BEECRYPT_API_H
 	case PGPHASHALGO_SHA256:
 	case PGPHASHALGO_SHA384:
 	case PGPHASHALGO_SHA512:
-#endif
 assert(dig->sha1ctx == NULL);
 	    dig->sha1ctx = fddig->hashctx;
 	    fddig->hashctx = NULL;
diff --git a/lib/rpmts.c b/lib/rpmts.c
index 13a7626b9..31ad5b534 100644
--- a/lib/rpmts.c
+++ b/lib/rpmts.c
@@ -4,7 +4,7 @@
  */
 #include "system.h"
 
-#include "rpmio_internal.h"	/* XXX for pgp and beecrypt */
+#include "rpmio_internal.h"	/* XXX for pgp */
 #include "rpmlib.h"
 #include "rpmmacro.h"		/* XXX rpmtsOpenDB() needs rpmGetPath */
 
@@ -483,7 +483,7 @@ rpmRC rpmtsImportPubkey(const rpmts ts, const unsigned char * pkt, ssize_t pktle
     if (rpmtsOpenDB(ts, (O_RDWR|O_CREAT)))
 	return RPMRC_FAIL;
 
-    if ((enc = b64encode(pkt, pktlen)) == NULL)
+    if ((enc = b64encode(pkt, pktlen, -1)) == NULL)
 	goto exit;
 
     dig = pgpNewDig();
diff --git a/lib/signature.c b/lib/signature.c
index 9644dcc9e..a839232a1 100644
--- a/lib/signature.c
+++ b/lib/signature.c
@@ -1093,9 +1093,10 @@ verifyRSASignature(rpmts ts, char * t,
     int32_t sigtag = rpmtsSigtag(ts);
     pgpDig dig = rpmtsDig(ts);
     pgpDigParams sigp = rpmtsSignature(ts);
-    const char * prefix = NULL;
+    SECOidTag sigalg;
     rpmRC res = RPMRC_OK;
     int xx;
+    SECItem digest;
 
     *t = '\0';
     if (dig != NULL && dig->hdrmd5ctx == md5ctx)
@@ -1125,43 +1126,40 @@ verifyRSASignature(rpmts ts, char * t,
     switch (sigp->hash_algo) {
     case PGPHASHALGO_MD5:
 	t = stpcpy(t, " RSA/MD5");
-	prefix = "3020300c06082a864886f70d020505000410";
+	sigalg = SEC_OID_PKCS1_MD5_WITH_RSA_ENCRYPTION;
 	break;
     case PGPHASHALGO_SHA1:
 	t = stpcpy(t, " RSA/SHA1");
-	prefix = "3021300906052b0e03021a05000414";
+	sigalg = SEC_OID_PKCS1_SHA1_WITH_RSA_ENCRYPTION;
 	break;
     case PGPHASHALGO_RIPEMD160:
 	res = RPMRC_NOKEY;
-	prefix = NULL;
 	break;
     case PGPHASHALGO_MD2:
 	t = stpcpy(t, " RSA/MD2");
-	prefix = "3020300c06082a864886f70d020205000410";
+	sigalg = SEC_OID_PKCS1_MD2_WITH_RSA_ENCRYPTION;
 	break;
     case PGPHASHALGO_TIGER192:
 	res = RPMRC_NOKEY;
-	prefix = NULL;
 	break;
     case PGPHASHALGO_HAVAL_5_160:
 	res = RPMRC_NOKEY;
-	prefix = NULL;
 	break;
     case PGPHASHALGO_SHA256:
 	t = stpcpy(t, " RSA/SHA256");
-	prefix = "3031300d060960864801650304020105000420";
+	sigalg = SEC_OID_PKCS1_SHA256_WITH_RSA_ENCRYPTION;
 	break;
     case PGPHASHALGO_SHA384:
 	t = stpcpy(t, " RSA/SHA384");
-	prefix = "3041300d060960864801650304020205000430";
+	sigalg = SEC_OID_PKCS1_SHA384_WITH_RSA_ENCRYPTION;
 	break;
     case PGPHASHALGO_SHA512:
 	t = stpcpy(t, " RSA/SHA512");
-	prefix = "3051300d060960864801650304020305000440";
+	sigalg = SEC_OID_PKCS1_SHA512_WITH_RSA_ENCRYPTION;
 	break;
     default:
 	res = RPMRC_NOKEY;
-	prefix = NULL;
+	sigalg = SEC_OID_UNKNOWN;
 	break;
     }
 
@@ -1172,8 +1170,6 @@ verifyRSASignature(rpmts ts, char * t,
 
     (void) rpmswEnter(rpmtsOp(ts, RPMTS_OP_DIGEST), 0);
     {	DIGEST_CTX ctx = rpmDigestDup(md5ctx);
-	byte signhash16[2];
-	const char * s;
 
 	if (sigp->hash != NULL)
 	    xx = rpmDigestUpdate(ctx, sigp->hash, sigp->hashlen);
@@ -1190,40 +1186,18 @@ verifyRSASignature(rpmts ts, char * t,
 	}
 #endif
 
-	xx = rpmDigestFinal(ctx, (void **)&dig->md5, &dig->md5len, 1);
+	xx = rpmDigestFinal(ctx, (void **)&dig->md5, &dig->md5len, 0);
 	(void) rpmswExit(rpmtsOp(ts, RPMTS_OP_DIGEST), sigp->hashlen);
 	rpmtsOp(ts, RPMTS_OP_DIGEST)->count--;	/* XXX one too many */
 
 	/* Compare leading 16 bits of digest for quick check. */
-	s = dig->md5;
-	signhash16[0] = (nibble(s[0]) << 4) | nibble(s[1]);
-	signhash16[1] = (nibble(s[2]) << 4) | nibble(s[3]);
-	if (memcmp(signhash16, sigp->signhash16, sizeof(signhash16))) {
+	if (memcmp(dig->md5, sigp->signhash16, 2)) {
 	    res = RPMRC_FAIL;
 	    goto exit;
 	}
-    }
-
-    /* Generate RSA modulus parameter. */
-    {	unsigned int nbits = MP_WORDS_TO_BITS(dig->c.size);
-	unsigned int nb = (nbits + 7) >> 3;
-	const char * hexstr;
-	char * tt;
-
-assert(prefix != NULL);
-	hexstr = tt = xmalloc(2 * nb + 1);
-	memset(tt, 'f', (2 * nb));
-	tt[0] = '0'; tt[1] = '0';
-	tt[2] = '0'; tt[3] = '1';
-	tt += (2 * nb) - strlen(prefix) - strlen(dig->md5) - 2;
-	*tt++ = '0'; *tt++ = '0';
-	tt = stpcpy(tt, prefix);
-	tt = stpcpy(tt, dig->md5);
-
-	mpnzero(&dig->rsahm);	(void) mpnsethex(&dig->rsahm, hexstr);
-
-	hexstr = _free(hexstr);
-
+	digest.type = siBuffer;
+	digest.data = dig->md5;
+	digest.len = dig->md5len;
     }
 
     /* Retrieve the matching public key. */
@@ -1232,12 +1206,7 @@ assert(prefix != NULL);
 	goto exit;
 
     (void) rpmswEnter(rpmtsOp(ts, RPMTS_OP_SIGNATURE), 0);
-#if HAVE_BEECRYPT_API_H
-    xx = rsavrfy(&dig->rsa_pk.n, &dig->rsa_pk.e, &dig->c, &dig->rsahm);
-#else
-    xx = rsavrfy(&dig->rsa_pk, &dig->rsahm, &dig->c);
-#endif
-    if (xx)
+    if (VFY_VerifyDigest(&digest, dig->rsa, dig->rsasig, sigalg, NULL) == SECSuccess)
 	res = RPMRC_OK;
     else
 	res = RPMRC_FAIL;
@@ -1274,6 +1243,7 @@ verifyDSASignature(rpmts ts, char * t,
     pgpDigParams sigp = rpmtsSignature(ts);
     rpmRC res;
     int xx;
+    SECItem digest;
 
     *t = '\0';
     if (dig != NULL && dig->hdrsha1ctx == sha1ctx)
@@ -1301,7 +1271,6 @@ verifyDSASignature(rpmts ts, char * t,
 
     (void) rpmswEnter(rpmtsOp(ts, RPMTS_OP_DIGEST), 0);
     {	DIGEST_CTX ctx = rpmDigestDup(sha1ctx);
-	byte signhash16[2];
 
 	if (sigp->hash != NULL)
 	    xx = rpmDigestUpdate(ctx, sigp->hash, sigp->hashlen);
@@ -1315,19 +1284,18 @@ verifyDSASignature(rpmts ts, char * t,
 	    memcpy(trailer+2, &nb, sizeof(nb));
 	    xx = rpmDigestUpdate(ctx, trailer, sizeof(trailer));
 	}
-	xx = rpmDigestFinal(ctx, (void **)&dig->sha1, &dig->sha1len, 1);
+	xx = rpmDigestFinal(ctx, (void **)&dig->sha1, &dig->sha1len, 0);
 	(void) rpmswExit(rpmtsOp(ts, RPMTS_OP_DIGEST), sigp->hashlen);
 	rpmtsOp(ts, RPMTS_OP_DIGEST)->count--;	/* XXX one too many */
 
-	mpnzero(&dig->hm);	(void) mpnsethex(&dig->hm, dig->sha1);
-
 	/* Compare leading 16 bits of digest for quick check. */
-	signhash16[0] = (*dig->hm.data >> 24) & 0xff;
-	signhash16[1] = (*dig->hm.data >> 16) & 0xff;
-	if (memcmp(signhash16, sigp->signhash16, sizeof(signhash16))) {
+	if (memcmp(dig->sha1, sigp->signhash16, 2)) {
 	    res = RPMRC_FAIL;
 	    goto exit;
 	}
+	digest.type = siBuffer;
+	digest.data = dig->sha1;
+	digest.len = dig->sha1len;
     }
 
     /* Retrieve the matching public key. */
@@ -1336,8 +1304,8 @@ verifyDSASignature(rpmts ts, char * t,
 	goto exit;
 
     (void) rpmswEnter(rpmtsOp(ts, RPMTS_OP_SIGNATURE), 0);
-    if (dsavrfy(&dig->p, &dig->q, &dig->g,
-		&dig->hm, &dig->y, &dig->r, &dig->s))
+    if (VFY_VerifyDigest(&digest, dig->dsa, dig->dsasig,
+    		SEC_OID_ANSIX9_DSA_SIGNATURE_WITH_SHA1_DIGEST, NULL) == SECSuccess)
 	res = RPMRC_OK;
     else
 	res = RPMRC_FAIL;