Last 8 bytes of RSA modulus is keyid for V3 pubkeys (#205080)

- ported from rpm5.org work of Jeff Johnson
- minimal fix suitable for 4.4.2.x too, full rpm5.org changes require
  api changes which would be ok for HEAD but that's a bigger task...

1 files changed, 34 insertions, 22 deletions

diff --git a/rpmio/rpmpgp.c b/rpmio/rpmpgp.c
index 09e3fa864..1a4abe4aa 100644
--- a/rpmio/rpmpgp.c
+++ b/rpmio/rpmpgp.c
@@ -875,27 +875,39 @@ int pgpPrtComment(pgpTag tag, const uint8_t *h, size_t hlen)
     return 0;
 }
 
-int pgpPubkeyFingerprint(const uint8_t * pkt, size_t pktlen,
-		pgpKeyID_t keyid)
+int pgpPubkeyFingerprint(const uint8_t * pkt, size_t pktlen, pgpKeyID_t keyid)
 {
-    const uint8_t *s = pkt;
+    unsigned int val = *pkt;
+    size_t plen, hlen;
+    pgpTag tag;
+    const uint8_t *se, *h;
     DIGEST_CTX ctx;
-    uint8_t version;
     int rc = -1;	/* assume failure. */
 
-    if (pkt[0] != 0x99)
+    if (!(val & 0x80))
 	return rc;
-    version = pkt[3];
 
-    switch (version) {
-    case 3:
-      {	pgpPktKeyV3 v = (pgpPktKeyV3) (pkt + 3);
+    if (val & 0x40) {
+	tag = (val & 0x3f);
+	plen = pgpLen(pkt+1, &hlen);
+    } else {
+	tag = (val >> 2) & 0xf;
+	plen = (1 << (val & 0x3));
+	hlen = pgpGrab(pkt+1, plen);
+    }
+    if (pktlen > 0 && 1 + plen + hlen > pktlen)
+	return rc;
+    
+    h = pkt + 1 + plen;
 
-	s += sizeof(pkt[0]) + sizeof(pkt[1]) + sizeof(pkt[2]) + sizeof(*v);
+    switch (h[0]) {
+    case 3:
+      {	pgpPktKeyV3 v = (pgpPktKeyV3) (h);
+	se = (uint8_t *)(v + 1);
 	switch (v->pubkey_algo) {
 	case PGPPUBKEYALGO_RSA:
-	    s += (pgpMpiLen(s) - 8);
-	    memmove(keyid, s, sizeof(keyid));
+	    se += pgpMpiLen(se);
+	    memmove(keyid, (se-8), 8);
 	    rc = 0;
 	    break;
 	default:	/* TODO: md5 of mpi bodies (i.e. no length) */
@@ -903,31 +915,31 @@ int pgpPubkeyFingerprint(const uint8_t * pkt, size_t pktlen,
 	}
       }	break;
     case 4:
-      {	pgpPktKeyV4 v = (pgpPktKeyV4) (pkt + 3);
-	uint8_t * SHA1 = NULL;
+      {	pgpPktKeyV4 v = (pgpPktKeyV4) (h);
+	uint8_t * d = NULL;
+	size_t dlen;
 	int i;
 
-	s += sizeof(pkt[0]) + sizeof(pkt[1]) + sizeof(pkt[2]) + sizeof(*v);
+	se = (uint8_t *)(v + 1);
 	switch (v->pubkey_algo) {
 	case PGPPUBKEYALGO_RSA:
 	    for (i = 0; i < 2; i++)
-		s += pgpMpiLen(s);
+		se += pgpMpiLen(se);
 	    break;
 	case PGPPUBKEYALGO_DSA:
 	    for (i = 0; i < 4; i++)
-		s += pgpMpiLen(s);
+		se += pgpMpiLen(se);
 	    break;
 	}
 
 	ctx = rpmDigestInit(PGPHASHALGO_SHA1, RPMDIGEST_NONE);
-	(void) rpmDigestUpdate(ctx, pkt, (s-pkt));
-	(void) rpmDigestFinal(ctx, (void **)&SHA1, NULL, 0);
+	(void) rpmDigestUpdate(ctx, pkt, (se-pkt));
+	(void) rpmDigestFinal(ctx, (void **)&d, &dlen, 0);
 
-	s = SHA1 + 12;
-	memmove(keyid, s, sizeof(keyid));
+	memmove(keyid, (d + (dlen-8)), 8);
+	if (d) free(d);
 	rc = 0;
 
-	if (SHA1) free(SHA1);
       }	break;
     }
     return rc;