Fix bugs in MD5, SHA256, and SHA512 setting-string generation.

Namely:

 + The SHA256 and SHA512 crypt algorithms accept up to 16 bytes of
   salt, but crypt_gensalt* would only emit 4 or 8 bytes of salt, even
   if more randomness was available.
 + MD5 does not have an adjustable number of rounds, but for
   consistency with all the other algorithms, crypt_gensalt* should
   accept a count parameter equal to either 0 or the actual fixed
   number of rounds that it uses (1000).
 + For SHA256 and SHA512, compute the required output size up-front
   rather than relying on snprintf and thus clobbering the buffer when
   there might not have been enough space.

In addition, crypt-gensalt.c now only contains a helper function;
all of the gensalt_* algorithm entry points are moved to their
respective modules.  (This is actually necessary in order to gain
access to some internal #defines for MD5/SHA256/SHA512.)

	* crypt-gensalt.c (gensalt_sha_rn): No longer static.
	Add more parameters: maxsalt, defcount, mincount, maxcount.
	Compute minimum output length upfront and do not overwrite
	the output buffer if there isn't enough space.  Allow arbitrarily
	long salt strings in the output (up to maxsalt).
	(gensalt_des_trd_rn, gensalt_des_xbsd_rn): Move to crypt-des.c.
	Use that file's base64 table ("ascii64" instead of "_xcrypt_itoa64").
	(gensalt_md5_rn): Move to crypt-md5.c; replace body with a call to
	gensalt_sha_rn.
	(gensalt_sha256_rn): Move to crypt-sha256.c; add new parameters to
	gensalt_sha_rn call.
	(gensalt_sha256_rn): Move to crypt-sha512.c; add new parameters to
	gensalt_sha_rn call.
	* crypt-port.h: Add gensalt_sha_rn to renames list.
	* crypt-private.h: Declare gensalt_sha_rn.
	* test-gensalt.c: Expect 16-character salts from SHA-{256,512}.

1 files changed, 64 insertions, 0 deletions

diff --git a/crypt-des.c b/crypt-des.c
index 7e9610c..2c4384d 100644
--- a/crypt-des.c
+++ b/crypt-des.c
@@ -342,3 +342,67 @@ crypt_des_xbsd_rn (const char *phrase, const char *setting,
   des_set_salt (ctx, salt);
   des_gen_hash (ctx, count, cp, pkbuf);
 }
+
+void
+gensalt_des_trd_rn (unsigned long count,
+                    const uint8_t *rbytes, size_t nrbytes,
+                    uint8_t *output, size_t output_size)
+{
+  if (output_size < 3)
+    {
+      errno = ERANGE;
+      return;
+    }
+
+  if (nrbytes < 2 || (count != 0 && count != 25))
+    {
+      errno = EINVAL;
+      return;
+    }
+
+  output[0] = ascii64[(unsigned int) rbytes[0] & 0x3f];
+  output[1] = ascii64[(unsigned int) rbytes[1] & 0x3f];
+  output[2] = '\0';
+}
+
+void
+gensalt_des_xbsd_rn (unsigned long count,
+                    const uint8_t *rbytes, size_t nrbytes,
+                    uint8_t *output, size_t output_size)
+{
+  if (output_size < 1 + 4 + 4 + 1)
+    {
+      errno = ERANGE;
+      return;
+    }
+
+  if (count == 0)
+    count = 725;
+
+  /* Even iteration counts make it easier to detect weak DES keys from a look
+     at the hash, so they should be avoided.  */
+  if (nrbytes < 3 || count > 0xffffff || count % 2 == 0)
+    {
+      errno = EINVAL;
+      return;
+    }
+
+  unsigned long value =
+    ((unsigned long) (unsigned char) rbytes[0] <<  0) |
+    ((unsigned long) (unsigned char) rbytes[1] <<  8) |
+    ((unsigned long) (unsigned char) rbytes[2] << 16);
+
+  output[0] = '_';
+
+  output[1] = ascii64[(count >>  0) & 0x3f];
+  output[2] = ascii64[(count >>  6) & 0x3f];
+  output[3] = ascii64[(count >> 12) & 0x3f];
+  output[4] = ascii64[(count >> 18) & 0x3f];
+
+  output[5] = ascii64[(value >>  0) & 0x3f];
+  output[6] = ascii64[(value >>  6) & 0x3f];
+  output[7] = ascii64[(value >> 12) & 0x3f];
+  output[8] = ascii64[(value >> 18) & 0x3f];
+
+  output[9] = '\0';
+}