Support __float*__ for floating-point numbers in expressions

Add special operators to allow the use of floating-point constants in
contexts other than DW/DD/DQ/DT/DO.

As part of this checkin, make MAX_KEYWORD generated by tokhash.pl,
since it knows what all the keywords are so it can tell which one is
the longest.

1 files changed, 121 insertions, 96 deletions

diff --git a/tokhash.pl b/tokhash.pl
index f45d07d..739172c 100755
--- a/tokhash.pl
+++ b/tokhash.pl
@@ -7,7 +7,7 @@
 
 require 'phash.ph';
 
-my($insns_dat, $regs_dat, $tokens_dat) = @ARGV;
+my($output, $insns_dat, $regs_dat, $tokens_dat) = @ARGV;
 
 %tokens = ();
 @tokendata = ();
@@ -120,102 +120,127 @@ while (defined($line = <TD>)) {
 }
 close(TD);
 
-#
-# Actually generate the hash
-#
-@hashinfo = gen_perfect_hash(\%tokens);
-if (!defined(@hashinfo)) {
-    die "$0: no hash found\n";
-}
+if ($output eq 'h') {
+    #
+    # keywords.h
+    #
 
-# Paranoia...
-verify_hash_table(\%tokens, \@hashinfo);
-
-($n, $sv, $g) = @hashinfo;
-$sv2 = $sv+2;
-
-die if ($n & ($n-1));
-
-print "/*\n";
-print " * This file is generated from insns.dat, regs.dat and token.dat\n";
-print " * by tokhash.pl; do not edit.\n";
-print " */\n";
-print "\n";
-
-print "#include <string.h>\n";
-print "#include \"nasm.h\"\n";
-print "#include \"insns.h\"\n";
-print "\n";
-
-print "#define rot(x,y) (((uint32_t)(x) << (y))+((uint32_t)(x) >> (32-(y))))\n";
-print "\n";
-
-# These somewhat odd sizes and ordering thereof are due to the
-# relative ranges of the types; this makes it fit in 16 bytes on
-# 64-bit machines and 12 bytes on 32-bit machines.
-print "struct tokendata {\n";
-print "    const char *string;\n";
-print "    int16_t tokentype;\n";
-print "    int16_t aux;\n";
-print "    int32_t num;\n";
-print "};\n";
-print "\n";
-
-print "int nasm_token_hash(const char *token, struct tokenval *tv)\n";
-print "{\n";
-
-# Put a large value in unused slots.  This makes it extremely unlikely
-# that any combination that involves unused slot will pass the range test.
-# This speeds up rejection of unrecognized tokens, i.e. identifiers.
-print "#define UNUSED 16383\n";
-
-print "    static const int16_t hash1[$n] = {\n";
-for ($i = 0; $i < $n; $i++) {
-    my $h = ${$g}[$i*2+0];
-    print "        ", defined($h) ? $h : 'UNUSED', ",\n";
-}
-print "    };\n";
+    $max_len = 0;
+    foreach $token (keys(%tokens)) {
+	if (length($token) > $max_len) {
+	    $max_len = length($token);
+	}
+    }
 
-print "    static const int16_t hash2[$n] = {\n";
-for ($i = 0; $i < $n; $i++) {
-    my $h = ${$g}[$i*2+1];
-    print "        ", defined($h) ? $h : 'UNUSED', ",\n";
-}
-print "    };\n";
+    print "/*\n";
+    print " * This file is generated from insns.dat, regs.dat and token.dat\n";
+    print " * by tokhash.pl; do not edit.\n";
+    print " */\n";
+    print "\n";
+
+    print "#ifndef NASM_TOKENS_H\n";
+    print "#define NASM_TOKENS_H\n";
+    print "\n";
+    print "#define MAX_KEYWORD $max_len /* length of longest keyword */\n";
+    print "\n";
+    print "#endif /* NASM_TOKENS_H */\n";
+} elsif ($output eq 'c') {
+    #
+    # tokhash.c
+    #
+
+    @hashinfo = gen_perfect_hash(\%tokens);
+    if (!defined(@hashinfo)) {
+	die "$0: no hash found\n";
+    }
 
-printf "    static const struct tokendata tokendata[%d] = {\n", scalar(@tokendata);
-foreach $d (@tokendata) {
-    print "        { ", $d, " },\n";
+    # Paranoia...
+    verify_hash_table(\%tokens, \@hashinfo);
+    
+    ($n, $sv, $g) = @hashinfo;
+    $sv2 = $sv+2;
+    
+    die if ($n & ($n-1));
+    
+    print "/*\n";
+    print " * This file is generated from insns.dat, regs.dat and token.dat\n";
+    print " * by tokhash.pl; do not edit.\n";
+    print " */\n";
+    print "\n";
+    
+    print "#include <string.h>\n";
+    print "#include \"nasm.h\"\n";
+    print "#include \"insns.h\"\n";
+    print "\n";
+    
+    print "#define rot(x,y) (((uint32_t)(x) << (y))+((uint32_t)(x) >> (32-(y))))\n";
+    print "\n";
+    
+    # These somewhat odd sizes and ordering thereof are due to the
+    # relative ranges of the types; this makes it fit in 16 bytes on
+    # 64-bit machines and 12 bytes on 32-bit machines.
+    print "struct tokendata {\n";
+    print "    const char *string;\n";
+    print "    int16_t tokentype;\n";
+    print "    int16_t aux;\n";
+    print "    int32_t num;\n";
+    print "};\n";
+    print "\n";
+    
+    print "int nasm_token_hash(const char *token, struct tokenval *tv)\n";
+    print "{\n";
+    
+    # Put a large value in unused slots.  This makes it extremely unlikely
+    # that any combination that involves unused slot will pass the range test.
+    # This speeds up rejection of unrecognized tokens, i.e. identifiers.
+    print "#define UNUSED 16383\n";
+    
+    print "    static const int16_t hash1[$n] = {\n";
+    for ($i = 0; $i < $n; $i++) {
+	my $h = ${$g}[$i*2+0];
+	print "        ", defined($h) ? $h : 'UNUSED', ",\n";
+    }
+    print "    };\n";
+    
+    print "    static const int16_t hash2[$n] = {\n";
+    for ($i = 0; $i < $n; $i++) {
+	my $h = ${$g}[$i*2+1];
+	print "        ", defined($h) ? $h : 'UNUSED', ",\n";
+    }
+    print "    };\n";
+    
+    printf "    static const struct tokendata tokendata[%d] = {\n", scalar(@tokendata);
+    foreach $d (@tokendata) {
+	print "        { ", $d, " },\n";
+    }
+    print  "    };\n";
+    
+    print  "    uint32_t k1 = 0, k2 = 0;\n";
+    print  "    uint8_t c;\n";
+    # For correct overflow behavior, "ix" should be unsigned of the same
+    # width as the hash arrays.
+    print  "    uint16_t ix;\n";
+    print  "    const struct tokendata *data;\n";
+    print  "    const char *p = token;\n";
+    print  "\n";
+    
+    print  "    while ((c = *p++) != 0) {\n";
+    printf "        uint32_t kn1 = rot(k1,%2d)^(rot(k2,%2d) + c);\n", ${$sv}[0], ${$sv}[1];
+    printf "        uint32_t kn2 = rot(k2,%2d)^(rot(k1,%2d) + c);\n", ${$sv}[2], ${$sv}[3];
+    print  "        k1 = kn1; k2 = kn2;\n";
+    print  "    }\n";
+    print  "\n";
+    printf "    ix = hash1[k1 & 0x%x] + hash2[k2 & 0x%x];\n", $n-1, $n-1;
+    printf "    if (ix >= %d)\n", scalar(@tokendata);
+    print  "        return tv->t_type = TOKEN_ID;\n";
+    print  "\n";
+    print  "    data = &tokendata[ix];\n";
+    
+    print  "    if (strcmp(data->string, token))\n";
+    print  "        return tv->t_type = TOKEN_ID;\n";
+    print  "\n";
+    print  "    tv->t_integer = data->num;\n";
+    print  "    tv->t_inttwo  = data->aux;\n";
+    print  "    return tv->t_type = data->tokentype;\n";
+    print  "}\n";
 }
-print  "    };\n";
-
-print  "    uint32_t k1 = 0, k2 = 0;\n";
-print  "    uint8_t c;\n";
-# For correct overflow behavior, "ix" should be unsigned of the same
-# width as the hash arrays.
-print  "    uint16_t ix;\n";
-print  "    const struct tokendata *data;\n";
-print  "    const char *p = token;\n";
-print  "\n";
-
-print  "    while ((c = *p++) != 0) {\n";
-printf "        uint32_t kn1 = rot(k1,%2d)^(rot(k2,%2d) + c);\n", ${$sv}[0], ${$sv}[1];
-printf "        uint32_t kn2 = rot(k2,%2d)^(rot(k1,%2d) + c);\n", ${$sv}[2], ${$sv}[3];
-print  "        k1 = kn1; k2 = kn2;\n";
-print  "    }\n";
-print  "\n";
-printf "    ix = hash1[k1 & 0x%x] + hash2[k2 & 0x%x];\n", $n-1, $n-1;
-printf "    if (ix >= %d)\n", scalar(@tokendata);
-print  "        return tv->t_type = TOKEN_ID;\n";
-print  "\n";
-print  "    data = &tokendata[ix];\n";
-
-# print  "    fprintf(stderr, \"Looked for: %s found: %s\\n\", token, data->string);\n\n";
-
-print  "    if (strcmp(data->string, token))\n";
-print  "        return tv->t_type = TOKEN_ID;\n";
-print  "\n";
-print  "    tv->t_integer = data->num;\n";
-print  "    tv->t_inttwo  = data->aux;\n";
-print  "    return tv->t_type = data->tokentype;\n";
-print  "}\n";