Add {vex3} and {vex2} prefixes by analogy with {evex}

Allow specifying {vex3} or {vex2} (the latter is currently always
redundant, unless we end up with instructions at some point can be
specified with legacy prefixes or VEX) to select a specific encoding
of VEX-encoded instructions.

Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>

8 files changed, 62 insertions, 13 deletions

diff --git a/assemble.c b/assemble.c
index 759e4b5..bf1490d 100644
--- a/assemble.c
+++ b/assemble.c
@@ -643,8 +643,8 @@ int64_t assemble(int32_t segment, int64_t offset, int bits, iflag_t cp,
                         c = 0x66;
                         break;
                     case P_EVEX:
-                        /* EVEX */
-                        break;
+                    case P_VEX3:
+                    case P_VEX2:
                     case P_none:
                         break;
                     default:
@@ -810,6 +810,8 @@ int64_t insn_size(int32_t segment, int64_t offset, int bits, iflag_t cp,
             case P_A64:
             case P_O64:
             case P_EVEX:
+            case P_VEX3:
+            case P_VEX2:
             case P_none:
                 break;
             default:
@@ -1270,6 +1272,20 @@ static int64_t calcsize(int32_t segment, int64_t offset, int bits,
         ins->rex &= ~REX_P;        /* Don't force REX prefix due to high reg */
     }
 
+    switch (ins->prefixes[PPS_VEX]) {
+    case P_EVEX:
+        if (!(ins->rex & REX_EV))
+            return -1;
+        break;
+    case P_VEX3:
+    case P_VEX2:
+        if (!(ins->rex & REX_V))
+            return -1;
+        break;
+    default:
+        break;
+    }
+
     if (ins->rex & (REX_V | REX_EV)) {
         int bad32 = REX_R|REX_W|REX_X|REX_B;
 
@@ -1301,7 +1317,8 @@ static int64_t calcsize(int32_t segment, int64_t offset, int bits,
         }
         if (ins->rex & REX_EV)
             length += 4;
-        else if (ins->vex_cm != 1 || (ins->rex & (REX_W|REX_X|REX_B)))
+        else if (ins->vex_cm != 1 || (ins->rex & (REX_W|REX_X|REX_B)) ||
+                 ins->prefixes[PPS_VEX] == P_VEX3)
             length += 3;
         else
             length += 2;
@@ -1600,7 +1617,8 @@ static void gencode(int32_t segment, int64_t offset, int bits,
         case4(0260):
         case 0270:
             codes += 2;
-            if (ins->vex_cm != 1 || (ins->rex & (REX_W|REX_X|REX_B))) {
+            if (ins->vex_cm != 1 || (ins->rex & (REX_W|REX_X|REX_B)) ||
+                ins->prefixes[PPS_VEX] == P_VEX3) {
                 bytes[0] = (ins->vex_cm >> 6) ? 0x8f : 0xc4;
                 bytes[1] = (ins->vex_cm & 31) | ((~ins->rex & 7) << 5);
                 bytes[2] = ((ins->rex & REX_W) << (7-3)) |
@@ -2098,8 +2116,18 @@ static enum match_result matches(const struct itemplate *itemp,
     /*
      * {evex} available?
      */
-	if (instruction->prefixes[PPS_EVEX] && !itemp_has(itemp, IF_EVEX)) {
-        return MERR_ENCMISMATCH;
+    switch (instruction->prefixes[PPS_VEX]) {
+    case P_EVEX:
+        if (!itemp_has(itemp, IF_EVEX))
+            return MERR_ENCMISMATCH;
+        break;
+    case P_VEX3:
+    case P_VEX2:
+        if (!itemp_has(itemp, IF_VEX))
+            return MERR_ENCMISMATCH;
+        break;
+    default:
+        break;
     }
 
     /*
diff --git a/doc/changes.src b/doc/changes.src
index 91e3078..fcf6f2a 100644
--- a/doc/changes.src
+++ b/doc/changes.src
@@ -29,8 +29,9 @@ each unique bitvector occupies only one hash index.
 
 \b Support \c{BND} prefix for branch instructions
 
-\b Add \c{{evex}} instruction prefix to have nasm encode the corresponding
-instruction line with EVEX
+\b Add \c{{evex}}, \c{{vex3}} and \c{{vex2}} instruction prefixes to
+have NASM encode the corresponding instruction, if possible, with an EVEX,
+3-byte VEX, or 2-byte VEX prefix, respectively.
 
 \b Ndisasm supports AVX-512/MPX/SHA instructions
 
diff --git a/insns-iflags.pl b/insns-iflags.pl
index be36193..97c0863 100644
--- a/insns-iflags.pl
+++ b/insns-iflags.pl
@@ -120,7 +120,6 @@ my %insns_flag_bit = (
     "TBM"               => [ 60, ""],
     "RTM"               => [ 61, ""],
     "INVPCID"           => [ 62, ""],
-    "EVEX"              => [ 63, ""],
 
     #
     # dword bound, index 2 - instruction filtering flags
@@ -132,6 +131,8 @@ my %insns_flag_bit = (
     "MPX"               => [ 68	,"MPX"],
     "SHA"               => [ 69	,"SHA"],
     "PREFETCHWT1"       => [ 70	,"PREFETCHWT1"],
+    "VEX"               => [ 94, "VEX or XOP encoded instruction"],
+    "EVEX"              => [ 95, "EVEX encoded instruction"],
 
     #
     # dword bound, index 3 - cpu type flags
diff --git a/insns.pl b/insns.pl
index 67333ad..451c4f2 100755
--- a/insns.pl
+++ b/insns.pl
@@ -495,7 +495,11 @@ sub format_insn($$$$$) {
     $nd = 1 if $flags =~ /(^|\,)ND($|\,)/;
     $flags =~ s/(^|\,)ND($|\,)/\1/g;
     $flags =~ s/(^|\,)X64($|\,)/\1LONG,X86_64\2/g;
-    $flags .= ",EVEX" if ($codes =~ /evex\./);
+    if ($codes =~ /evex\./) {
+	$flags .= ",EVEX";
+    } elsif ($codes =~ /(vex|xop)\./) {
+	$flags .= ",VEX";
+    }
     $rawflags = $flags;
     $flagsindex = insns_flag_index(split(',',$flags));
 
diff --git a/nasm.h b/nasm.h
index 87e82d2..f499c49 100644
--- a/nasm.h
+++ b/nasm.h
@@ -565,6 +565,8 @@ enum prefixes { /* instruction prefixes */
     P_XRELEASE,
     P_BND,
     P_EVEX,
+    P_VEX3,
+    P_VEX2,
     PREFIX_ENUM_LIMIT
 };
 
@@ -647,7 +649,7 @@ enum prefix_pos {
     PPS_SEG,    /* Segment override prefix */
     PPS_OSIZE,  /* Operand size prefix */
     PPS_ASIZE,  /* Address size prefix */
-    PPS_EVEX,   /* EVEX prefix */
+    PPS_VEX,    /* VEX type */
     MAXPREFIX   /* Total number of prefix slots */
 };
 
diff --git a/parser.c b/parser.c
index f73c7b5..4f0898c 100644
--- a/parser.c
+++ b/parser.c
@@ -102,7 +102,9 @@ static int prefix_slot(int prefix)
     case P_ASP:
         return PPS_ASIZE;
     case P_EVEX:
-        return PPS_EVEX;
+    case P_VEX3:
+    case P_VEX2:
+        return PPS_VEX;
     default:
         nasm_error(ERR_PANIC, "Invalid value %d passed to prefix_slot()", prefix);
         return -1;
diff --git a/test/vex.asm b/test/vex.asm
new file mode 100644
index 0000000..6772c7c
--- /dev/null
+++ b/test/vex.asm
@@ -0,0 +1,9 @@
+	bits 64
+	vcomisd xmm0,xmm31
+	vcomisd xmm0,xmm1
+	{vex2} vcomisd xmm0,xmm1
+	{vex3} vcomisd xmm0,xmm1
+	{evex} vcomisd xmm0,xmm1
+%ifdef ERROR
+	{vex3} add eax,edx
+%endif
diff --git a/tokens.dat b/tokens.dat
index 284cf30..09d4b9f 100644
--- a/tokens.dat
+++ b/tokens.dat
@@ -1,6 +1,6 @@
 ## --------------------------------------------------------------------------
 ##   
-##   Copyright 1996-2012 The NASM Authors - All Rights Reserved
+##   Copyright 1996-2013 The NASM Authors - All Rights Reserved
 ##   See the file AUTHORS included with the NASM distribution for
 ##   the specific copyright holders.
 ##
@@ -125,3 +125,5 @@ z
 
 % TOKEN_PREFIX, 0, TFLAG_BRC, P_*
 evex
+vex3
+vex2