BR3058845: mostly fix bogus warning with implicit operand size override

The implicit operand size override code didn't set the operand size
prefix, which confused the size calculation code for the range check.

The BITS 64 operand size calculation is still off, but "fixing" it by
making it 32-bit unless REX.W is set breaks PUSH and maybe others.

2 files changed, 30 insertions, 6 deletions

diff --git a/assemble.c b/assemble.c
index 99b833c..1ee7f7f 100644
--- a/assemble.c
+++ b/assemble.c
@@ -1636,22 +1636,32 @@ static void gencode(int32_t segment, int64_t offset, int bits,
             break;
 
         case 0320:
-            if (bits != 16) {
+        {
+            enum prefixes pfx = ins->prefixes[PPS_OSIZE];
+            if (pfx != P_O16 && pfx != P_none)
+                nasm_error(ERR_WARNING, "Invalid operand size prefix");
+            if (pfx != P_O16 && bits != 16) {
+                ins->prefixes[PPS_OSIZE] = P_O16;
                 *bytes = 0x66;
                 out(offset, segment, bytes, OUT_RAWDATA, 1, NO_SEG, NO_SEG);
                 offset += 1;
-            } else
-                offset += 0;
+            }
             break;
+        }
 
         case 0321:
-            if (bits == 16) {
+        {
+            enum prefixes pfx = ins->prefixes[PPS_OSIZE];
+            if (pfx != P_O32 && pfx != P_none)
+                nasm_error(ERR_WARNING, "Invalid operand size prefix");
+            if (pfx != P_O32 && bits == 16) {
+                ins->prefixes[PPS_OSIZE] = P_O32;
                 *bytes = 0x66;
                 out(offset, segment, bytes, OUT_RAWDATA, 1, NO_SEG, NO_SEG);
                 offset += 1;
-            } else
-                offset += 0;
+            }
             break;
+        }
 
         case 0322:
         case 0323:
diff --git a/test/br3058845.asm b/test/br3058845.asm
new file mode 100644
index 0000000..b0de560
--- /dev/null
+++ b/test/br3058845.asm
@@ -0,0 +1,14 @@
+;Testname=unoptimized; Arguments=-O0 -fbin -obr3058845.bin; Files=stdout stderr br3058845.bin
+;Testname=optimized;   Arguments=-Ox -fbin -obr3058845.bin; Files=stdout stderr br3058845.bin
+
+BITS 16
+cmp ax, 0xFFFF
+cmp eax, 0xFFFF_FFFF
+
+BITS 32
+cmp ax, 0xFFFF
+cmp eax, 0xFFFF_FFFF
+
+BITS 64
+cmp ax, 0xFFFF
+cmp eax, 0xFFFF_FFFF    ; shouldn't warn, but does currently