Fix x86 encoder to use 64-bit type to accumulate opcode/prefix bits

The encoder was using size_t, a 32-bit type on x86, to accumulate opcode
and prefix bits to emit. AVX support uses 3 bytes for prefixes that are
higher than the 32-bit type can handle. So, change all code byte related types
from size_t to a new code_t, defined as "unsigned __int64" on RyuJIT x86
(there is precedence for this type on the ARM architectures).

Fixes #8331

1 files changed, 12 insertions, 0 deletions

diff --git a/src/jit/emit.h b/src/jit/emit.h
index b3716b81bf..5b1a395379 100644
--- a/src/jit/emit.h
+++ b/src/jit/emit.h
@@ -1664,6 +1664,18 @@ private:
     unsigned char emitOutputLong(BYTE* dst, ssize_t val);
     unsigned char emitOutputSizeT(BYTE* dst, ssize_t val);
 
+#if !defined(LEGACY_BACKEND) && defined(_TARGET_X86_)
+    unsigned char emitOutputByte(BYTE* dst, size_t val);
+    unsigned char emitOutputWord(BYTE* dst, size_t val);
+    unsigned char emitOutputLong(BYTE* dst, size_t val);
+    unsigned char emitOutputSizeT(BYTE* dst, size_t val);
+
+    unsigned char emitOutputByte(BYTE* dst, unsigned __int64 val);
+    unsigned char emitOutputWord(BYTE* dst, unsigned __int64 val);
+    unsigned char emitOutputLong(BYTE* dst, unsigned __int64 val);
+    unsigned char emitOutputSizeT(BYTE* dst, unsigned __int64 val);
+#endif // !defined(LEGACY_BACKEND) && defined(_TARGET_X86_)
+
     size_t emitIssue1Instr(insGroup* ig, instrDesc* id, BYTE** dp);
     size_t emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp);