Initialize unused upper bits of Vector3 arguments and return registers for

Linux

JIT requires the unused upper bits to be cleared but native compiler
doesn't clear the upper bits. This changes clear those bits for Reverse
PInvoke arguments and PInvoke return.

1 files changed, 24 insertions, 3 deletions

diff --git a/src/jit/codegencommon.cpp b/src/jit/codegencommon.cpp
index 0ce079c6d3..42a404afbe 100644..100755
--- a/src/jit/codegencommon.cpp
+++ b/src/jit/codegencommon.cpp
@@ -3931,9 +3931,20 @@ void            CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg,
                 // RyuJit backend is making another implicit assumption that Vector3 type args when passed in
                 // registers or on stack, the upper most 4-bytes will be zero.  
                 //
-                // TODO-64bit: assumptions 1 and 2 hold within RyuJIT generated code. It is not clear whether
-                // these assumptions hold when a Vector3 type arg is passed by native code. Example: PInvoke
-                // returning Vector3 type value or RPInvoke passing Vector3 type args.
+                // For P/Invoke return and Reverse P/Invoke argument passing, native compiler doesn't guarantee
+                // that upper 4-bytes of a Vector3 type struct is zero initialized and hence assumption 2 is 
+                // invalid.
+                //
+                // RyuJIT x64 Windows: arguments are treated as passed by ref and hence read/written just 12
+                // bytes. In case of Vector3 returns, Caller allocates a zero initialized Vector3 local and
+                // passes it retBuf arg and Callee method writes only 12 bytes to retBuf. For this reason,
+                // there is no need to clear upper 4-bytes of Vector3 type args.
+                //
+                // RyuJIT x64 Unix: arguments are treated as passed by value and read/writen as if TYP_SIMD16.
+                // Vector3 return values are returned two return registers and Caller assembles them into a 
+                // single xmm reg. Hence RyuJIT explicitly generates code to clears upper 4-bytes of Vector3 
+                // type args in prolog and Vector3 type return value of a call
+
                 if (varDsc->lvType == TYP_SIMD12)
                 {
                     regType = TYP_DOUBLE;
@@ -8519,6 +8530,16 @@ void                CodeGen::genFnProlog()
         genPrologPadForReJit();
         getEmitter()->emitMarkPrologEnd();
     }
+    
+#if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING) && defined(FEATURE_SIMD)
+    // The unused bits of Vector3 arguments must be cleared
+    // since native compiler doesn't initize the upper bits to zeros.
+    //
+    // TODO-Cleanup: This logic can be implemented in
+    // genFnPrologCalleeRegArgs() for argument registers and 
+    // genEnregisterIncomingStackArgs() for stack arguments.
+    genClearStackVec3ArgUpperBits();
+#endif //FEATURE_UNIX_AMD64_STRUCT_PASSING && FEATURE_SIMD
 
     /*-----------------------------------------------------------------------------
      * Take care of register arguments first