[x86/Linux] Fix IL_STUB_PInvoke with RetBuf (#10144)

* [x86/Linux] Fix IL_STUB_PInvoke with RetBuf

Fix calling convention and IL_STUB_PInvoke for native functions
which was problematic for native functions that has
a RetBuf argument(struct size <= 8) on x86/Linux.

Fix #10027

4 files changed, 27 insertions, 1 deletions

diff --git a/src/vm/callingconvention.h b/src/vm/callingconvention.h
index c9a27c2371..cde2ba465a 100644
--- a/src/vm/callingconvention.h
+++ b/src/vm/callingconvention.h
@@ -1701,6 +1701,17 @@ inline BOOL HasRetBuffArg(MetaSig * pSig)
     return argit.HasRetBuffArg();
 }
 
+#ifdef UNIX_X86_ABI
+// For UNIX_X86_ABI and unmanaged function, we always need RetBuf if the return type is VALUETYPE
+inline BOOL HasRetBuffArgUnmanagedFixup(MetaSig * pSig)
+{
+    WRAPPER_NO_CONTRACT;
+    // We cannot just pSig->GetReturnType() here since it will return ELEMENT_TYPE_VALUETYPE for enums
+    CorElementType type = pSig->GetRetTypeHandleThrowing().GetVerifierCorElementType();
+    return type == ELEMENT_TYPE_VALUETYPE;
+}
+#endif
+
 inline BOOL IsRetBuffPassedAsFirstArg()
 {
     WRAPPER_NO_CONTRACT;
diff --git a/src/vm/dllimport.cpp b/src/vm/dllimport.cpp
index ba56831bd6..c0a73e8b80 100644
--- a/src/vm/dllimport.cpp
+++ b/src/vm/dllimport.cpp
@@ -3684,8 +3684,15 @@ static void CreateNDirectStubWorker(StubState*         pss,
         // return buffer in correct register.
         // The return structure secret arg comes first, however byvalue return is processed at
         // the end because it could be the HRESULT-swapped argument which always comes last.
+
+#ifdef UNIX_X86_ABI
+        // For functions with value type class, managed and unmanaged calling convention differ
+        fMarshalReturnValueFirst = HasRetBuffArgUnmanagedFixup(&msig);
+#else // UNIX_X86_ABI
         fMarshalReturnValueFirst = HasRetBuffArg(&msig);
-#endif
+#endif // UNIX_X86_ABI
+
+#endif // defined(_TARGET_X86_) || defined(_TARGET_ARM_)
 
     }
     
diff --git a/src/vm/i386/cgencpu.h b/src/vm/i386/cgencpu.h
index ac239533f7..ff76d992fc 100644
--- a/src/vm/i386/cgencpu.h
+++ b/src/vm/i386/cgencpu.h
@@ -483,10 +483,15 @@ inline BOOL IsUnmanagedValueTypeReturnedByRef(UINT sizeofvaluetype)
 {
     LIMITED_METHOD_CONTRACT;
 
+#ifndef UNIX_X86_ABI
     // odd-sized small structures are not 
     //  enregistered e.g. struct { char a,b,c; }
     return (sizeofvaluetype > 8) ||
         (sizeofvaluetype & (sizeofvaluetype - 1)); // check that the size is power of two
+#else
+    // For UNIX_X86_ABI, we always return the value type by reference regardless of its size
+    return true;
+#endif
 }
 
 #include <pshpack1.h>
diff --git a/src/vm/ilmarshalers.h b/src/vm/ilmarshalers.h
index 7c7f9a6553..5ac5e41242 100644
--- a/src/vm/ilmarshalers.h
+++ b/src/vm/ilmarshalers.h
@@ -607,12 +607,15 @@ public:
 
             // for X86 and AMD64-Windows we bash the return type from struct to U1, U2, U4 or U8
             // and use byrefNativeReturn for all other structs.
+            // for UNIX_X86_ABI, we always need a return buffer argument for any size of structs.
             switch (nativeSize)
             {
+#ifndef UNIX_X86_ABI
                 case 1: typ = ELEMENT_TYPE_U1; break;
                 case 2: typ = ELEMENT_TYPE_U2; break;
                 case 4: typ = ELEMENT_TYPE_U4; break;
                 case 8: typ = ELEMENT_TYPE_U8; break;
+#endif
                 default: byrefNativeReturn = true; break;
             }
 #endif