[x86/Linux] Initial fix of arguments passing in FunctionPtrTest (WIP) (#9855)

* [x86/Linux] Initial fix of incorrect arguments passing in FunctionPtrTest

4 files changed, 111 insertions, 22 deletions

diff --git a/src/vm/dllimportcallback.cpp b/src/vm/dllimportcallback.cpp
index 92604eccd7..c53b080968 100644
--- a/src/vm/dllimportcallback.cpp
+++ b/src/vm/dllimportcallback.cpp
@@ -1411,13 +1411,57 @@ VOID UMThunkMarshInfo::RunTimeInit()
 
             pStubMD = GetILStubMethodDesc(pMD, &sigInfo, dwStubFlags);
             pFinalILStub = JitILStub(pStubMD);
+
         }
     }
-
     //
     // m_cbActualArgSize gets the number of arg bytes for the NATIVE signature
     //
-    m_cbActualArgSize = (pStubMD != NULL) ? pStubMD->AsDynamicMethodDesc()->GetNativeStackArgSize() : pMD->SizeOfArgStack();
+    m_cbActualArgSize =
+        (pStubMD != NULL) ? pStubMD->AsDynamicMethodDesc()->GetNativeStackArgSize() : pMD->SizeOfArgStack();
+
+#if defined(_TARGET_X86_)
+    MetaSig sig(pMD);
+    ArgIterator argit(&sig);
+    int numRegistersUsed = 0;
+    m_ecxArgOffset = -1;
+    m_edxArgOffset = -1;
+
+    int offs = 0;
+    for (UINT i = 0 ; i < sig.NumFixedArgs(); i++)
+    {
+        TypeHandle thValueType;
+        CorElementType type = sig.NextArgNormalized(&thValueType);
+        int cbSize = sig.GetElemSize(type, thValueType);
+        if (ArgIterator::IsArgumentInRegister(&numRegistersUsed, type))
+        {
+            if (numRegistersUsed == 1)
+                m_ecxArgOffset = offs;
+            else if (numRegistersUsed == 2)
+                m_edxArgOffset = offs;
+            offs += STACK_ELEM_SIZE;
+        }
+        else
+        {
+            offs += StackElemSize(cbSize);
+        }
+    }
+    PInvokeStaticSigInfo sigInfo;
+    if (pMD != NULL)
+        new (&sigInfo) PInvokeStaticSigInfo(pMD);
+    else
+        new (&sigInfo) PInvokeStaticSigInfo(GetSignature(), GetModule());
+    if (sigInfo.GetCallConv() == pmCallConvCdecl)
+    {
+        // caller pop
+        m_cbRetPop = 0;
+    }
+    else
+    {
+        // callee pop
+        m_cbRetPop = static_cast<UINT16>(m_cbActualArgSize);
+    }
+#endif // _TARGET_X86_
 
 #endif // _TARGET_X86_ && !FEATURE_STUBS_AS_IL
 
diff --git a/src/vm/dllimportcallback.h b/src/vm/dllimportcallback.h
index c2ed6d0039..c6f5788739 100644
--- a/src/vm/dllimportcallback.h
+++ b/src/vm/dllimportcallback.h
@@ -210,11 +210,17 @@ private:
                                             // On x86, NULL for no-marshal signatures
                                             // On non-x86, the managed entrypoint for no-delegate no-marshal signatures
     UINT32            m_cbActualArgSize;    // caches m_pSig.SizeOfFrameArgumentArray()
-#if defined(_TARGET_X86_) && !defined(FEATURE_STUBS_AS_IL)
-    Stub*             m_pExecStub;          // UMEntryThunk jumps directly here
+#if defined(_TARGET_X86_)
     UINT16            m_cbRetPop;           // stack bytes popped by callee (for UpdateRegDisplay)
+#if defined(FEATURE_STUBS_AS_IL)
+    UINT32            m_ecxArgOffset;
+    UINT32            m_edxArgOffset;
+#else
+    Stub*             m_pExecStub;          // UMEntryThunk jumps directly here
     UINT16            m_callConv;           // unmanaged calling convention and flags (CorPinvokeMap)
-#endif
+#endif // FEATURE_STUBS_AS_IL
+#endif // _TARGET_X86_
+
     MethodDesc *      m_pMD;                // maybe null
     Module *          m_pModule;
     Signature         m_sig;
diff --git a/src/vm/i386/asmconstants.h b/src/vm/i386/asmconstants.h
index 4ad08ba133..c5e9ae52fe 100644
--- a/src/vm/i386/asmconstants.h
+++ b/src/vm/i386/asmconstants.h
@@ -438,6 +438,17 @@ ASMCONSTANTS_C_ASSERT(UMThunkMarshInfo__m_pILStub == offsetof(UMThunkMarshInfo,
 #define               UMThunkMarshInfo__m_cbActualArgSize   0x04
 ASMCONSTANTS_C_ASSERT(UMThunkMarshInfo__m_cbActualArgSize == offsetof(UMThunkMarshInfo, m_cbActualArgSize))
 
+#ifdef FEATURE_STUBS_AS_IL
+#define               UMThunkMarshInfo__m_cbRetPop   0x08
+ASMCONSTANTS_C_ASSERT(UMThunkMarshInfo__m_cbRetPop == offsetof(UMThunkMarshInfo, m_cbRetPop))
+
+#define               UMThunkMarshInfo__m_ecxArgOffset   0xc
+ASMCONSTANTS_C_ASSERT(UMThunkMarshInfo__m_ecxArgOffset == offsetof(UMThunkMarshInfo, m_ecxArgOffset))
+
+#define               UMThunkMarshInfo__m_edxArgOffset  0x10
+ASMCONSTANTS_C_ASSERT(UMThunkMarshInfo__m_edxArgOffset == offsetof(UMThunkMarshInfo, m_edxArgOffset))
+#endif //FEATURE_STUBS_AS_IL
+
 #ifndef CROSSGEN_COMPILE
 #define               Thread__m_pDomain                     0x14
 ASMCONSTANTS_C_ASSERT(Thread__m_pDomain == offsetof(Thread, m_pDomain));
diff --git a/src/vm/i386/umthunkstub.S b/src/vm/i386/umthunkstub.S
index 5a557d4b32..6cefda9ec4 100644
--- a/src/vm/i386/umthunkstub.S
+++ b/src/vm/i386/umthunkstub.S
@@ -103,12 +103,20 @@ LOCAL_LABEL(PostCall):
     mov     dword ptr [ebx + Thread_m_fPreemptiveGCDisabled], 0
 
     lea     esp, [ebp - UMThunkStub_SAVEDREG]  // deallocate arguments
+
+    mov     ecx, dword ptr [ebp - UMThunkStub_UMENTRYTHUNK_OFFSET]
+    mov     edx, dword ptr [ecx + UMEntryThunk__m_pUMThunkMarshInfo]
+    mov     edx, dword ptr [edx + UMThunkMarshInfo__m_cbRetPop]
+
     EPILOG_BEG
     EPILOG_POP edi
     EPILOG_POP esi
     EPILOG_POP ebx
     EPILOG_END
-    ret
+
+    pop     ecx  // pop return address
+    add     esp, edx // adjust ESP
+    jmp     ecx // return to caller
 
 LOCAL_LABEL(DoThreadSetup):
 
@@ -139,30 +147,50 @@ LOCAL_LABEL(UMThunkStub_CopyStackArgs):
     // esi = src
     // edi = dest
     // ebx = scratch
-    lea     esi, [ebp + 0x08]
-
-    // first [esi] goes to ecx, in LTR
-    add     eax, -4
-    mov     ecx, dword ptr [esi]
-    jz      LOCAL_LABEL(UMThunkStub_ArgumentsSetup)
-
-    // second [esi+04] goes to edx
-    add     eax, -4
-    mov     edx, dword ptr [esi + 0x04]
-    jz      LOCAL_LABEL(UMThunkStub_ArgumentsSetup)
+    lea     esi, [ebp + 0x8]
 
     sub     esp, eax
     and     esp, -16    // align with 16 byte
     lea     edi, [esp]
 
-LOCAL_LABEL(CopyLoop):
+    // First, we copy arguments to ecx and edx registers (if needed).
+    mov     edx, dword ptr [ebp - UMThunkStub_UMENTRYTHUNK_OFFSET]
+    mov     edx, dword ptr [edx + UMEntryThunk__m_pUMThunkMarshInfo]
+    mov     ebx, dword ptr [edx + UMThunkMarshInfo__m_ecxArgOffset]
+    cmp     ebx, -1
+    je      LOCAL_LABEL(InitCopyStack)
+    mov     ecx, dword ptr [esi + ebx]
+    add     eax, -4
+    jz      LOCAL_LABEL(UMThunkStub_ArgumentsSetup)
 
-    // copy rest of the arguments to [esp+08+n], in RTL
+    mov     ebx, dword ptr [edx + UMThunkMarshInfo__m_edxArgOffset]
+    cmp     ebx, -1
+    je      LOCAL_LABEL(InitCopyStack)
+    mov     edx, dword ptr [esi + ebx]
     add     eax, -4
-    mov     ebx, dword ptr [esi + 0x08 + eax]
-    mov     dword ptr [edi + eax], ebx
-    jnz     LOCAL_LABEL(CopyLoop)
+    jz      LOCAL_LABEL(UMThunkStub_ArgumentsSetup)
 
+LOCAL_LABEL(InitCopyStack):
+    push    ecx
+    push    edx
+    mov     edx, dword ptr [ebp - UMThunkStub_UMENTRYTHUNK_OFFSET]
+    mov     edx, dword ptr [edx + UMEntryThunk__m_pUMThunkMarshInfo]
+    mov     ebx, [edx + UMThunkMarshInfo__m_cbActualArgSize]
+    add     ebx, -4
+LOCAL_LABEL(CopyStack):
+    cmp     ebx, dword ptr [edx + UMThunkMarshInfo__m_ecxArgOffset]
+    je      LOCAL_LABEL(IncreaseOffset)
+    cmp     ebx, dword ptr [edx + UMThunkMarshInfo__m_edxArgOffset]
+    je      LOCAL_LABEL(IncreaseOffset)
+    add     eax, -4
+    mov     ecx, dword ptr [esi + ebx]
+    mov     dword ptr [edi + eax], ecx
+LOCAL_LABEL(IncreaseOffset):
+    add     ebx, -4
+    jc      LOCAL_LABEL(CopyStack)
+
+    pop     edx
+    pop     ecx
     jmp     LOCAL_LABEL(UMThunkStub_ArgumentsSetup)
 
 #if _DEBUG