[x86/Linux] Implement libunwind-based unwindLazyStack (#8686)

* [x86/Linux] Implement libunwind-based unwindLazyStack

This commit implements libunwind-based unwindLazyStack for x86/Linux
in order to fix #8625.

4 files changed, 116 insertions, 9 deletions

diff --git a/compileoptions.cmake b/compileoptions.cmake
index 7baa86baf1..75d51fd5bb 100644
--- a/compileoptions.cmake
+++ b/compileoptions.cmake
@@ -15,11 +15,7 @@ if (CLR_CMAKE_PLATFORM_UNIX)
     # We cannot enable "stack-protector-strong" on OS X due to a bug in clang compiler (current version 7.0.2)
     add_compile_options(-fstack-protector)
   else()
-    if(NOT CLR_CMAKE_PLATFORM_ARCH_I386)
-      # x86 unwinder cannot handle stack protection code, yet
-      # see https://github.com/dotnet/coreclr/issues/8625 for details
-      add_compile_options(-fstack-protector-strong)
-    endif(NOT CLR_CMAKE_PLATFORM_ARCH_I386)
+    add_compile_options(-fstack-protector-strong)
   endif(CLR_CMAKE_PLATFORM_DARWIN)
 
   add_definitions(-DDISABLE_CONTRACTS)
diff --git a/src/debug/daccess/dacfn.cpp b/src/debug/daccess/dacfn.cpp
index 88d45993b3..d8bae7746f 100644
--- a/src/debug/daccess/dacfn.cpp
+++ b/src/debug/daccess/dacfn.cpp
@@ -217,7 +217,7 @@ DacWriteAll(TADDR addr, PVOID buffer, ULONG32 size, bool throwEx)
     return S_OK;
 }
 
-#if defined(WIN64EXCEPTIONS) && defined(FEATURE_PAL)
+#ifdef FEATURE_PAL
 HRESULT 
 DacVirtualUnwind(DWORD threadId, PCONTEXT context, PT_KNONVOLATILE_CONTEXT_POINTERS contextPointers)
 {
@@ -242,7 +242,7 @@ DacVirtualUnwind(DWORD threadId, PCONTEXT context, PT_KNONVOLATILE_CONTEXT_POINT
 
     return hr;
 }
-#endif // defined(WIN64EXCEPTIONS) && defined(FEATURE_PAL)
+#endif // FEATURE_PAL
 
 // DacAllocVirtual - Allocate memory from the target process
 // Note: this is only available to clients supporting the legacy
diff --git a/src/inc/daccess.h b/src/inc/daccess.h
index b4c5044bd9..6d9fb3265f 100644
--- a/src/inc/daccess.h
+++ b/src/inc/daccess.h
@@ -774,18 +774,18 @@ interface IMDInternalImport* DacGetMDImport(const ReflectionModule* reflectionMo
 
 int DacGetIlMethodSize(TADDR methAddr);
 struct COR_ILMETHOD* DacGetIlMethod(TADDR methAddr);
-#if defined(WIN64EXCEPTIONS)
+#ifdef WIN64EXCEPTIONS
 struct _UNWIND_INFO * DacGetUnwindInfo(TADDR taUnwindInfo);
 
 // virtually unwind a CONTEXT out-of-process
 struct _KNONVOLATILE_CONTEXT_POINTERS;
 BOOL DacUnwindStackFrame(T_CONTEXT * pContext, T_KNONVOLATILE_CONTEXT_POINTERS* pContextPointers);
+#endif // WIN64EXCEPTIONS
 
 #if defined(FEATURE_PAL)
 // call back through data target to unwind out-of-process
 HRESULT DacVirtualUnwind(ULONG32 threadId, PCONTEXT context, PT_KNONVOLATILE_CONTEXT_POINTERS contextPointers);
 #endif // FEATURE_PAL
-#endif // _WIN64
 
 #ifdef FEATURE_MINIMETADATA_IN_TRIAGEDUMPS
 class SString;
diff --git a/src/vm/i386/gmsx86.cpp b/src/vm/i386/gmsx86.cpp
index ae4b9dda98..34d65856fe 100644
--- a/src/vm/i386/gmsx86.cpp
+++ b/src/vm/i386/gmsx86.cpp
@@ -9,6 +9,11 @@
 #include "common.h"
 #include "gmscpu.h"
 
+#ifdef FEATURE_PAL
+#define USE_EXTERNAL_UNWINDER
+#endif
+
+#ifndef USE_EXTERNAL_UNWINDER
 /***************************************************************/
 /* setMachState figures out what the state of the CPU will be
    when the function that calls 'setMachState' returns.  It stores
@@ -1264,3 +1269,109 @@ done:
 #ifdef _PREFAST_
 #pragma warning(pop)
 #endif
+#else  // !USE_EXTERNAL_UNWINDER
+
+void LazyMachState::unwindLazyState(LazyMachState* baseState,
+                                    MachState* lazyState,
+                                    DWORD threadId,
+                                    int funCallDepth /* = 1 */,
+                                    HostCallPreference hostCallPreference /* = (HostCallPreference)(-1) */)
+{
+    CONTRACTL {
+        NOTHROW;
+        GC_NOTRIGGER;
+        SO_TOLERANT;
+        SUPPORTS_DAC;
+    } CONTRACTL_END;
+
+    CONTEXT                         ctx;
+    KNONVOLATILE_CONTEXT_POINTERS   nonVolRegPtrs;
+
+    ctx.Eip = baseState->captureEip;
+    ctx.Esp = baseState->captureEsp;
+    ctx.Ebp = baseState->captureEbp;
+
+    ctx.Edi = lazyState->_edi = baseState->_edi;
+    ctx.Esi = lazyState->_esi = baseState->_esi;
+    ctx.Ebx = lazyState->_ebx = baseState->_ebx;
+
+    nonVolRegPtrs.Edi = &(lazyState->_edi);
+    nonVolRegPtrs.Esi = &(lazyState->_esi);
+    nonVolRegPtrs.Ebx = &(lazyState->_ebx);
+    nonVolRegPtrs.Ebp = &(lazyState->_ebp);
+
+    PCODE pvControlPc;
+
+    LOG((LF_GCROOTS, LL_INFO100000, "STACKWALK    LazyMachState::unwindLazyState(ip:%p,bp:%p,sp:%p)\n", baseState->captureEip, baseState->captureEbp, baseState->captureEsp));
+
+    do
+    {
+#ifdef DACCESS_COMPILE
+        HRESULT hr = DacVirtualUnwind(threadId, &ctx, &nonVolRegPtrs);
+        if (FAILED(hr))
+        {
+            DacError(hr);
+        }
+#else
+        BOOL success = PAL_VirtualUnwind(&ctx, &nonVolRegPtrs);
+        if (!success)
+        {
+            _ASSERTE(!"unwindLazyState: Unwinding failed");
+            EEPOLICY_HANDLE_FATAL_ERROR(COR_E_EXECUTIONENGINE);
+        }
+#endif // DACCESS_COMPILE
+
+        pvControlPc = GetIP(&ctx);
+
+        if (funCallDepth > 0)
+        {
+            --funCallDepth;
+            if (funCallDepth == 0)
+                break;
+        }
+        else
+        {
+            // Determine  whether given IP resides in JITted code. (It returns nonzero in that case.) 
+            // Use it now to see if we've unwound to managed code yet.
+            BOOL fFailedReaderLock = FALSE;
+            BOOL fIsManagedCode = ExecutionManager::IsManagedCode(pvControlPc, hostCallPreference, &fFailedReaderLock);            
+            if (fFailedReaderLock)
+            {
+                // We don't know if we would have been able to find a JIT
+                // manager, because we couldn't enter the reader lock without
+                // yielding (and our caller doesn't want us to yield).  So abort
+                // now.
+                
+                // Invalidate the lazyState we're returning, so the caller knows
+                // we aborted before we could fully unwind
+                lazyState->_pRetAddr = NULL;                
+                return;
+            }
+
+            if (fIsManagedCode)
+                break;
+        }
+    }
+    while(TRUE);    
+
+    lazyState->_esp = ctx.Esp;
+    lazyState->_pRetAddr = PTR_TADDR(lazyState->_esp - 4);
+
+    lazyState->_edi = ctx.Edi;
+    lazyState->_esi = ctx.Esi;
+    lazyState->_ebx = ctx.Ebx;
+    lazyState->_ebp = ctx.Ebp;
+   
+#ifdef DACCESS_COMPILE
+    lazyState->_pEdi = NULL;
+    lazyState->_pEsi = NULL;
+    lazyState->_pEbx = NULL;
+    lazyState->_pEbp = NULL;
+#else  // DACCESS_COMPILE
+    lazyState->_pEdi = nonVolRegPtrs.Edi;
+    lazyState->_pEsi = nonVolRegPtrs.Esi;
+    lazyState->_pEbx = nonVolRegPtrs.Ebx;
+    lazyState->_pEbp = nonVolRegPtrs.Ebp;
+#endif // DACCESS_COMPILE
+}
+#endif // !USE_EXTERNAL_UNWINDER