Use out-of-proc libunwind to unwind native stack from DAC

Implementation of PAL_VirtualUnwindOutOfProc that uses ptrace libunwind
to be able to unwind native stack in debugee. This allows to get valid managed
stack for threads that passed HelperMethodFrames.

3 files changed, 33 insertions, 9 deletions

diff --git a/src/debug/di/shimdatatarget.cpp b/src/debug/di/shimdatatarget.cpp
index 59f56e798e..d7bfbdc11d 100644
--- a/src/debug/di/shimdatatarget.cpp
+++ b/src/debug/di/shimdatatarget.cpp
@@ -36,6 +36,10 @@ HRESULT STDMETHODCALLTYPE ShimDataTarget::QueryInterface(
     {
         *pInterface = static_cast<ICorDebugMutableDataTarget *>(this);
     }
+    else if (InterfaceId == IID_ICorDebugDataTarget4)
+    {
+        *pInterface = static_cast<ICorDebugDataTarget4 *>(this);
+    }
     else
     {
         *pInterface = NULL;
@@ -71,9 +75,15 @@ ULONG STDMETHODCALLTYPE ShimDataTarget::Release()
 //
 // Return Value: 
 //     The OS PID of the process this data target is representing.
-DWORD ShimDataTarget::GetPid()
+HRESULT STDMETHODCALLTYPE ShimDataTarget::GetPid(DWORD *pdwProcessId)
 {
-    return m_processId;
+    if (pdwProcessId == NULL)  
+    {
+        return E_INVALIDARG;
+    }
+
+    *pdwProcessId = m_processId;
+    return S_OK;
 }
 
 //---------------------------------------------------------------------------------------
diff --git a/src/debug/di/shimdatatarget.h b/src/debug/di/shimdatatarget.h
index 1fabad4778..9e3502bb48 100644
--- a/src/debug/di/shimdatatarget.h
+++ b/src/debug/di/shimdatatarget.h
@@ -21,7 +21,7 @@ typedef HRESULT (*FPContinueStatusChanged)(void * pUserData, DWORD dwThreadId, C
 //---------------------------------------------------------------------------------------
 // Data target for a live process. This is used by Shim. 
 // 
-class ShimDataTarget : public ICorDebugMutableDataTarget
+class ShimDataTarget : public ICorDebugMutableDataTarget, ICorDebugDataTarget4
 {
 public:
     virtual ~ShimDataTarget() {}
@@ -36,9 +36,6 @@ public:
     // is unavailable because it's running
     void SetError(HRESULT hr);
 
-    // Get the OS Process ID that this DataTarget is for.
-    DWORD GetPid();
-
     //
     // IUnknown.
     //
@@ -85,6 +82,13 @@ public:
 
     // @dbgtodo - add Native Patch Table support
 
+    //
+    // ICorDebugDataTarget4
+    //    
+
+    // Get the OS Process ID that this DataTarget is for.
+    virtual HRESULT STDMETHODCALLTYPE GetPid(DWORD *pdwProcessId); 
+
 protected:
     // Pid of the target process.
     DWORD m_processId;
diff --git a/src/debug/di/shimprocess.cpp b/src/debug/di/shimprocess.cpp
index 6e0350b8eb..e4b511abec 100644
--- a/src/debug/di/shimprocess.cpp
+++ b/src/debug/di/shimprocess.cpp
@@ -133,7 +133,9 @@ void ShimProcess::SetProcess(ICorDebugProcess * pProcess)
     if (pProcess != NULL)
     {
         // Verify that DataTarget + new process have the same pid?
-        _ASSERTE(m_pProcess->GetPid() == m_pLiveDataTarget->GetPid());
+        DWORD pid = 0;
+        _ASSERTE(SUCCEEDED(m_pLiveDataTarget->GetPid(&pid)));
+        _ASSERTE(m_pProcess->GetPid() == pid);
     }
 }
 
@@ -738,7 +740,11 @@ HRESULT ShimProcess::HandleWin32DebugEvent(const DEBUG_EVENT * pEvent)
             // This assert could be our only warning of various catastrophic failures in the left-side.
             if (!dwFirstChance && (pRecord->ExceptionCode == STATUS_BREAKPOINT) && !m_fIsInteropDebugging)
             {            
-                DWORD pid = (m_pLiveDataTarget == NULL) ? 0 : m_pLiveDataTarget->GetPid();
+                DWORD pid = 0;
+                if (m_pLiveDataTarget != NULL) 
+                {
+                    m_pLiveDataTarget->GetPid(&pid);
+                }
                 CONSISTENCY_CHECK_MSGF(false, 
                     ("Unhandled breakpoint exception in debuggee (pid=%d (0x%x)) on thread %d(0x%x)\n"
                     "This may mean there was an assert in the debuggee on that thread.\n"
@@ -1742,7 +1748,11 @@ void ShimProcess::PreDispatchEvent(bool fRealCreateProcessEvent /*= false*/)
 CORDB_ADDRESS ShimProcess::GetCLRInstanceBaseAddress()
 {
     CORDB_ADDRESS baseAddress = CORDB_ADDRESS(NULL);
-    DWORD dwPid = m_pLiveDataTarget->GetPid();
+    DWORD dwPid = 0;
+    if (FAILED(m_pLiveDataTarget->GetPid(&dwPid)))
+    {
+        return baseAddress;
+    }
 #if defined(FEATURE_CORESYSTEM)
     // Debugger attaching to CoreCLR via CoreCLRCreateCordbObject should have already specified CLR module address.
     // Code that help to find it now lives in dbgshim.