Fix exception in PreStubWorker

This change fixes a problem when exception happens in managed code called from
the PreStubWorker and the PreStubWorker (resp. its caller, ThePreStub) was called
from native code.
The issue was that the INSTALL_MANAGED_EXCEPTION_DISPATCHER calls DispatchManagedException
and that function expected that the INSTALL_MANAGED_EXCEPTION_DISPATCHER was always
at the boundary between managed and native frames and so it skipped the native frames
upto the first managed frame.
The PreStubWorker is the only case where this is not always true and so the native frames
need to be unwound by rethrowing the C++ exception when the PreStubWorker was called from
native code.

3 files changed, 45 insertions, 1 deletions

diff --git a/src/pal/inc/pal.h b/src/pal/inc/pal.h
index 56963673db..4e408ebbf7 100644
--- a/src/pal/inc/pal.h
+++ b/src/pal/inc/pal.h
@@ -6896,6 +6896,22 @@ public:
     }
 };
 
+// This is a native exception holder that doesn't catch any exceptions.
+class NativeExceptionHolderNoCatch : public NativeExceptionHolderBase
+{
+
+public:
+    NativeExceptionHolderNoCatch()
+        : NativeExceptionHolderBase()
+    {
+    }
+
+    virtual EXCEPTION_DISPOSITION InvokeFilter(PAL_SEHException& ex)
+    {
+        return EXCEPTION_CONTINUE_SEARCH;
+    }
+};
+
 //
 // This factory class for the native exception holder is necessary because
 // templated functions don't need the explicit type parameter and can infer
diff --git a/src/vm/callhelpers.h b/src/vm/callhelpers.h
index 497835f6d5..a3fa41b58a 100644
--- a/src/vm/callhelpers.h
+++ b/src/vm/callhelpers.h
@@ -447,6 +447,20 @@ void FillInRegTypeMap(int argOffset, CorElementType typ, BYTE * pMap);
 /* Macros used to indicate a call to managed code is starting/ending   */
 /***********************************************************************/
 
+#ifdef FEATURE_PAL
+// Install a native exception holder that doesn't catch any exceptions but its presence
+// in a stack range of native frames indicates that there was a call from native to
+// managed code. It is used by the DispatchManagedException to detect the case when
+// the INSTALL_MANAGED_EXCEPTION_DISPATCHER was not at the managed to native boundary.
+// For example in the PreStubWorker, which can be called from both native and managed
+// code.
+#define INSTALL_CALL_TO_MANAGED_EXCEPTION_HOLDER() \
+    NativeExceptionHolderNoCatch __exceptionHolder;    \
+    __exceptionHolder.Push();                           
+#else // FEATURE_PAL
+#define INSTALL_CALL_TO_MANAGED_EXCEPTION_HOLDER()
+#endif // FEATURE_PAL
+
 enum EEToManagedCallFlags
 {
     EEToManagedDefault                  = 0x0000,
@@ -478,6 +492,7 @@ enum EEToManagedCallFlags
         }                                                                       \
     }                                                                           \
     BEGIN_SO_TOLERANT_CODE(CURRENT_THREAD);                                     \
+    INSTALL_CALL_TO_MANAGED_EXCEPTION_HOLDER();                                 \
     INSTALL_COMPLUS_EXCEPTION_HANDLER_NO_DECLARE();
 
 #define END_CALL_TO_MANAGED()                                                   \
diff --git a/src/vm/exceptionhandling.cpp b/src/vm/exceptionhandling.cpp
index cbb7bc2310..c45db6de38 100644
--- a/src/vm/exceptionhandling.cpp
+++ b/src/vm/exceptionhandling.cpp
@@ -4671,8 +4671,18 @@ VOID DECLSPEC_NORETURN DispatchManagedException(PAL_SEHException& ex)
                 // Get the managed frame to continue unwinding from.
                 CONTEXT frameContext;
                 RtlCaptureContext(&frameContext);
+                UINT_PTR currentSP = GetSP(&frameContext);
                 Thread::VirtualUnwindToFirstManagedCallFrame(&frameContext);
- 
+                UINT_PTR firstManagedFrameSP = GetSP(&frameContext);
+            
+                // Check if there is any exception holder in the skipped frames. If there is one, we need to unwind them
+                // using the C++ handling. This is a special case when the UNINSTALL_MANAGED_EXCEPTION_DISPATCHER was
+                // not at the managed to native boundary.
+                if (NativeExceptionHolderBase::FindNextHolder(nullptr, (void*)currentSP, (void*)firstManagedFrameSP) != nullptr)
+                {
+                    break;
+                }
+
                 UnwindManagedExceptionPass2(ex, &frameContext);
             }
             UNREACHABLE();
@@ -4681,8 +4691,11 @@ VOID DECLSPEC_NORETURN DispatchManagedException(PAL_SEHException& ex)
         {
             ex = ex2;
         }
+
     }
     while (true);
+
+    throw ex;
 }
 
 #ifdef _AMD64_