Fix Unix exception handling in finalizers

When unhandled exception happens in a finalizer thread and there are
no managed frames till the bottom of the stack,
the Thread::VirtualUnwindToFirstManagedCallFrame then fails fast.
The fix is to make the Thread::VirtualUnwindToFirstManagedCallFrame to
return even in case no managed frame is called, which is indicated by
its returning 0 as the resulting IP address. The DispatchManagedException
then checks that and rethrows the exception instead of trying to call
UnwindManagedExceptionPass1.
I have also added INSTALL_UNHANDLED_MANAGED_EXCEPTION_TRAP to the
FinalizerThread::FinalizerThreadStart so that the unhandled exception
doesn't escape the stack.
And I also needed to modify the UNINSTALL_UNHANDLED_MANAGED_EXCEPTION_TRAP
to detect case when the unhandled exception filter was already executed so
that it doesn't double-report the unhandled exception.

4 files changed, 23 insertions, 19 deletions

diff --git a/src/vm/exceptionhandling.cpp b/src/vm/exceptionhandling.cpp
index 6f4ddec53e..35d95120dd 100644
--- a/src/vm/exceptionhandling.cpp
+++ b/src/vm/exceptionhandling.cpp
@@ -4661,7 +4661,14 @@ VOID DECLSPEC_NORETURN DispatchManagedException(PAL_SEHException& ex)
             CONTEXT frameContext;
             RtlCaptureContext(&frameContext);
             UINT_PTR currentSP = GetSP(&frameContext);
-            Thread::VirtualUnwindToFirstManagedCallFrame(&frameContext);
+
+            if (Thread::VirtualUnwindToFirstManagedCallFrame(&frameContext) == 0)
+            {
+                // There are no managed frames on the stack, so we need to continue unwinding using C++ exception
+                // handling
+                break;
+            }
+
             UINT_PTR firstManagedFrameSP = GetSP(&frameContext);
 
             // Check if there is any exception holder in the skipped frames. If there is one, we need to unwind them
diff --git a/src/vm/exceptmacros.h b/src/vm/exceptmacros.h
index 054601c2f5..5808102504 100644
--- a/src/vm/exceptmacros.h
+++ b/src/vm/exceptmacros.h
@@ -343,17 +343,17 @@ VOID DECLSPEC_NORETURN DispatchManagedException(PAL_SEHException& ex);
         try {                                                                                   
 
 // Uninstall trap that catches unhandled managed exception and dumps its stack
-#define UNINSTALL_UNHANDLED_MANAGED_EXCEPTION_TRAP                                          \
-        }                                                                                   \
-        catch (PAL_SEHException& ex)                                                        \
-        {                                                                                   \
-            DefaultCatchHandler(NULL /*pExceptionInfo*/,                                    \
-                                NULL /*Throwable*/,                                         \
-                                TRUE /*useLastThrownObject*/,                               \
-                                TRUE /*isTerminating*/,                                     \
-                                FALSE /*isThreadBaseFIlter*/,                               \
-                                FALSE /*sendAppDomainEvents*/);                             \
-            EEPOLICY_HANDLE_FATAL_ERROR(COR_E_EXECUTIONENGINE);                             \
+#define UNINSTALL_UNHANDLED_MANAGED_EXCEPTION_TRAP                                                  \
+        }                                                                                           \
+        catch (PAL_SEHException& ex)                                                                \
+        {                                                                                           \
+            if (!GetThread()->HasThreadStateNC(Thread::TSNC_ProcessedUnhandledException))           \
+            {                                                                                       \
+                LONG disposition = InternalUnhandledExceptionFilter_Worker(&ex.ExceptionPointers);  \
+                _ASSERTE(disposition == EXCEPTION_CONTINUE_SEARCH);                                 \
+            }                                                                                       \
+            TerminateProcess(GetCurrentProcess(), 1);                                               \
+            UNREACHABLE();                                                                          \
         }
 
 #else
diff --git a/src/vm/finalizerthread.cpp b/src/vm/finalizerthread.cpp
index 80ce9ca4c6..ba00cf66db 100644
--- a/src/vm/finalizerthread.cpp
+++ b/src/vm/finalizerthread.cpp
@@ -802,6 +802,8 @@ DWORD __stdcall FinalizerThread::FinalizerThreadStart(void *args)
 
     if (s_FinalizerThreadOK)
     {
+        INSTALL_UNHANDLED_MANAGED_EXCEPTION_TRAP;
+
 #ifdef _DEBUG       // The only purpose of this try/finally is to trigger an assertion
         EE_TRY_FOR_FINALLY(void *, unused, NULL)
         {
@@ -915,6 +917,7 @@ DWORD __stdcall FinalizerThread::FinalizerThreadStart(void *args)
         }
         EE_END_FINALLY;
 #endif
+        UNINSTALL_UNHANDLED_MANAGED_EXCEPTION_TRAP;
     }
     // finalizer should always park in default domain
     _ASSERTE(GetThread()->GetDomain()->IsDefaultDomain());
diff --git a/src/vm/stackwalk.cpp b/src/vm/stackwalk.cpp
index ec9acfbac2..7237c4c37d 100644
--- a/src/vm/stackwalk.cpp
+++ b/src/vm/stackwalk.cpp
@@ -789,14 +789,8 @@ UINT_PTR Thread::VirtualUnwindToFirstManagedCallFrame(T_CONTEXT* pContext)
 
         if (uControlPc == 0)
         {
-            // This displays the managed stack in case the unwind has walked out of the stack and
-            // a managed exception was being unwound.
-            DefaultCatchHandler(NULL /*pExceptionInfo*/, NULL /*Throwable*/, TRUE /*useLastThrownObject*/,
-                                TRUE /*isTerminating*/, FALSE /*isThreadBaseFIlter*/, FALSE /*sendAppDomainEvents*/);
-
-            EEPOLICY_HANDLE_FATAL_ERROR(COR_E_EXECUTIONENGINE);
+            break;
         }
-
 #endif // !FEATURE_PAL
     }