Break into debugger on assertion failures (#19702)

* Break into debugger on assertion failures

Assertion failures terminated the process by default that made them hard to debug. Changed them to
break into debugger or trigger fail fast when the debugger is not attached. This should make the day-to-day
CoreCLR developer experience better and it is simular to what we had on .NET Framework in the past.

* Fix Unix build break

Add RaiseFailFastException to Unix PAL

6 files changed, 36 insertions, 62 deletions

diff --git a/src/dlls/mscordac/mscordac_unixexports.src b/src/dlls/mscordac/mscordac_unixexports.src
index 60fae97efd..995f5da28f 100644
--- a/src/dlls/mscordac/mscordac_unixexports.src
+++ b/src/dlls/mscordac/mscordac_unixexports.src
@@ -154,6 +154,7 @@ nativeStringResourceTable_mscorrc_debug
 #QueryPerformanceCounter
 #QueryPerformanceFrequency
 #RaiseException
+#RaiseFailFastException
 #ReadFile
 #ReleaseMutex
 #ReleaseSemaphore
diff --git a/src/pal/inc/pal.h b/src/pal/inc/pal.h
index 9676b437b0..e7ec886e87 100644
--- a/src/pal/inc/pal.h
+++ b/src/pal/inc/pal.h
@@ -3187,14 +3187,13 @@ RaiseException(
            IN DWORD nNumberOfArguments,
            IN CONST ULONG_PTR *lpArguments);
 
-#ifdef FEATURE_PAL_SXS
 PALIMPORT
-PAL_NORETURN
 VOID
 PALAPI
-PAL_RaiseException(
-           IN PEXCEPTION_POINTERS ExceptionPointers);
-#endif // FEATURE_PAL_SXS
+RaiseFailFastException(
+    IN PEXCEPTION_RECORD pExceptionRecord,
+    IN PCONTEXT pContextRecord,
+    IN DWORD dwFlags);
 
 PALIMPORT
 DWORD
diff --git a/src/pal/src/thread/process.cpp b/src/pal/src/thread/process.cpp
index 5794def818..2dfedc09d7 100644
--- a/src/pal/src/thread/process.cpp
+++ b/src/pal/src/thread/process.cpp
@@ -1316,6 +1316,29 @@ TerminateProcess(
 
 /*++
 Function:
+  RaiseFailFastException
+
+See MSDN doc.
+--*/
+VOID
+PALAPI
+RaiseFailFastException(
+    IN PEXCEPTION_RECORD pExceptionRecord,
+    IN PCONTEXT pContextRecord,
+    IN DWORD dwFlags)
+{
+    PERF_ENTRY(RaiseFailFastException);
+    ENTRY("RaiseFailFastException");
+
+    TerminateCurrentProcessNoExit(TRUE);
+    PROCAbort();
+
+    LOGEXIT("RaiseFailFastException");
+    PERF_EXIT(RaiseFailFastException);
+}
+
+/*++
+Function:
   PROCEndProcess
   
   Called from TerminateProcess and ExitProcess. This does the work of
diff --git a/src/utilcode/debug.cpp b/src/utilcode/debug.cpp
index 3d8704826b..a19e7a4f63 100644
--- a/src/utilcode/debug.cpp
+++ b/src/utilcode/debug.cpp
@@ -180,7 +180,7 @@ VOID TerminateOnAssert()
     STATIC_CONTRACT_DEBUG_ONLY;
 
     ShutdownLogging();
-    TerminateProcess(GetCurrentProcess(), 123456789);
+    RaiseFailFastException(NULL, NULL, 0);
 }
 
 // Whether this thread is already displaying an assert dialog.
@@ -431,14 +431,14 @@ bool _DbgBreakCheck(
         return false;       // don't stop debugger. No gui.
     }
 
-    if (NoGuiOnAssert())
+    if (IsDebuggerPresent() || DebugBreakOnAssert())
     {
-        TerminateOnAssert();
+        return true;       // like a retry
     }
 
-    if (DebugBreakOnAssert())
+    if (NoGuiOnAssert())
     {
-        return true;       // like a retry
+        TerminateOnAssert();
     }
 
     if (IsDisplayingAssertDlg())
@@ -870,12 +870,9 @@ void DECLSPEC_NORETURN __FreeBuildAssertFail(const char *szFile, int iLine, cons
 
     _flushall();
 
-    //    TerminateOnAssert();
     ShutdownLogging();
 
-    // Failing here implies an error in the runtime - hence we use
-    // COR_E_EXECUTIONENGINE
-    TerminateProcess(GetCurrentProcess(), COR_E_EXECUTIONENGINE);
+    RaiseFailFastException(NULL, NULL, 0);
 
     UNREACHABLE();
 }
diff --git a/src/vm/excep.cpp b/src/vm/excep.cpp
index 306e4d0d83..223ad83ae7 100644
--- a/src/vm/excep.cpp
+++ b/src/vm/excep.cpp
@@ -4172,7 +4172,7 @@ LONG WatsonLastChance(                  // EXCEPTION_CONTINUE_SEARCH, _CONTINUE_
                     pThread->GetFrame()->Pop(pThread);
                 }
 
-                LOG((LF_EH, LL_INFO10, "D::WLC: Call RaiseFailFastExceptionOnWin7\n"));
+                LOG((LF_EH, LL_INFO10, "D::WLC: Call RaiseFailFastException\n"));
 
                 // enable preemptive mode before call into OS to allow runtime suspend to finish
                 GCX_PREEMP();
@@ -6377,49 +6377,8 @@ FCIMPL1(Object*, MissingMemberException_FormatSignature, I1Array* pPersistedSigU
 
     HELPER_METHOD_FRAME_END();
     return OBJECTREFToObject(pString);
-    }
-FCIMPLEND
-
-// Check if the Win32 Error code is an IO error.
-BOOL IsWin32IOError(SCODE scode)
-{
-    LIMITED_METHOD_CONTRACT;
-
-    switch (scode)
-    {
-    case ERROR_FILE_NOT_FOUND:
-    case ERROR_PATH_NOT_FOUND:
-    case ERROR_TOO_MANY_OPEN_FILES:
-    case ERROR_ACCESS_DENIED:
-    case ERROR_INVALID_HANDLE:
-    case ERROR_INVALID_DRIVE:
-    case ERROR_WRITE_PROTECT:
-    case ERROR_NOT_READY:
-    case ERROR_WRITE_FAULT:
-    case ERROR_SHARING_VIOLATION:
-    case ERROR_LOCK_VIOLATION:
-    case ERROR_SHARING_BUFFER_EXCEEDED:
-    case ERROR_HANDLE_DISK_FULL:
-    case ERROR_BAD_NETPATH:
-    case ERROR_DEV_NOT_EXIST:
-    case ERROR_FILE_EXISTS:
-    case ERROR_CANNOT_MAKE:
-    case ERROR_NET_WRITE_FAULT:
-    case ERROR_DRIVE_LOCKED:
-    case ERROR_OPEN_FAILED:
-    case ERROR_BUFFER_OVERFLOW:
-    case ERROR_DISK_FULL:
-    case ERROR_INVALID_NAME:
-    case ERROR_FILENAME_EXCED_RANGE:
-    case ERROR_IO_DEVICE:
-    case ERROR_DISK_OPERATION_FAILED:
-        return TRUE;
-
-    default:
-        return FALSE;
-    }
 }
-
+FCIMPLEND
 
 // Check if there is a pending exception or the thread is already aborting. Returns 0 if yes.
 // Otherwise, sets the thread up for generating an abort and returns address of ThrowControlForThread
diff --git a/src/vm/excep.h b/src/vm/excep.h
index 8c49071a81..0364b1a0ff 100644
--- a/src/vm/excep.h
+++ b/src/vm/excep.h
@@ -33,11 +33,6 @@ bool IsIPInEpilog(PTR_CONTEXT pContextToCheck, EECodeInfo *pCodeInfo, BOOL *pSaf
 
 #endif // defined(_TARGET_AMD64_) && defined(FEATURE_HIJACK)
 
-void RaiseFailFastExceptionOnWin7(PEXCEPTION_RECORD pExceptionRecord, PT_CONTEXT pContext);
-
-// Check if the Win32 Error code is an IO error.
-BOOL IsWin32IOError(SCODE scode);
-
 //******************************************************************************
 //
 //  SwallowUnhandledExceptions