Only register signals and create alt exception stack in coreclr. (#19309)

There was a couple of places where the DAC (IsValidObject, GetAppDomainForObject)
assumed that a NULL target/debuggee address would throw an exception that would
be caught by try/catch. Any other invalid address is handled with a software
exception throwed by the read memory functions. In general it is a better overall
design not to have any of the DBI/DAC, etc. code depend on hardware exceptions
being caught. On Linux the C++ runtime sometimes can't handle it. There is a
slight risk that there are other places in the DAC that make the NULL address
assumption but testing so far has found any.

Added PAL_SetInitializeDLLFlags as a fallback to allow the PAL_InitializeDLL flags
to be set for a PAL instance for the DAC where we could still register h/w signals
but not the altstack switching to reduce this risk. The flags can't be build time
conditional because we only build one coreclrpal.a library that all the modules
used. Having a PAL_InitializeFlags function doesn't really help either because of
the PAL_RegisterModule call to PAL_IntializeDLL and the LoadLibrary dance/protocol
that uses it to call the loading module's DLLMain.

Add PAL_SetInitializeFlags; remove flags from PAL_INITIALIZE and PAL_INITIALIZE_DLL
default. Add PAL_InitializeFlags() to allowing the default to be overriden.

1 files changed, 27 insertions, 5 deletions

diff --git a/src/pal/inc/pal.h b/src/pal/inc/pal.h
index 993d76ed9e..d474a7c978 100644
--- a/src/pal/inc/pal.h
+++ b/src/pal/inc/pal.h
@@ -321,15 +321,24 @@ typedef long time_t;
 #define PAL_INITIALIZE_REGISTER_SIGTERM_HANDLER     0x08
 #define PAL_INITIALIZE_DEBUGGER_EXCEPTIONS          0x10
 #define PAL_INITIALIZE_ENSURE_STACK_SIZE            0x20
+#define PAL_INITIALIZE_REGISTER_SIGNALS             0x40
+#define PAL_INITIALIZE_ENSURE_ALT_SIGNAL_STACK      0x80
 
 // PAL_Initialize() flags
-#define PAL_INITIALIZE                 (PAL_INITIALIZE_SYNC_THREAD | PAL_INITIALIZE_STD_HANDLES)
+#define PAL_INITIALIZE                 (PAL_INITIALIZE_SYNC_THREAD | \
+                                        PAL_INITIALIZE_STD_HANDLES)
 
-// PAL_InitializeDLL() flags - don't start any of the helper threads
-#define PAL_INITIALIZE_DLL             PAL_INITIALIZE_NONE       
+// PAL_InitializeDLL() flags - don't start any of the helper threads or register any exceptions
+#define PAL_INITIALIZE_DLL              PAL_INITIALIZE_NONE       
 
 // PAL_InitializeCoreCLR() flags
-#define PAL_INITIALIZE_CORECLR         (PAL_INITIALIZE | PAL_INITIALIZE_EXEC_ALLOCATOR | PAL_INITIALIZE_REGISTER_SIGTERM_HANDLER | PAL_INITIALIZE_DEBUGGER_EXCEPTIONS | PAL_INITIALIZE_ENSURE_STACK_SIZE)
+#define PAL_INITIALIZE_CORECLR         (PAL_INITIALIZE | \
+                                        PAL_INITIALIZE_EXEC_ALLOCATOR | \
+                                        PAL_INITIALIZE_REGISTER_SIGTERM_HANDLER | \
+                                        PAL_INITIALIZE_DEBUGGER_EXCEPTIONS | \
+                                        PAL_INITIALIZE_ENSURE_STACK_SIZE | \
+                                        PAL_INITIALIZE_REGISTER_SIGNALS | \
+                                        PAL_INITIALIZE_ENSURE_ALT_SIGNAL_STACK)
 
 typedef DWORD (PALAPI *PTHREAD_START_ROUTINE)(LPVOID lpThreadParameter);
 typedef PTHREAD_START_ROUTINE LPTHREAD_START_ROUTINE;
@@ -344,9 +353,22 @@ PAL_Initialize(
     const char * const argv[]);
 
 PALIMPORT
+void
+PALAPI
+PAL_SetInitializeFlags(
+    DWORD flags);
+
+PALIMPORT
 int
 PALAPI
-PAL_InitializeDLL(VOID);
+PAL_InitializeDLL(
+    VOID);
+
+PALIMPORT
+void
+PALAPI
+PAL_SetInitializeDLLFlags(
+    DWORD flags);
 
 PALIMPORT
 DWORD