Only register signals and create alt exception stack in coreclr. (#19309)

There was a couple of places where the DAC (IsValidObject, GetAppDomainForObject)
assumed that a NULL target/debuggee address would throw an exception that would
be caught by try/catch. Any other invalid address is handled with a software
exception throwed by the read memory functions. In general it is a better overall
design not to have any of the DBI/DAC, etc. code depend on hardware exceptions
being caught. On Linux the C++ runtime sometimes can't handle it. There is a
slight risk that there are other places in the DAC that make the NULL address
assumption but testing so far has found any.

Added PAL_SetInitializeDLLFlags as a fallback to allow the PAL_InitializeDLL flags
to be set for a PAL instance for the DAC where we could still register h/w signals
but not the altstack switching to reduce this risk. The flags can't be build time
conditional because we only build one coreclrpal.a library that all the modules
used. Having a PAL_InitializeFlags function doesn't really help either because of
the PAL_RegisterModule call to PAL_IntializeDLL and the LoadLibrary dance/protocol
that uses it to call the loading module's DLLMain.

Add PAL_SetInitializeFlags; remove flags from PAL_INITIALIZE and PAL_INITIALIZE_DLL
default. Add PAL_InitializeFlags() to allowing the default to be overriden.

6 files changed, 202 insertions, 103 deletions

diff --git a/src/pal/inc/pal.h b/src/pal/inc/pal.h
index 993d76ed9e..d474a7c978 100644
--- a/src/pal/inc/pal.h
+++ b/src/pal/inc/pal.h
@@ -321,15 +321,24 @@ typedef long time_t;
 #define PAL_INITIALIZE_REGISTER_SIGTERM_HANDLER     0x08
 #define PAL_INITIALIZE_DEBUGGER_EXCEPTIONS          0x10
 #define PAL_INITIALIZE_ENSURE_STACK_SIZE            0x20
+#define PAL_INITIALIZE_REGISTER_SIGNALS             0x40
+#define PAL_INITIALIZE_ENSURE_ALT_SIGNAL_STACK      0x80
 
 // PAL_Initialize() flags
-#define PAL_INITIALIZE                 (PAL_INITIALIZE_SYNC_THREAD | PAL_INITIALIZE_STD_HANDLES)
+#define PAL_INITIALIZE                 (PAL_INITIALIZE_SYNC_THREAD | \
+                                        PAL_INITIALIZE_STD_HANDLES)
 
-// PAL_InitializeDLL() flags - don't start any of the helper threads
-#define PAL_INITIALIZE_DLL             PAL_INITIALIZE_NONE       
+// PAL_InitializeDLL() flags - don't start any of the helper threads or register any exceptions
+#define PAL_INITIALIZE_DLL              PAL_INITIALIZE_NONE       
 
 // PAL_InitializeCoreCLR() flags
-#define PAL_INITIALIZE_CORECLR         (PAL_INITIALIZE | PAL_INITIALIZE_EXEC_ALLOCATOR | PAL_INITIALIZE_REGISTER_SIGTERM_HANDLER | PAL_INITIALIZE_DEBUGGER_EXCEPTIONS | PAL_INITIALIZE_ENSURE_STACK_SIZE)
+#define PAL_INITIALIZE_CORECLR         (PAL_INITIALIZE | \
+                                        PAL_INITIALIZE_EXEC_ALLOCATOR | \
+                                        PAL_INITIALIZE_REGISTER_SIGTERM_HANDLER | \
+                                        PAL_INITIALIZE_DEBUGGER_EXCEPTIONS | \
+                                        PAL_INITIALIZE_ENSURE_STACK_SIZE | \
+                                        PAL_INITIALIZE_REGISTER_SIGNALS | \
+                                        PAL_INITIALIZE_ENSURE_ALT_SIGNAL_STACK)
 
 typedef DWORD (PALAPI *PTHREAD_START_ROUTINE)(LPVOID lpThreadParameter);
 typedef PTHREAD_START_ROUTINE LPTHREAD_START_ROUTINE;
@@ -344,9 +353,22 @@ PAL_Initialize(
     const char * const argv[]);
 
 PALIMPORT
+void
+PALAPI
+PAL_SetInitializeFlags(
+    DWORD flags);
+
+PALIMPORT
 int
 PALAPI
-PAL_InitializeDLL(VOID);
+PAL_InitializeDLL(
+    VOID);
+
+PALIMPORT
+void
+PALAPI
+PAL_SetInitializeDLLFlags(
+    DWORD flags);
 
 PALIMPORT
 DWORD
diff --git a/src/pal/src/exception/signal.cpp b/src/pal/src/exception/signal.cpp
index 7f2e5c5b3d..ff3096a48c 100644
--- a/src/pal/src/exception/signal.cpp
+++ b/src/pal/src/exception/signal.cpp
@@ -103,7 +103,11 @@ static void restore_signal(int signal_id, struct sigaction *previousAction);
 
 /* internal data declarations *********************************************/
 
+static bool ensure_alt_signal_stack = false;
 static bool registered_sigterm_handler = false;
+#if !HAVE_MACH_EXCEPTIONS
+static bool registered_signal_handlers = false;
+#endif // !HAVE_MACH_EXCEPTIONS
 
 struct sigaction g_previous_sigterm;
 #if !HAVE_MACH_EXCEPTIONS
@@ -141,42 +145,47 @@ Return :
 --*/
 BOOL EnsureSignalAlternateStack()
 {
-    stack_t oss;
-
-    // Query the current alternate signal stack
-    int st = sigaltstack(NULL, &oss);
+    int st = 0;
 
-    if ((st == 0) && (oss.ss_flags == SS_DISABLE))
+    if (ensure_alt_signal_stack)
     {
-        // There is no alternate stack for SIGSEGV handling installed yet so allocate one
+        stack_t oss;
 
-        // We include the size of the SignalHandlerWorkerReturnPoint in the alternate stack size since the 
-        // context contained in it is large and the SIGSTKSZ was not sufficient on ARM64 during testing.
-        int altStackSize = SIGSTKSZ + ALIGN_UP(sizeof(SignalHandlerWorkerReturnPoint), 16) + GetVirtualPageSize();
-#ifdef HAS_ASAN
-        // Asan also uses alternate stack so we increase its size on the SIGSTKSZ * 4 that enough for asan
-        // (see kAltStackSize in compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc)
-        altStackSize += SIGSTKSZ * 4;
-#endif
-        altStackSize = ALIGN_UP(altStackSize, GetVirtualPageSize());
-        void* altStack = mmap(NULL, altStackSize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_STACK | MAP_PRIVATE, -1, 0);
-        if (altStack != MAP_FAILED)
+        // Query the current alternate signal stack
+        st = sigaltstack(NULL, &oss);
+
+        if ((st == 0) && (oss.ss_flags == SS_DISABLE))
         {
-            // create a guard page for the alternate stack
-            st = mprotect(altStack, GetVirtualPageSize(), PROT_NONE);
-            if (st == 0)
-            {
-                stack_t ss;
-                ss.ss_sp = (char*)altStack;
-                ss.ss_size = altStackSize;
-                ss.ss_flags = 0;
-                st = sigaltstack(&ss, NULL);
-            }
+            // There is no alternate stack for SIGSEGV handling installed yet so allocate one
 
-            if (st != 0)
+            // We include the size of the SignalHandlerWorkerReturnPoint in the alternate stack size since the 
+            // context contained in it is large and the SIGSTKSZ was not sufficient on ARM64 during testing.
+            int altStackSize = SIGSTKSZ + ALIGN_UP(sizeof(SignalHandlerWorkerReturnPoint), 16) + GetVirtualPageSize();
+#ifdef HAS_ASAN
+            // Asan also uses alternate stack so we increase its size on the SIGSTKSZ * 4 that enough for asan
+            // (see kAltStackSize in compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc)
+            altStackSize += SIGSTKSZ * 4;
+#endif
+            altStackSize = ALIGN_UP(altStackSize, GetVirtualPageSize());
+            void* altStack = mmap(NULL, altStackSize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_STACK | MAP_PRIVATE, -1, 0);
+            if (altStack != MAP_FAILED)
             {
-               int st2 = munmap(altStack, altStackSize);
-               _ASSERTE(st2 == 0);
+                // create a guard page for the alternate stack
+                st = mprotect(altStack, GetVirtualPageSize(), PROT_NONE);
+                if (st == 0)
+                {
+                    stack_t ss;
+                    ss.ss_sp = (char*)altStack;
+                    ss.ss_size = altStackSize;
+                    ss.ss_flags = 0;
+                    st = sigaltstack(&ss, NULL);
+                }
+
+                if (st != 0)
+                {
+                    int st2 = munmap(altStack, altStackSize);
+                    _ASSERTE(st2 == 0);
+                }
             }
         }
     }
@@ -198,17 +207,20 @@ Return :
 --*/
 void FreeSignalAlternateStack()
 {
-    stack_t ss, oss;
-    // The man page for sigaltstack says that when the ss.ss_flags is set to SS_DISABLE,
-    // all other ss fields are ignored. However, MUSL implementation checks that the 
-    // ss_size is >= MINSIGSTKSZ even in this case.
-    ss.ss_size = MINSIGSTKSZ;
-    ss.ss_flags = SS_DISABLE;
-    int st = sigaltstack(&ss, &oss);
-    if ((st == 0) && (oss.ss_flags != SS_DISABLE))
-    {
-        int st = munmap(oss.ss_sp, oss.ss_size);
-        _ASSERTE(st == 0);
+    if (ensure_alt_signal_stack)
+    {
+        stack_t ss, oss;
+        // The man page for sigaltstack says that when the ss.ss_flags is set to SS_DISABLE,
+        // all other ss fields are ignored. However, MUSL implementation checks that the 
+        // ss_size is >= MINSIGSTKSZ even in this case.
+        ss.ss_size = MINSIGSTKSZ;
+        ss.ss_flags = SS_DISABLE;
+        int st = sigaltstack(&ss, &oss);
+        if ((st == 0) && (oss.ss_flags != SS_DISABLE))
+        {
+            int st = munmap(oss.ss_sp, oss.ss_size);
+            _ASSERTE(st == 0);
+        }
     }
 }
 #endif // !HAVE_MACH_EXCEPTIONS
@@ -230,48 +242,48 @@ BOOL SEHInitializeSignals(DWORD flags)
     TRACE("Initializing signal handlers\n");
 
 #if !HAVE_MACH_EXCEPTIONS
-    /* we call handle_signal for every possible signal, even
-       if we don't provide a signal handler.
-
-       handle_signal will set SA_RESTART flag for specified signal.
-       Therefore, all signals will have SA_RESTART flag set, preventing
-       slow Unix system calls from being interrupted. On systems without
-       siginfo_t, SIGKILL and SIGSTOP can't be restarted, so we don't
-       handle those signals. Both the Darwin and FreeBSD man pages say
-       that SIGKILL and SIGSTOP can't be handled, but FreeBSD allows us
-       to register a handler for them anyway. We don't do that.
-
-       see sigaction man page for more details
-       */
-    handle_signal(SIGILL, sigill_handler, &g_previous_sigill);
-    handle_signal(SIGTRAP, sigtrap_handler, &g_previous_sigtrap);
-    handle_signal(SIGFPE, sigfpe_handler, &g_previous_sigfpe);
-    handle_signal(SIGBUS, sigbus_handler, &g_previous_sigbus);
-    // SIGSEGV handler runs on a separate stack so that we can handle stack overflow
-    handle_signal(SIGSEGV, sigsegv_handler, &g_previous_sigsegv, SA_ONSTACK);
-    // We don't setup a handler for SIGINT/SIGQUIT when those signals are ignored.
-    // Otherwise our child processes would reset to the default on exec causing them
-    // to terminate on these signals.
-    handle_signal(SIGINT, sigint_handler, &g_previous_sigint   , 0 /* additionalFlags */, true /* skipIgnored */);
-    handle_signal(SIGQUIT, sigquit_handler, &g_previous_sigquit, 0 /* additionalFlags */, true /* skipIgnored */);
-
-    if (!EnsureSignalAlternateStack())
-    {
-        return FALSE;
+    if (flags & PAL_INITIALIZE_REGISTER_SIGNALS)
+    {
+        registered_signal_handlers = true;
+        /* we call handle_signal for every possible signal, even
+           if we don't provide a signal handler.
+
+           handle_signal will set SA_RESTART flag for specified signal.
+           Therefore, all signals will have SA_RESTART flag set, preventing
+           slow Unix system calls from being interrupted. On systems without
+           siginfo_t, SIGKILL and SIGSTOP can't be restarted, so we don't
+           handle those signals. Both the Darwin and FreeBSD man pages say
+           that SIGKILL and SIGSTOP can't be handled, but FreeBSD allows us
+           to register a handler for them anyway. We don't do that.
+
+           see sigaction man page for more details
+           */
+        handle_signal(SIGILL, sigill_handler, &g_previous_sigill);
+        handle_signal(SIGTRAP, sigtrap_handler, &g_previous_sigtrap);
+        handle_signal(SIGFPE, sigfpe_handler, &g_previous_sigfpe);
+        handle_signal(SIGBUS, sigbus_handler, &g_previous_sigbus);
+        // SIGSEGV handler runs on a separate stack so that we can handle stack overflow
+        handle_signal(SIGSEGV, sigsegv_handler, &g_previous_sigsegv, SA_ONSTACK);
+        // We don't setup a handler for SIGINT/SIGQUIT when those signals are ignored.
+        // Otherwise our child processes would reset to the default on exec causing them
+        // to terminate on these signals.
+        handle_signal(SIGINT, sigint_handler, &g_previous_sigint, 0 /* additionalFlags */, true /* skipIgnored */);
+        handle_signal(SIGQUIT, sigquit_handler, &g_previous_sigquit, 0 /* additionalFlags */, true /* skipIgnored */);
+
+#ifdef INJECT_ACTIVATION_SIGNAL
+        handle_signal(INJECT_ACTIVATION_SIGNAL, inject_activation_handler, &g_previous_activation);
+#endif
     }
-#endif // !HAVE_MACH_EXCEPTIONS
 
-    if (flags & PAL_INITIALIZE_REGISTER_SIGTERM_HANDLER)
+    if (flags & PAL_INITIALIZE_ENSURE_ALT_SIGNAL_STACK)
     {
-        handle_signal(SIGTERM, sigterm_handler, &g_previous_sigterm);
-        registered_sigterm_handler = true;
+        ensure_alt_signal_stack = true;
+        if (!EnsureSignalAlternateStack())
+        {
+            return FALSE;
+        }
     }
 
-#if !HAVE_MACH_EXCEPTIONS
-#ifdef INJECT_ACTIVATION_SIGNAL
-    handle_signal(INJECT_ACTIVATION_SIGNAL, inject_activation_handler, &g_previous_activation);
-#endif
-
     /* The default action for SIGPIPE is process termination.
        Since SIGPIPE can be signaled when trying to write on a socket for which
        the connection has been dropped, we need to tell the system we want
@@ -283,6 +295,12 @@ BOOL SEHInitializeSignals(DWORD flags)
     signal(SIGPIPE, SIG_IGN);
 #endif // !HAVE_MACH_EXCEPTIONS
 
+    if (flags & PAL_INITIALIZE_REGISTER_SIGTERM_HANDLER)
+    {
+        registered_sigterm_handler = true;
+        handle_signal(SIGTERM, sigterm_handler, &g_previous_sigterm);
+    }
+
     return TRUE;
 }
 
@@ -307,25 +325,25 @@ void SEHCleanupSignals()
     TRACE("Restoring default signal handlers\n");
 
 #if !HAVE_MACH_EXCEPTIONS
-    restore_signal(SIGILL, &g_previous_sigill);
-    restore_signal(SIGTRAP, &g_previous_sigtrap);
-    restore_signal(SIGFPE, &g_previous_sigfpe);
-    restore_signal(SIGBUS, &g_previous_sigbus);
-    restore_signal(SIGSEGV, &g_previous_sigsegv);
-    restore_signal(SIGINT, &g_previous_sigint);
-    restore_signal(SIGQUIT, &g_previous_sigquit);
+    if (registered_signal_handlers)
+    {
+        restore_signal(SIGILL, &g_previous_sigill);
+        restore_signal(SIGTRAP, &g_previous_sigtrap);
+        restore_signal(SIGFPE, &g_previous_sigfpe);
+        restore_signal(SIGBUS, &g_previous_sigbus);
+        restore_signal(SIGSEGV, &g_previous_sigsegv);
+        restore_signal(SIGINT, &g_previous_sigint);
+        restore_signal(SIGQUIT, &g_previous_sigquit);
+#ifdef INJECT_ACTIVATION_SIGNAL
+        restore_signal(INJECT_ACTIVATION_SIGNAL, &g_previous_activation);
+#endif
+    }
 #endif // !HAVE_MACH_EXCEPTIONS
 
     if (registered_sigterm_handler)
     {
         restore_signal(SIGTERM, &g_previous_sigterm);
     }
-
-#if !HAVE_MACH_EXCEPTIONS
-#ifdef INJECT_ACTIVATION_SIGNAL
-    restore_signal(INJECT_ACTIVATION_SIGNAL, &g_previous_activation);
-#endif
-#endif // !HAVE_MACH_EXCEPTIONS
 }
 
 /* internal function definitions **********************************************/
diff --git a/src/pal/src/init/pal.cpp b/src/pal/src/init/pal.cpp
index 005fadaed4..9d0be758b0 100644
--- a/src/pal/src/init/pal.cpp
+++ b/src/pal/src/init/pal.cpp
@@ -94,7 +94,6 @@ using namespace CorUnix;
 
 extern "C" BOOL CRTInitStdStreams( void );
 
-
 Volatile<INT> init_count = 0;
 Volatile<BOOL> shutdown_intent = 0;
 Volatile<LONG> g_coreclrInitialized = 0;
@@ -108,6 +107,8 @@ SIZE_T g_defaultStackSize = 0;
    very first PAL_Initialize call, and is freed afterward. */
 static PCRITICAL_SECTION init_critsec = NULL;
 
+static DWORD g_initializeDLLFlags = PAL_INITIALIZE_DLL;
+
 static int Initialize(int argc, const char *const argv[], DWORD flags);
 static BOOL INIT_IncreaseDescriptorLimit(void);
 static LPWSTR INIT_FormatCommandLine (int argc, const char * const *argv);
@@ -157,6 +158,30 @@ PAL_Initialize(
 
 /*++
 Function:
+  PAL_InitializeFlags
+
+Abstract:
+  This function is the first function of the PAL to be called.
+  Internal structure initialization is done here. It could be called
+  several time by the same process, a reference count is kept.
+
+Return:
+  0 if successful
+  -1 if it failed
+
+--*/
+int
+PALAPI
+PAL_InitializeFlags(
+    int argc,
+    const char *const argv[],
+    DWORD flags)
+{
+    return Initialize(argc, argv, flags);
+}
+
+/*++
+Function:
   PAL_InitializeDLL
 
 Abstract:
@@ -171,7 +196,29 @@ int
 PALAPI
 PAL_InitializeDLL()
 {
-    return Initialize(0, NULL, PAL_INITIALIZE_DLL);
+    return Initialize(0, NULL, g_initializeDLLFlags);
+}
+
+/*++
+Function:
+  PAL_SetInitializeDLLFlags
+
+Abstract:
+  This sets the global PAL_INITIALIZE flags that PAL_InitializeDLL
+  will use. It needs to be called before any PAL_InitializeDLL call
+  is made so typical it is used in a __attribute__((constructor))
+  function to make sure. 
+
+Return:
+  none
+
+--*/
+void
+PALAPI
+PAL_SetInitializeDLLFlags(
+    DWORD flags)
+{
+    g_initializeDLLFlags = flags;
 }
 
 #ifdef ENSURE_PRIMARY_STACK_SIZE
diff --git a/src/pal/src/thread/process.cpp b/src/pal/src/thread/process.cpp
index da55bff9c3..5794def818 100644
--- a/src/pal/src/thread/process.cpp
+++ b/src/pal/src/thread/process.cpp
@@ -1589,9 +1589,12 @@ public:
 
         // See semaphore name format for details about this value. We store it so that
         // it can be used by the cleanup code that removes the semaphore with sem_unlink.
-        INDEBUG(BOOL disambiguationKeyRet = )
-        GetProcessIdDisambiguationKey(m_processId, &m_processIdDisambiguationKey);
-        _ASSERTE(disambiguationKeyRet == TRUE || m_processIdDisambiguationKey == 0);
+        BOOL ret = GetProcessIdDisambiguationKey(m_processId, &m_processIdDisambiguationKey);
+
+        // If GetProcessIdDisambiguationKey failed for some reason, it should set the value 
+        // to 0. We expect that anyone else opening the semaphore name will also fail and thus 
+        // will also try to use 0 as the value.
+        _ASSERTE(ret == TRUE || m_processIdDisambiguationKey == 0);
 
         sprintf_s(startupSemName,
                   sizeof(startupSemName),
@@ -1901,7 +1904,12 @@ PAL_NotifyRuntimeStarted()
     BOOL launched = FALSE;
 
     UINT64 processIdDisambiguationKey = 0;
-    GetProcessIdDisambiguationKey(gPID, &processIdDisambiguationKey);
+    BOOL ret = GetProcessIdDisambiguationKey(gPID, &processIdDisambiguationKey);
+
+    // If GetProcessIdDisambiguationKey failed for some reason, it should set the value 
+    // to 0. We expect that anyone else making the semaphore name will also fail and thus 
+    // will also try to use 0 as the value.
+    _ASSERTE(ret == TRUE || processIdDisambiguationKey == 0);
 
     sprintf_s(startupSemName, sizeof(startupSemName), RuntimeStartupSemaphoreName, HashSemaphoreName(gPID, processIdDisambiguationKey));
     sprintf_s(continueSemName, sizeof(continueSemName), RuntimeContinueSemaphoreName, HashSemaphoreName(gPID, processIdDisambiguationKey));
@@ -2040,6 +2048,7 @@ GetProcessIdDisambiguationKey(DWORD processId, UINT64 *disambiguationKey)
     FILE *statFile = fopen(statFileName, "r");
     if (statFile == nullptr) 
     {
+        TRACE("GetProcessIdDisambiguationKey: fopen() FAILED");
         SetLastError(ERROR_INVALID_HANDLE);
         return FALSE;
     }
@@ -2048,7 +2057,8 @@ GetProcessIdDisambiguationKey(DWORD processId, UINT64 *disambiguationKey)
     size_t lineLen = 0;
     if (getline(&line, &lineLen, statFile) == -1)
     {
-        _ASSERTE(!"Failed to getline from the stat file for a process.");
+        TRACE("GetProcessIdDisambiguationKey: getline() FAILED");
+        SetLastError(ERROR_INVALID_HANDLE);
         return FALSE;
     }
 
diff --git a/src/pal/tests/palsuite/exception_handling/pal_sxs/test1/dlltest1.cpp b/src/pal/tests/palsuite/exception_handling/pal_sxs/test1/dlltest1.cpp
index 000ed6271c..8eb10c91ac 100644
--- a/src/pal/tests/palsuite/exception_handling/pal_sxs/test1/dlltest1.cpp
+++ b/src/pal/tests/palsuite/exception_handling/pal_sxs/test1/dlltest1.cpp
@@ -17,6 +17,7 @@
 extern "C"
 int InitializeDllTest1()
 {
+    PAL_SetInitializeDLLFlags(PAL_INITIALIZE_DLL | PAL_INITIALIZE_REGISTER_SIGNALS | PAL_INITIALIZE_ENSURE_ALT_SIGNAL_STACK);
     return PAL_InitializeDLL();
 }
 
diff --git a/src/pal/tests/palsuite/exception_handling/pal_sxs/test1/dlltest2.cpp b/src/pal/tests/palsuite/exception_handling/pal_sxs/test1/dlltest2.cpp
index 27fc213e4f..5b1d2781d1 100644
--- a/src/pal/tests/palsuite/exception_handling/pal_sxs/test1/dlltest2.cpp
+++ b/src/pal/tests/palsuite/exception_handling/pal_sxs/test1/dlltest2.cpp
@@ -17,6 +17,7 @@
 extern "C"
 int InitializeDllTest2()
 {
+    PAL_SetInitializeDLLFlags(PAL_INITIALIZE_DLL | PAL_INITIALIZE_REGISTER_SIGNALS | PAL_INITIALIZE_ENSURE_ALT_SIGNAL_STACK);
     return PAL_InitializeDLL();
 }