Fix ensuring primary stack size (#16698)

1. Make it happen for Alpine only (it was incorrectly being done for
   OSX too.
2. Do it only when the Initialize is called for CLR initialization

3 files changed, 15 insertions, 8 deletions

diff --git a/src/pal/inc/pal.h b/src/pal/inc/pal.h
index 462accd4f9..7aadbda126 100644
--- a/src/pal/inc/pal.h
+++ b/src/pal/inc/pal.h
@@ -379,6 +379,7 @@ typedef long time_t;
 #define PAL_INITIALIZE_STD_HANDLES                  0x04
 #define PAL_INITIALIZE_REGISTER_SIGTERM_HANDLER     0x08
 #define PAL_INITIALIZE_DEBUGGER_EXCEPTIONS          0x10
+#define PAL_INITIALIZE_ENSURE_STACK_SIZE            0x20
 
 // PAL_Initialize() flags
 #define PAL_INITIALIZE                 (PAL_INITIALIZE_SYNC_THREAD | PAL_INITIALIZE_STD_HANDLES)
@@ -387,7 +388,7 @@ typedef long time_t;
 #define PAL_INITIALIZE_DLL             PAL_INITIALIZE_NONE       
 
 // PAL_InitializeCoreCLR() flags
-#define PAL_INITIALIZE_CORECLR         (PAL_INITIALIZE | PAL_INITIALIZE_EXEC_ALLOCATOR | PAL_INITIALIZE_REGISTER_SIGTERM_HANDLER | PAL_INITIALIZE_DEBUGGER_EXCEPTIONS)
+#define PAL_INITIALIZE_CORECLR         (PAL_INITIALIZE | PAL_INITIALIZE_EXEC_ALLOCATOR | PAL_INITIALIZE_REGISTER_SIGTERM_HANDLER | PAL_INITIALIZE_DEBUGGER_EXCEPTIONS | PAL_INITIALIZE_ENSURE_STACK_SIZE)
 
 typedef DWORD (PALAPI *PTHREAD_START_ROUTINE)(LPVOID lpThreadParameter);
 typedef PTHREAD_START_ROUTINE LPTHREAD_START_ROUTINE;
diff --git a/src/pal/src/CMakeLists.txt b/src/pal/src/CMakeLists.txt
index 56df33fa3f..a3ccd724f4 100644
--- a/src/pal/src/CMakeLists.txt
+++ b/src/pal/src/CMakeLists.txt
@@ -101,6 +101,9 @@ endif(PAL_CMAKE_PLATFORM_ARCH_AMD64 AND CMAKE_SYSTEM_NAME STREQUAL Linux AND NOT
 if(CLR_CMAKE_PLATFORM_ALPINE_LINUX)
   # Setting RLIMIT_NOFILE breaks debugging of coreclr on Alpine Linux for some reason
   add_definitions(-DDONT_SET_RLIMIT_NOFILE)
+  # On Alpine Linux, we need to ensure that the reported stack range for the primary thread is
+  # larger than the initial committed stack size.
+  add_definitions(-DENSURE_PRIMARY_STACK_SIZE)
 endif(CLR_CMAKE_PLATFORM_ALPINE_LINUX)
 
 # turn off capability to remove unused functions (which was enabled in debug build with sanitizers)
diff --git a/src/pal/src/init/pal.cpp b/src/pal/src/init/pal.cpp
index 996292e2c4..005fadaed4 100644
--- a/src/pal/src/init/pal.cpp
+++ b/src/pal/src/init/pal.cpp
@@ -174,7 +174,7 @@ PAL_InitializeDLL()
     return Initialize(0, NULL, PAL_INITIALIZE_DLL);
 }
 
-#ifndef __GLIBC__
+#ifdef ENSURE_PRIMARY_STACK_SIZE
 /*++
 Function:
   EnsureStackSize
@@ -197,7 +197,7 @@ EnsureStackSize(SIZE_T stackSize)
     volatile uint8_t *s = (uint8_t *)_alloca(stackSize);
     *s = 0;
 }
-#endif // __GLIBC__
+#endif // ENSURE_PRIMARY_STACK_SIZE
 
 /*++
 Function:
@@ -224,14 +224,14 @@ InitializeDefaultStackSize()
         }
     }
 
-#ifndef __GLIBC__
+#ifdef ENSURE_PRIMARY_STACK_SIZE
     if (g_defaultStackSize == 0)
     {
         // Set the default minimum stack size for MUSL to the same value as we
         // use on Windows.
         g_defaultStackSize = 1536 * 1024;
     }
-#endif // __GLIBC__
+#endif // ENSURE_PRIMARY_STACK_SIZE
 }
 
 /*++
@@ -312,9 +312,12 @@ Initialize(
 
         InitializeDefaultStackSize();
 
-#ifndef __GLIBC__
-        EnsureStackSize(g_defaultStackSize);
-#endif // __GLIBC__
+#ifdef ENSURE_PRIMARY_STACK_SIZE
+        if (flags & PAL_INITIALIZE_ENSURE_STACK_SIZE)
+        {
+            EnsureStackSize(g_defaultStackSize);
+        }
+#endif // ENSURE_PRIMARY_STACK_SIZE
 
         // Initialize the TLS lookaside cache
         if (FALSE == TLSInitialize())