Added support for debugging a sandboxed app on Mac (#21068)

* Fixed bug in StackString where the size is not initialized correctly to STACK_COUNT

* Added CharString and WCharString template classes and a generic CharStringFromLPCWSTR method

* Added support for debugging a sandboxed app on Mac

This change fixes the usage of IPC while debugging while running in a sandbox. When running in a sandbox, the temporary folder for each process will be different. Thus the pipes being created in TwoWayPipe right now would be created in different directories in the debugger process and the process being debugged.
This change configures the folder to be used based on the application group ID that the sandboxed app belongs to.
For the same reasons, the names sempahores being used to synchronize the debugger attach need to be prefixed with the application group ID. This change was abit more involved since the name of the semaphore is limited to 31 characters, so we had to encode the semaphore names differently to make them shorter.

Last, new APIs to the debugger shim were added to support this new feature. This change only handles the runtime side and the dbgshim. An additional change to vsdbg needs to be done to use the new APIs.

fixes #21066

* Fixed build breaks on non Mac Unix platforms

* Fixed usage of gApplicationGroupId in non apple environments

* Fixed bug in semaphore names

* Got rid of usage of StackString

* Made PAL_GetApplicationGroupId Apple specific

* Added comment about pragma pack

* Fixed comment

* Added exported symbols

* Duplicated applicationGroupId so it can be used from another thread during register complete callback

* Renamed BitNum2ByteNum to GetExtraEncodedAreaSize to make intent clearer

* Fixed nit comments

* Removed redundant changes in StackString

* Fixed windows build break

* Fixed compilation switch from __APPLE to __APPLE__

* Added missing exports

6 files changed, 157 insertions, 31 deletions

diff --git a/src/dlls/dbgshim/dbgshim.cpp b/src/dlls/dbgshim/dbgshim.cpp
index 1e5c876888..020dbdc564 100644
--- a/src/dlls/dbgshim/dbgshim.cpp
+++ b/src/dlls/dbgshim/dbgshim.cpp
@@ -194,11 +194,61 @@ GetContinueStartupEvent(
 
 // Functions that we'll look for in the loaded Mscordbi module.
 typedef HRESULT (STDAPICALLTYPE *FPCoreCLRCreateCordbObject)(
-    int iDebuggerVersion, 
-    DWORD pid, 
-    HMODULE hmodTargetCLR, 
+    int iDebuggerVersion,
+    DWORD pid,
+    HMODULE hmodTargetCLR,
+    IUnknown **ppCordb);
+
+typedef HRESULT (STDAPICALLTYPE *FPCoreCLRCreateCordbObjectEx)(
+    int iDebuggerVersion,
+    DWORD pid,
+    LPCWSTR lpApplicationGroupId,
+    HMODULE hmodTargetCLR,
     IUnknown **ppCordb);
 
+HRESULT CreateCoreDbgWithSandboxSupport(
+    HMODULE hCLRModule, HMODULE hDBIModule, DWORD processId, LPCWSTR lpApplicationGroupId, int iDebuggerVersion, IUnknown **ppCordb)
+{
+    FPCoreCLRCreateCordbObjectEx fpCreate =
+        (FPCoreCLRCreateCordbObjectEx)GetProcAddress(hDBIModule, "CoreCLRCreateCordbObjectEx");
+    if (fpCreate == NULL)
+    {
+        return CORDBG_E_INCOMPATIBLE_PROTOCOL;
+    }
+
+    return fpCreate(iDebuggerVersion, processId, lpApplicationGroupId, hCLRModule, ppCordb);
+}
+
+HRESULT CreateCoreDbgWithoutSandboxSupport(
+    HMODULE hCLRModule, HMODULE hDBIModule, DWORD processId, int iDebuggerVersion, IUnknown **ppCordb)
+{
+    FPCoreCLRCreateCordbObject fpCreate =
+        (FPCoreCLRCreateCordbObject)GetProcAddress(hDBIModule, "CoreCLRCreateCordbObject");
+    if (fpCreate == NULL)
+    {
+        return CORDBG_E_INCOMPATIBLE_PROTOCOL;
+    }
+
+    return fpCreate(iDebuggerVersion, processId, hCLRModule, ppCordb);
+}
+
+HRESULT CreateCoreDbg(
+    HMODULE hCLRModule, HMODULE hDBIModule, DWORD processId, LPCWSTR lpApplicationGroupId, int iDebuggerVersion, IUnknown **ppCordb)
+{
+    HRESULT hr = S_OK;
+
+    if (lpApplicationGroupId != NULL)
+    {
+        hr = CreateCoreDbgWithSandboxSupport(hCLRModule, hDBIModule, processId, lpApplicationGroupId, iDebuggerVersion, ppCordb);
+    }
+    else
+    {
+        hr = CreateCoreDbgWithoutSandboxSupport(hCLRModule, hDBIModule, processId, iDebuggerVersion, ppCordb);
+    }
+
+    return hr;
+}
+
 //
 // Helper class for RegisterForRuntimeStartup
 //
@@ -210,6 +260,7 @@ class RuntimeStartupHelper
     PVOID m_parameter;
 #ifdef FEATURE_PAL
     PVOID m_unregisterToken;
+    LPWSTR m_applicationGroupId;
 #else
     bool m_canceled;
     HANDLE m_startupEvent;
@@ -224,7 +275,8 @@ public:
         m_callback(pfnCallback),
         m_parameter(parameter),
 #ifdef FEATURE_PAL
-        m_unregisterToken(NULL)
+        m_unregisterToken(NULL),
+        m_applicationGroupId(NULL)
 #else
         m_canceled(false),
         m_startupEvent(NULL),
@@ -236,7 +288,12 @@ public:
 
     ~RuntimeStartupHelper()
     {
-#ifndef FEATURE_PAL
+#ifdef FEATURE_PAL
+        if (m_applicationGroupId != NULL)
+        {
+            delete m_applicationGroupId;
+        }
+#else // FEATURE_PAL
         if (m_startupEvent != NULL)
         {
             CloseHandle(m_startupEvent);
@@ -266,9 +323,20 @@ public:
 
 #ifdef FEATURE_PAL
 
-    HRESULT Register()
+    HRESULT Register(LPCWSTR lpApplicationGroupId)
     {
-        DWORD pe = PAL_RegisterForRuntimeStartup(m_processId, RuntimeStartupHandler, this, &m_unregisterToken);
+        if (lpApplicationGroupId != NULL)
+        {
+            int size = wcslen(lpApplicationGroupId) + 1;
+            m_applicationGroupId = new (nothrow) WCHAR[size];
+            if (m_applicationGroupId == NULL)
+            {
+                return E_OUTOFMEMORY;
+            }
+            wcscpy_s(m_applicationGroupId, size, lpApplicationGroupId);
+        }
+
+        DWORD pe = PAL_RegisterForRuntimeStartup(m_processId, m_applicationGroupId, RuntimeStartupHandler, this, &m_unregisterToken);
         if (pe != NO_ERROR)
         {
             return HRESULT_FROM_WIN32(pe);
@@ -297,7 +365,6 @@ public:
 
         PAL_CPP_TRY
         {
-            FPCoreCLRCreateCordbObject fpCreate = NULL;
             char dbiPath[MAX_LONGPATH];
 
             char *pszLast = strrchr(pszModulePath, DIRECTORY_SEPARATOR_CHAR_A);
@@ -318,14 +385,7 @@ public:
                 goto exit;
             }
 
-            fpCreate = (FPCoreCLRCreateCordbObject)GetProcAddress(hMod, "CoreCLRCreateCordbObject");
-            if (fpCreate == NULL)
-            {
-                hr = CORDBG_E_INCOMPATIBLE_PROTOCOL;
-                goto exit;
-            }
-
-            HRESULT hr = fpCreate(CorDebugVersion_2_0, m_processId, hModule, &pCordb);
+            HRESULT hr = CreateCoreDbg(hModule, hMod, m_processId, m_applicationGroupId, CorDebugVersion_2_0, &pCordb);
             _ASSERTE((pCordb == NULL) == FAILED(hr));
             if (FAILED(hr))
             {
@@ -358,7 +418,7 @@ public:
 
 #else // FEATURE_PAL
 
-    HRESULT Register()
+    HRESULT Register(LPCWSTR lpApplicationGroupId)
     {
         HRESULT hr = GetStartupNotificationEvent(m_processId, &m_startupEvent);
         if (FAILED(hr))
@@ -623,7 +683,28 @@ StartupHelperThread(LPVOID p)
 //-----------------------------------------------------------------------------
 // Public API.
 //
-// RegisterForRuntimeStartup -- executes the callback when the coreclr runtime 
+// RegisterForRuntimeStartup -- Refer to RegisterForRuntimeStartupEx.
+//      This method calls RegisterForRuntimeStartupEx with null application group ID value
+//
+// dwProcessId -- process id of the target process
+// pfnCallback -- invoked when coreclr runtime starts
+// parameter -- data to pass to callback
+// ppUnregisterToken -- pointer to put the UnregisterForRuntimeStartup token.
+//
+//-----------------------------------------------------------------------------
+HRESULT
+RegisterForRuntimeStartup(
+    __in DWORD dwProcessId,
+    __in PSTARTUP_CALLBACK pfnCallback,
+    __in PVOID parameter,
+    __out PVOID *ppUnregisterToken)
+{
+    return RegisterForRuntimeStartupEx(dwProcessId, NULL, pfnCallback, parameter, ppUnregisterToken);
+}
+//-----------------------------------------------------------------------------
+// Public API.
+//
+// RegisterForRuntimeStartupEx -- executes the callback when the coreclr runtime 
 //      starts in the specified process. The callback is passed the proper ICorDebug
 //      instance for the version of the runtime or an error if something fails. This
 //      API works for launch and attach (and even the attach scenario if the runtime
@@ -643,14 +724,18 @@ StartupHelperThread(LPVOID p)
 //      supported.
 //
 // dwProcessId -- process id of the target process
+// lpApplicationGroupId - A string representing the application group ID of a sandboxed
+//                        process running in Mac. Pass NULL if the process is not 
+//                        running in a sandbox and other platforms.
 // pfnCallback -- invoked when coreclr runtime starts
 // parameter -- data to pass to callback
 // ppUnregisterToken -- pointer to put the UnregisterForRuntimeStartup token.
 //
 //-----------------------------------------------------------------------------
 HRESULT
-RegisterForRuntimeStartup(
+RegisterForRuntimeStartupEx(
     __in DWORD dwProcessId,
+    __in LPCWSTR lpApplicationGroupId,
     __in PSTARTUP_CALLBACK pfnCallback,
     __in PVOID parameter,
     __out PVOID *ppUnregisterToken)
@@ -671,7 +756,7 @@ RegisterForRuntimeStartup(
     }
     else
     {
-        hr = helper->Register();
+        hr = helper->Register(lpApplicationGroupId);
         if (FAILED(hr))
         {
             helper->Release();
@@ -1628,6 +1713,35 @@ CreateDebuggingInterfaceFromVersionEx(
     __in LPCWSTR szDebuggeeVersion,
     __out IUnknown ** ppCordb)
 {
+    return CreateDebuggingInterfaceFromVersion2(iDebuggerVersion, szDebuggeeVersion, NULL, ppCordb);
+}
+
+//-----------------------------------------------------------------------------
+// Public API.
+// Given a version string, create the matching mscordbi.dll for it.
+// Create a managed debugging interface for the specified version.
+//
+// Parameters:
+//    iDebuggerVersion - the version of interface the debugger (eg, Cordbg) expects.
+//    szDebuggeeVersion - the version of the debuggee. This will map to a version of mscordbi.dll
+//    lpApplicationGroupId - A string representing the application group ID of a sandboxed
+//                           process running in Mac. Pass NULL if the process is not 
+//                           running in a sandbox and other platforms.
+//    ppCordb - the outparameter used to return the debugging interface object.
+//
+// Return:
+//  S_OK on success. *ppCordb will be non-null.
+//  CORDBG_E_INCOMPATIBLE_PROTOCOL - if the proper DBI is not available. This can be a very common error if
+//    the right debug pack is not installed.
+//  else Error. (*ppCordb will be null)
+//-----------------------------------------------------------------------------
+HRESULT 
+CreateDebuggingInterfaceFromVersion2(
+    __in int iDebuggerVersion,
+    __in LPCWSTR szDebuggeeVersion,
+    __in LPCWSTR szApplicationGroupId,
+    __out IUnknown ** ppCordb)
+{
     PUBLIC_CONTRACT;
 
     HRESULT hrIgnore = S_OK; // ignored HResult
@@ -1707,17 +1821,8 @@ CreateDebuggingInterfaceFromVersionEx(
 
     //
     // Step 3: Now that module is loaded, instantiate an ICorDebug.
-    // 
-    fpCreate2 = (FPCoreCLRCreateCordbObject)GetProcAddress(hMod, "CoreCLRCreateCordbObject");
-    if (fpCreate2 == NULL)
-    {
-        // New-style creation API didn't exist - this DBI must be the wrong version, for the Mix07 protocol
-        hr = CORDBG_E_INCOMPATIBLE_PROTOCOL;
-        goto Exit;
-    }
-
-    // Invoke to instantiate an ICorDebug. This export was introduced after the Mix'07 release.
-    hr = fpCreate2(iDebuggerVersion, pidDebuggee, hmodTargetCLR, &pCordb);
+    //
+    hr = CreateCoreDbg(hmodTargetCLR, hMod, pidDebuggee, szApplicationGroupId, iDebuggerVersion, &pCordb);
     _ASSERTE((pCordb == NULL) == FAILED(hr));
 
 Exit:
diff --git a/src/dlls/dbgshim/dbgshim.h b/src/dlls/dbgshim/dbgshim.h
index 777c706ade..018058bbe0 100644
--- a/src/dlls/dbgshim/dbgshim.h
+++ b/src/dlls/dbgshim/dbgshim.h
@@ -35,6 +35,14 @@ RegisterForRuntimeStartup(
     __out PVOID *ppUnregisterToken);
 
 EXTERN_C HRESULT
+RegisterForRuntimeStartupEx(
+    __in DWORD dwProcessId,
+    __in LPCWSTR szApplicationGroupId,
+    __in PSTARTUP_CALLBACK pfnCallback,
+    __in PVOID parameter,
+    __out PVOID *ppUnregisterToken);
+
+EXTERN_C HRESULT
 UnregisterForRuntimeStartup(
     __in PVOID pUnregisterToken);
 
@@ -70,6 +78,13 @@ CreateDebuggingInterfaceFromVersionEx(
     __out IUnknown ** ppCordb);
 
 EXTERN_C HRESULT 
+CreateDebuggingInterfaceFromVersion2(
+    __in int iDebuggerVersion,
+    __in LPCWSTR szDebuggeeVersion,
+    __in LPCWSTR szApplicationGroupId,
+    __out IUnknown ** ppCordb);
+
+EXTERN_C HRESULT 
 CreateDebuggingInterfaceFromVersion(
     __in LPCWSTR szDebuggeeVersion, 
     __out IUnknown ** ppCordb);
diff --git a/src/dlls/dbgshim/dbgshim.ntdef b/src/dlls/dbgshim/dbgshim.ntdef
index 6915beebf9..1376cbcfe8 100644
--- a/src/dlls/dbgshim/dbgshim.ntdef
+++ b/src/dlls/dbgshim/dbgshim.ntdef
@@ -7,6 +7,7 @@ EXPORTS
         ResumeProcess
         CloseResumeHandle
         RegisterForRuntimeStartup
+	RegisterForRuntimeStartupEx
         UnregisterForRuntimeStartup
 	GetStartupNotificationEvent
 	EnumerateCLRs
@@ -14,4 +15,5 @@ EXPORTS
 	CreateVersionStringFromModule
 	CreateDebuggingInterfaceFromVersion
 	CreateDebuggingInterfaceFromVersionEx
+	CreateDebuggingInterfaceFromVersion2
 	CLRCreateInstance
diff --git a/src/dlls/dbgshim/dbgshim_unixexports.src b/src/dlls/dbgshim/dbgshim_unixexports.src
index b1cc36bc2f..013b739e6a 100644
--- a/src/dlls/dbgshim/dbgshim_unixexports.src
+++ b/src/dlls/dbgshim/dbgshim_unixexports.src
@@ -6,6 +6,7 @@ CreateProcessForLaunch
 ResumeProcess
 CloseResumeHandle
 RegisterForRuntimeStartup
+RegisterForRuntimeStartupEx
 UnregisterForRuntimeStartup
 GetStartupNotificationEvent
 EnumerateCLRs
@@ -13,4 +14,5 @@ CloseCLREnumeration
 CreateVersionStringFromModule
 CreateDebuggingInterfaceFromVersion
 CreateDebuggingInterfaceFromVersionEx
+CreateDebuggingInterfaceFromVersion2
 CLRCreateInstance
diff --git a/src/dlls/mscordbi/mscordbi.src b/src/dlls/mscordbi/mscordbi.src
index 0e14701243..9c3f928310 100644
--- a/src/dlls/mscordbi/mscordbi.src
+++ b/src/dlls/mscordbi/mscordbi.src
@@ -20,6 +20,7 @@ EXPORTS
     OpenVirtualProcess2
 
     CoreCLRCreateCordbObject private
+    CoreCLRCreateCordbObjectEx private
 
 #if defined(FEATURE_DBGIPC_TRANSPORT_DI)
     DllGetClassObject private
diff --git a/src/dlls/mscordbi/mscordbi_unixexports.src b/src/dlls/mscordbi/mscordbi_unixexports.src
index 88fa9cdab1..aeb92368a0 100644
--- a/src/dlls/mscordbi/mscordbi_unixexports.src
+++ b/src/dlls/mscordbi/mscordbi_unixexports.src
@@ -8,6 +8,7 @@ DllGetClassObject
 
 ; CoreClr API
 CoreCLRCreateCordbObject
+CoreCLRCreateCordbObjectEx
 
 ; Out-of-proc creation path from the shim - ICLRDebugging
 OpenVirtualProcessImpl