[Local GC] Scaffolding for loading a standalone GC (#11242)

* Configure the build system to build a CoreCLR capable of loading a standalone GC

* Proto-implementation of dynamic GC loading

* Build the GC with the VM's CMakeLists when doing a non-standalone build of the GC

* [Local GC] Introduce a new feature define, FEATURE_STANDALONE_GC_ONLY, to be used by the CI to explicitly test local GC dynamic loading code paths

* Fix the FEATURE_STANDALONE_GC_ONLY build for unix linkers

* Rebase against master

* Code review feedback: use the existing Unix exports file

1 files changed, 106 insertions, 17 deletions

diff --git a/src/vm/ceemain.cpp b/src/vm/ceemain.cpp
index de6059a6ec..617b022516 100644
--- a/src/vm/ceemain.cpp
+++ b/src/vm/ceemain.cpp
@@ -2440,6 +2440,101 @@ BOOL ExecuteDLL_ReturnOrThrow(HRESULT hr, BOOL fFromThunk)
 // Initialize the Garbage Collector
 //
 
+// Prototype for the function that initialzes the garbage collector.
+// Should only be called once: here, during EE startup.
+// Returns true if the initialization was successful, false otherwise.
+//
+// When using a standalone GC, this function is loaded dynamically using
+// GetProcAddress.
+extern "C" bool InitializeGarbageCollector(IGCToCLR* clrToGC, IGCHeap** gcHeap, IGCHandleManager** gcHandleManager, GcDacVars* gcDacVars);
+
+#ifdef FEATURE_STANDALONE_GC
+
+void LoadGarbageCollector()
+{
+    CONTRACTL {
+        THROWS;
+        GC_NOTRIGGER;
+        MODE_ANY;
+    } CONTRACTL_END;
+
+    TCHAR *standaloneGc = nullptr;
+    CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_GCStandaloneLocation, &standaloneGc);
+    HMODULE hMod;
+    if (!standaloneGc)
+    {
+#ifdef FEATURE_STANDALONE_GC_ONLY
+        // if the user has set GCUseStandalone but has not given us a standalone location,
+        // try and load the initialization symbol from the current module.
+        hMod = GetModuleInst();
+#else
+        ThrowHR(E_FAIL);
+#endif // FEATURE_STANDALONE_GC_ONLY
+    }
+    else
+    {
+        hMod = CLRLoadLibrary(standaloneGc);
+    }
+
+    if (!hMod)
+    {
+        ThrowHR(E_FAIL);
+    }
+
+    InitializeGarbageCollectorFunction igcf = (InitializeGarbageCollectorFunction)GetProcAddress(hMod, INITIALIZE_GC_FUNCTION_NAME);
+    if (!igcf)
+    {
+        ThrowHR(E_FAIL);
+    }
+
+    // at this point we are committing to using the standalone GC
+    // given to us.
+    IGCToCLR* gcToClr = new (nothrow) standalone::GCToEEInterface();
+    if (!gcToClr)
+    {
+        ThrowOutOfMemory();
+    }
+
+    IGCHandleManager *pGcHandleManager;
+    IGCHeap *pGCHeap;
+    if (!igcf(gcToClr, &pGCHeap, &pGcHandleManager, &g_gc_dac_vars))
+    {
+        ThrowOutOfMemory();
+    }
+
+    assert(pGCHeap != nullptr);
+    assert(pGcHandleManager != nullptr);
+    g_pGCHeap = pGCHeap;
+    g_pGCHandleManager = pGcHandleManager;
+    g_gcDacGlobals = &g_gc_dac_vars;
+}
+
+#endif // FEATURE_STANDALONE_GC
+
+void LoadStaticGarbageCollector()
+{
+    CONTRACTL{
+        THROWS;
+        GC_TRIGGERS;
+        MODE_ANY;
+    } CONTRACTL_END;
+
+    IGCHandleManager *pGcHandleManager;
+    IGCHeap *pGCHeap;
+
+    if (!InitializeGarbageCollector(nullptr, &pGCHeap, &pGcHandleManager, &g_gc_dac_vars)) 
+    {
+        ThrowOutOfMemory();
+    }
+
+    assert(pGCHeap != nullptr);
+    assert(pGcHandleManager != nullptr);
+    g_pGCHeap = pGCHeap;
+    g_pGCHandleManager = pGcHandleManager;
+    g_gcDacGlobals = &g_gc_dac_vars;
+}
+
+
 void InitializeGarbageCollector()
 {
     CONTRACTL{
@@ -2463,25 +2558,19 @@ void InitializeGarbageCollector()
     g_pFreeObjectMethodTable->SetComponentSize(1);
 
 #ifdef FEATURE_STANDALONE_GC
-    IGCToCLR* gcToClr = new (nothrow) GCToEEInterface();
-    if (!gcToClr)
-        ThrowOutOfMemory();
-#else
-    IGCToCLR* gcToClr = nullptr;
-#endif
-
-    IGCHandleManager *pGcHandleManager;
-
-    IGCHeap *pGCHeap;
-    if (!InitializeGarbageCollector(gcToClr, &pGCHeap, &pGcHandleManager, &g_gc_dac_vars)) 
+    if (CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_GCUseStandalone)
+#ifdef FEATURE_STANDALONE_GC_ONLY
+        || true
+#endif // FEATURE_STANDALONE_GC_ONLY
+        )
     {
-        ThrowOutOfMemory();
+        LoadGarbageCollector();
+    }
+    else
+#endif // FEATURE_STANDALONE_GC
+    {
+        LoadStaticGarbageCollector();
     }
-
-    assert(pGCHeap != nullptr);
-    g_pGCHeap = pGCHeap;
-    g_pGCHandleManager = pGcHandleManager;
-    g_gcDacGlobals = &g_gc_dac_vars;
 
     // Apparently the Windows linker removes global variables if they are never
     // read from, which is a problem for g_gcDacGlobals since it's expected that