[Local GC] Ensure that handle creation returns null on failure instead of throwing (#11092)

* [Local GC] Ensure that handle creation returns null on failure instead of throwing

* Fix some clang pedantry about jumping past variable initialization

* Throw OOM if initialization of handle store fails

* Perform the handle OOM check in each handle helper

5 files changed, 47 insertions, 34 deletions

diff --git a/src/gc/gchandletable.cpp b/src/gc/gchandletable.cpp
index 75646e848a..52fede6299 100644
--- a/src/gc/gchandletable.cpp
+++ b/src/gc/gchandletable.cpp
@@ -47,8 +47,12 @@ OBJECTHANDLE GCHandleStore::CreateDependentHandle(Object* primary, Object* secon
 {
     HHANDLETABLE handletable = _underlyingBucket.pTable[GetCurrentThreadHomeHeapNumber()];
     OBJECTHANDLE handle = ::HndCreateHandle(handletable, HNDTYPE_DEPENDENT, ObjectToOBJECTREF(primary));
-    ::SetDependentHandleSecondary(handle, ObjectToOBJECTREF(secondary));
+    if (!handle)
+    {
+        return nullptr;
+    }
 
+    ::SetDependentHandleSecondary(handle, ObjectToOBJECTREF(secondary));
     return handle;
 }
 
diff --git a/src/gc/handletable.cpp b/src/gc/handletable.cpp
index eee181959f..05137e4d68 100644
--- a/src/gc/handletable.cpp
+++ b/src/gc/handletable.cpp
@@ -285,12 +285,7 @@ OBJECTHANDLE HndCreateHandle(HHANDLETABLE hTable, uint32_t uType, OBJECTREF obje
 {
     CONTRACTL
     {
-#ifdef FEATURE_REDHAWK
-        // Redhawk returns NULL on failure.
         NOTHROW;
-#else
-        THROWS;
-#endif
         GC_NOTRIGGER;
         if (object != NULL) 
         { 
@@ -308,8 +303,7 @@ OBJECTHANDLE HndCreateHandle(HHANDLETABLE hTable, uint32_t uType, OBJECTREF obje
     if (g_pConfig->ShouldInjectFault(INJECTFAULT_HANDLETABLE))
     {
         FAULT_NOT_FATAL();
-        char *a = new char;
-        delete a;
+        return NULL;
     }
 #endif // _DEBUG && !FEATURE_REDHAWK
 
@@ -331,11 +325,7 @@ OBJECTHANDLE HndCreateHandle(HHANDLETABLE hTable, uint32_t uType, OBJECTREF obje
     // did the allocation succeed?
     if (!handle)
     {
-#ifdef FEATURE_REDHAWK
         return NULL;
-#else
-        ThrowOutOfMemory();
-#endif
     }
 
 #ifdef DEBUG_DestroyedHandleValue
diff --git a/src/gc/handletablecore.cpp b/src/gc/handletablecore.cpp
index edd0d949b2..00ab6a24b9 100644
--- a/src/gc/handletablecore.cpp
+++ b/src/gc/handletablecore.cpp
@@ -961,12 +961,12 @@ BOOL SegmentHandleAsyncPinHandles (TableSegment *pSegment)
 }
 
 // Replace an async pin handle with one from default domain
-void SegmentRelocateAsyncPinHandles (TableSegment *pSegment, HandleTable *pTargetTable)
+bool SegmentRelocateAsyncPinHandles (TableSegment *pSegment, HandleTable *pTargetTable)
 {
     CONTRACTL
     {
         GC_NOTRIGGER;
-        THROWS;
+        NOTHROW;
         MODE_COOPERATIVE;
     }
     CONTRACTL_END;
@@ -975,7 +975,7 @@ void SegmentRelocateAsyncPinHandles (TableSegment *pSegment, HandleTable *pTarge
     if (uBlock == BLOCK_INVALID)
     {
         // There is no pinning handles.
-        return;
+        return true;
     }
     for (uBlock = 0; uBlock < pSegment->bEmptyLine; uBlock ++)
     {
@@ -1005,11 +1005,19 @@ void SegmentRelocateAsyncPinHandles (TableSegment *pSegment, HandleTable *pTarge
                 BashMTForPinnedObject(ObjectToOBJECTREF(value));
 
                 overlapped->m_pinSelf = HndCreateHandle((HHANDLETABLE)pTargetTable, HNDTYPE_ASYNCPINNED, ObjectToOBJECTREF(value));
+                if (!overlapped->m_pinSelf)
+                {
+                    // failed to allocate a new handle - callers have to handle this.
+                    return false;
+                }
+
                 *pValue = NULL;
             }
             pValue ++;
         } while (pValue != pLast);
     }
+
+    return true;
 }
 
 // Mark all non-pending AsyncPinHandle ready for cleanup.
@@ -1068,6 +1076,7 @@ void TableRelocateAsyncPinHandles(HandleTable *pTable, HandleTable *pTargetTable
 
     BOOL fGotException = FALSE;
     TableSegment *pSegment = pTable->pSegmentList;
+    bool wasSuccessful = true;
     
 #ifdef _DEBUG
     // on debug builds, execute the OOM path 10% of the time.
@@ -1076,21 +1085,18 @@ void TableRelocateAsyncPinHandles(HandleTable *pTable, HandleTable *pTargetTable
 #endif
 
     // Step 1: replace pinning handles with ones from default domain
-    EX_TRY
+    while (pSegment)
     {
-        while (pSegment)
+        wasSuccessful = wasSuccessful && SegmentRelocateAsyncPinHandles (pSegment, pTargetTable);
+        if (!wasSuccessful)
         {
-            SegmentRelocateAsyncPinHandles (pSegment, pTargetTable);
-            pSegment = pSegment->pNextSegment;
+            break;
         }
+
+        pSegment = pSegment->pNextSegment;
     }
-    EX_CATCH
-    {
-        fGotException = TRUE;
-    }
-    EX_END_CATCH(SwallowAllExceptions);
 
-    if (!fGotException)
+    if (wasSuccessful)
     {
         return;
     }
@@ -2720,9 +2726,8 @@ void TableFreeBulkUnpreparedHandles(HandleTable *pTable, uint32_t uType, const O
 {
     CONTRACTL
     {
-        THROWS;
+        NOTHROW;
         WRAPPER(GC_TRIGGERS);
-        INJECT_FAULT(COMPlusThrowOM());
     }
     CONTRACTL_END;
 
diff --git a/src/gc/handletablescan.cpp b/src/gc/handletablescan.cpp
index 065ba0d58b..967aca5095 100644
--- a/src/gc/handletablescan.cpp
+++ b/src/gc/handletablescan.cpp
@@ -1423,7 +1423,7 @@ PTR_TableSegment CALLBACK StandardSegmentIterator(PTR_HandleTable pTable, PTR_Ta
 {
     CONTRACTL
     {
-        WRAPPER(THROWS);
+        WRAPPER(NOTHROW);
         WRAPPER(GC_TRIGGERS);
         FORBID_FAULT;
         SUPPORTS_DAC;
diff --git a/src/gc/objecthandle.cpp b/src/gc/objecthandle.cpp
index cd64ae23f9..dd43ec23d5 100644
--- a/src/gc/objecthandle.cpp
+++ b/src/gc/objecthandle.cpp
@@ -740,9 +740,9 @@ bool Ref_InitializeHandleTableBucket(HandleTableBucket* bucket, void* context)
 {
     CONTRACTL
     {
-        THROWS;
+        NOTHROW;
         WRAPPER(GC_TRIGGERS);
-        INJECT_FAULT(COMPlusThrowOM());
+        INJECT_FAULT(return false);
     }
     CONTRACTL_END;
 
@@ -759,13 +759,18 @@ bool Ref_InitializeHandleTableBucket(HandleTableBucket* bucket, void* context)
 
     HandleTableBucketHolder bucketHolder(result, n_slots);
 
-    result->pTable = new HHANDLETABLE[n_slots];
+    result->pTable = new (nothrow) HHANDLETABLE[n_slots];
+    if (!result->pTable)
+    {
+        return false;
+    }
+
     ZeroMemory(result->pTable, n_slots * sizeof(HHANDLETABLE));
 
     for (int uCPUindex=0; uCPUindex < n_slots; uCPUindex++) {
         result->pTable[uCPUindex] = HndCreateHandleTable(s_rgTypeFlags, _countof(s_rgTypeFlags), ADIndex((DWORD)(uintptr_t)context));
         if (!result->pTable[uCPUindex])
-            COMPlusThrowOM();
+            return false;
     }
 
     for (;;) {
@@ -792,9 +797,18 @@ bool Ref_InitializeHandleTableBucket(HandleTableBucket* bucket, void* context)
         // No free slot.
         // Let's create a new node
         NewHolder<HandleTableMap> newMap;
-        newMap = new HandleTableMap;
+        newMap = new (nothrow) HandleTableMap;
+        if (!newMap)
+        {
+            return false;
+        }
+
+        newMap->pBuckets = new (nothrow) HandleTableBucket * [ INITIAL_HANDLE_TABLE_ARRAY_SIZE ];
+        if (!newMap->pBuckets)
+        {
+            return false;
+        }
 
-        newMap->pBuckets = new HandleTableBucket * [ INITIAL_HANDLE_TABLE_ARRAY_SIZE ];
         newMap.SuppressRelease();
 
         newMap->dwMaxIndex = last->dwMaxIndex + INITIAL_HANDLE_TABLE_ARRAY_SIZE;