[Local GC] Move knowledge of overlapped I/O objects to the EE through four callbacks (#14982)

* [Local GC] Move knowledge of overlapped I/O objects to the EE through
four callbacks

* Code review feedback:
        1. Rename OverlappedData->AsyncPinned in interface methods
        2. Remove additional FEATURE_REDHAWK defines around async pin
        relocation code

* Eliminate two GCToEEInterface callbacks by passing the callbacks directly as arguments to a method on IGCHandleStore

* Repair clang build

* Split pin and async pin handle scans into two separate callbacks

* Fix the clang and non-Windows builds

20 files changed, 287 insertions, 192 deletions

diff --git a/src/gc/env/common.h b/src/gc/env/common.h
index 32c0d93577..1c2f75c9d6 100644
--- a/src/gc/env/common.h
+++ b/src/gc/env/common.h
@@ -16,6 +16,7 @@
 #include <stdint.h>
 #include <stddef.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 #include <wchar.h>
 #include <assert.h>
diff --git a/src/gc/env/gcenv.base.h b/src/gc/env/gcenv.base.h
index 734b46fd3d..e1d40d6003 100644
--- a/src/gc/env/gcenv.base.h
+++ b/src/gc/env/gcenv.base.h
@@ -320,6 +320,11 @@ inline void* ALIGN_DOWN(void* ptr, size_t alignment)
     return reinterpret_cast<void*>(ALIGN_DOWN(as_size_t, alignment));
 }
 
+inline int GetRandomInt(int max)
+{
+    return rand() % max;
+}
+
 typedef struct _PROCESSOR_NUMBER {
     uint16_t Group;
     uint8_t Number;
diff --git a/src/gc/env/gcenv.ee.h b/src/gc/env/gcenv.ee.h
index d747a5bbef..44828b7551 100644
--- a/src/gc/env/gcenv.ee.h
+++ b/src/gc/env/gcenv.ee.h
@@ -80,6 +80,8 @@ public:
     static bool IsGCThread();
     static bool WasCurrentThreadCreatedByGC();
     static bool CreateThread(void (*threadStart)(void*), void* arg, bool is_suspendable, const char* name);
+    static void WalkAsyncPinnedForPromotion(Object* object, ScanContext* sc, promote_func* callback);
+    static void WalkAsyncPinned(Object* object, void* context, void(*callback)(Object*, Object*, void*));
 };
 
 #endif // __GCENV_EE_H__
diff --git a/src/gc/gcenv.ee.standalone.inl b/src/gc/gcenv.ee.standalone.inl
index a9e45c953b..c114b33e9f 100644
--- a/src/gc/gcenv.ee.standalone.inl
+++ b/src/gc/gcenv.ee.standalone.inl
@@ -258,5 +258,16 @@ inline bool GCToEEInterface::CreateThread(void (*threadStart)(void*), void* arg,
     return g_theGCToCLR->CreateThread(threadStart, arg, is_suspendable, name);
 }
 
+inline void GCToEEInterface::WalkAsyncPinnedForPromotion(Object* object, ScanContext* sc, promote_func* callback)
+{
+    assert(g_theGCToCLR != nullptr);
+    return g_theGCToCLR->WalkAsyncPinnedForPromotion(object, sc, callback);
+}
+
+inline void GCToEEInterface::WalkAsyncPinned(Object* object, void* context, void(*callback)(Object*, Object*, void*))
+{
+    assert(g_theGCToCLR != nullptr);
+    return g_theGCToCLR->WalkAsyncPinned(object, context, callback);
+}
 
 #endif // __GCTOENV_EE_STANDALONE_INL__
diff --git a/src/gc/gchandletable.cpp b/src/gc/gchandletable.cpp
index 03464b29ec..7389706897 100644
--- a/src/gc/gchandletable.cpp
+++ b/src/gc/gchandletable.cpp
@@ -57,11 +57,11 @@ OBJECTHANDLE GCHandleStore::CreateDependentHandle(Object* primary, Object* secon
     return handle;
 }
 
-void GCHandleStore::RelocateAsyncPinnedHandles(IGCHandleStore* pTarget)
+void GCHandleStore::RelocateAsyncPinnedHandles(IGCHandleStore* pTarget, void (*clearIfComplete)(Object*), void (*setHandle)(Object*, OBJECTHANDLE))
 {
     // assumption - the IGCHandleStore is an instance of GCHandleStore
     GCHandleStore* other = static_cast<GCHandleStore*>(pTarget);
-    ::Ref_RelocateAsyncPinHandles(&_underlyingBucket, &other->_underlyingBucket);
+    ::Ref_RelocateAsyncPinHandles(&_underlyingBucket, &other->_underlyingBucket, clearIfComplete, setHandle);
 }
 
 bool GCHandleStore::EnumerateAsyncPinnedHandles(async_pin_enum_fn callback, void* context)
diff --git a/src/gc/gchandletableimpl.h b/src/gc/gchandletableimpl.h
index f336a3b205..77af352d4e 100644
--- a/src/gc/gchandletableimpl.h
+++ b/src/gc/gchandletableimpl.h
@@ -23,7 +23,7 @@ public:
 
     virtual OBJECTHANDLE CreateDependentHandle(Object* primary, Object* secondary);
 
-    virtual void RelocateAsyncPinnedHandles(IGCHandleStore* pTarget);
+    virtual void RelocateAsyncPinnedHandles(IGCHandleStore* pTarget, void (*clearIfCompleteCallback)(Object* object), void (*setHandle)(Object* object, OBJECTHANDLE handle));
 
     virtual bool EnumerateAsyncPinnedHandles(async_pin_enum_fn callback, void* context);
 
diff --git a/src/gc/gcinterface.ee.h b/src/gc/gcinterface.ee.h
index 84578b6d8a..113af9de3d 100644
--- a/src/gc/gcinterface.ee.h
+++ b/src/gc/gcinterface.ee.h
@@ -219,6 +219,38 @@ public:
     // or a server GC thread.
     virtual
     bool WasCurrentThreadCreatedByGC() = 0;
+
+    // Given an object, if this object is an instance of `System.Threading.OverlappedData`,
+    // and the runtime treats instances of this class specially, traverses the objects that
+    // are directly or (once) indirectly pinned by this object and reports them to the GC for
+    // the purposes of relocation and promotion.
+    //
+    // Overlapped objects are very special and as such the objects they wrap can't be promoted in
+    // the same manner as normal objects. This callback gives the EE the opportunity to hide these
+    // details, if they are implemented at all.
+    //
+    // This function is a no-op if "object" is not an OverlappedData object.
+    virtual
+    void WalkAsyncPinnedForPromotion(Object* object, ScanContext* sc, promote_func* callback) = 0;
+
+    // Given an object, if this object is an instance of `System.Threading.OverlappedData` and the
+    // runtime treats instances of this class specially, traverses the objects that are directly
+    // or once indirectly pinned by this object and invokes the given callback on them. The callback
+    // is passed the following arguments:
+    //     Object* "from" - The object that "caused" the "to" object to be pinned. If a single object
+    //                      is pinned directly by this OverlappedData, this object will be the
+    //                      OverlappedData object itself. If an array is pinned by this OverlappedData,
+    //                      this object will be the pinned array.
+    //     Object* "to"   - The object that is pinned by the "from" object. If a single object is pinned
+    //                      by an OverlappedData, "to" will be that single object. If an array is pinned
+    //                      by an OverlappedData, the callback will be invoked on all elements of that
+    //                      array and each element will be a "to" object.
+    //     void* "context" - Passed verbatim from "WalkOverlappedObject" to the callback function.
+    // The "context" argument will be passed directly to the callback without modification or inspection.
+    //
+    // This function is a no-op if "object" is not an OverlappedData object.
+    virtual
+    void WalkAsyncPinned(Object* object, void* context, void(*callback)(Object*, Object*, void*)) = 0;
 };
 
 #endif // _GCINTERFACE_EE_H_
diff --git a/src/gc/gcinterface.h b/src/gc/gcinterface.h
index b1d6b8090c..138251cb2e 100644
--- a/src/gc/gcinterface.h
+++ b/src/gc/gcinterface.h
@@ -111,6 +111,17 @@ struct WriteBarrierParameters
     uint8_t* write_watch_table;
 };
 
+// Opaque type for tracking object pointers
+#ifndef DACCESS_COMPILE
+struct OBJECTHANDLE__
+{
+    void* unused;
+};
+typedef struct OBJECTHANDLE__* OBJECTHANDLE;
+#else
+typedef uintptr_t OBJECTHANDLE;
+#endif
+
  /*
   * Scanning callback.
   */
@@ -393,16 +404,7 @@ typedef void (* fq_scan_fn)(Object** ppObject, ScanContext *pSC, uint32_t dwFlag
 typedef void (* handle_scan_fn)(Object** pRef, Object* pSec, uint32_t flags, ScanContext* context, bool isDependent);
 typedef bool (* async_pin_enum_fn)(Object* object, void* context);
 
-// Opaque type for tracking object pointers
-#ifndef DACCESS_COMPILE
-struct OBJECTHANDLE__
-{
-    void* unused;
-};
-typedef struct OBJECTHANDLE__* OBJECTHANDLE;
-#else
-typedef uintptr_t OBJECTHANDLE;
-#endif
+
 
 class IGCHandleStore {
 public:
@@ -419,7 +421,14 @@ public:
 
     virtual OBJECTHANDLE CreateDependentHandle(Object* primary, Object* secondary) = 0;
 
-    virtual void RelocateAsyncPinnedHandles(IGCHandleStore* pTarget) = 0;
+    // Relocates async pinned handles from a condemned handle store to the default domain's handle store.
+    //
+    // The two callbacks are called when:
+    //   1. clearIfComplete is called whenever the handle table observes an async pin that is still live.
+    //      The callback gives a chance for the EE to unpin the referents if the overlapped operation is complete.
+    //   2. setHandle is called whenever the GC has relocated the async pin to a new handle table. The passed-in
+    //      handle is the newly-allocated handle in the default domain that should be assigned to the overlapped object.
+    virtual void RelocateAsyncPinnedHandles(IGCHandleStore* pTarget, void (*clearIfComplete)(Object*), void (*setHandle)(Object*, OBJECTHANDLE)) = 0;
 
     virtual bool EnumerateAsyncPinnedHandles(async_pin_enum_fn callback, void* context) = 0;
 
diff --git a/src/gc/handletable.cpp b/src/gc/handletable.cpp
index 64d51d17d0..48b763d9e8 100644
--- a/src/gc/handletable.cpp
+++ b/src/gc/handletable.cpp
@@ -20,10 +20,6 @@
 #include "objecthandle.h"
 #include "handletablepriv.h"
 
-#ifndef FEATURE_REDHAWK
-#include "nativeoverlapped.h"
-#endif
-
 /****************************************************************************
  *
  * FORWARD DECLARATIONS
@@ -626,34 +622,31 @@ void HndLogSetEvent(OBJECTHANDLE handle, _UNCHECKED_OBJECTREF value)
         FireEtwSetGCHandle((void*) handle, value, hndType, generation, (int64_t) pAppDomain, GetClrInstanceId());
         FireEtwPrvSetGCHandle((void*) handle, value, hndType, generation, (int64_t) pAppDomain, GetClrInstanceId());
 
-#ifndef FEATURE_REDHAWK
         // Also fire the things pinned by Async pinned handles
         if (hndType == HNDTYPE_ASYNCPINNED)
         {
-            if (value->GetMethodTable() == g_pOverlappedDataClass)
+            // the closure passed to "WalkOverlappedObject" is not permitted to implicitly
+            // capture any variables in this scope, since WalkForOverlappedObject takes a bare
+            // function pointer and context pointer as arguments. We can still /explicitly/
+            // close over values in this scope by doing what the compiler would do and introduce
+            // a structure that contains all of the things we closed over, while passing a pointer
+            // to this structure as our closure's context pointer.
+            struct ClosureCapture
             {
-                OverlappedDataObject* overlapped = (OverlappedDataObject*) value;
-                if (overlapped->m_isArray)
-                {
-                    ArrayBase* pUserObject = (ArrayBase*)OBJECTREFToObject(overlapped->m_userObject);
-                    Object **ppObj = (Object**)pUserObject->GetDataPtr(TRUE);
-                    size_t num = pUserObject->GetNumComponents();
-                    for (size_t i = 0; i < num; i ++)
-                    {
-                        value = ppObj[i];
-                        uint32_t generation = value != 0 ? g_theGCHeap->WhichGeneration(value) : 0;
-                        FireEtwSetGCHandle(overlapped, value, HNDTYPE_PINNED, generation, (int64_t) pAppDomain, GetClrInstanceId());
-                    }
-                }
-                else
-                {
-                    value = OBJECTREF_TO_UNCHECKED_OBJECTREF(overlapped->m_userObject);
-                    uint32_t generation = value != 0 ? g_theGCHeap->WhichGeneration(value) : 0;
-                    FireEtwSetGCHandle(overlapped, value, HNDTYPE_PINNED, generation, (int64_t) pAppDomain, GetClrInstanceId());
-                }
-            }
+                AppDomain* pAppDomain;
+                Object* overlapped;
+            };
+
+            ClosureCapture captured;
+            captured.pAppDomain = pAppDomain;
+            captured.overlapped = value;
+            GCToEEInterface::WalkAsyncPinned(value, &captured, [](Object*, Object* to, void* ctx)
+            {
+                ClosureCapture* captured = reinterpret_cast<ClosureCapture*>(ctx);
+                uint32_t generation = to != nullptr ? g_theGCHeap->WhichGeneration(to) : 0;
+                FireEtwSetGCHandle(captured->overlapped, to, HNDTYPE_PINNED, generation, (int64_t) captured->pAppDomain, GetClrInstanceId());
+            });
         }
-#endif // FEATURE_REDHAWK
     }
 #else
     UNREFERENCED_PARAMETER(handle);
@@ -709,14 +702,12 @@ void HndWriteBarrier(OBJECTHANDLE handle, OBJECTREF objref)
         int generation = g_theGCHeap->WhichGeneration(value);
         uint32_t uType = HandleFetchType(handle);
 
-#ifndef FEATURE_REDHAWK
         //OverlappedData need special treatment: because all user data pointed by it needs to be reported by this handle,
         //its age is consider to be min age of the user data, to be simple, we just make it 0
-        if (uType == HNDTYPE_ASYNCPINNED && objref->GetGCSafeMethodTable () == g_pOverlappedDataClass)
+        if (uType == HNDTYPE_ASYNCPINNED)
         {
             generation = 0;
         }
-#endif // !FEATURE_REDHAWK
         
         if (uType == HNDTYPE_DEPENDENT)
         {
@@ -1165,7 +1156,6 @@ uint32_t HndCountAllHandles(BOOL fUseLocks)
 
 BOOL  Ref_HandleAsyncPinHandles(async_pin_enum_fn asyncPinCallback, void* context)
 {
-#ifndef FEATURE_REDHAWK
     CONTRACTL
     {
         NOTHROW;
@@ -1186,14 +1176,13 @@ BOOL  Ref_HandleAsyncPinHandles(async_pin_enum_fn asyncPinCallback, void* contex
     }
 
     return result;
-#else
-    return true;
-#endif // !FEATURE_REDHAWK
 }
 
-void  Ref_RelocateAsyncPinHandles(HandleTableBucket *pSource, HandleTableBucket *pTarget)
+void  Ref_RelocateAsyncPinHandles(HandleTableBucket *pSource,
+    HandleTableBucket *pTarget,
+    void (*clearIfComplete)(Object* object),
+    void (*setHandle)(Object* object, OBJECTHANDLE handle))
 {
-#ifndef FEATURE_REDHAWK
     CONTRACTL
     {
         NOTHROW;
@@ -1204,9 +1193,8 @@ void  Ref_RelocateAsyncPinHandles(HandleTableBucket *pSource, HandleTableBucket
     int limit = getNumberOfSlots();
     for (int n = 0; n < limit; n ++ )
     {
-        TableRelocateAsyncPinHandles(Table(pSource->pTable[n]), Table(pTarget->pTable[n]));
+        TableRelocateAsyncPinHandles(Table(pSource->pTable[n]), Table(pTarget->pTable[n]), clearIfComplete, setHandle);
     }
-#endif // !FEATURE_REDHAWK
 }
 
 /*--------------------------------------------------------------------------*/
diff --git a/src/gc/handletablecore.cpp b/src/gc/handletablecore.cpp
index 2a69afc01d..8fbdbe9719 100644
--- a/src/gc/handletablecore.cpp
+++ b/src/gc/handletablecore.cpp
@@ -16,11 +16,6 @@
 #include "gcenv.h"
 #include "gcenv.inl"
 #include "gc.h"
-
-#ifndef FEATURE_REDHAWK
-#include "nativeoverlapped.h"
-#endif // FEATURE_REDHAWK
-
 #include "handletablepriv.h"
 
 /****************************************************************************
@@ -666,10 +661,9 @@ __inline void SegmentUnMarkFreeMask(TableSegment *pSegment, _UNCHECKED_OBJECTREF
     pSegment->rgFreeMask[uMask] &= ~(1<<uBit);
 }
 
-#ifndef FEATURE_REDHAWK
 // Prepare a segment to be moved to default domain.
 // Remove all non-async pin handles.
-void SegmentPreCompactAsyncPinHandles(TableSegment *pSegment)
+void SegmentPreCompactAsyncPinHandles(TableSegment *pSegment, void (*clearIfComplete)(Object*))
 {
     CONTRACTL
     {
@@ -761,14 +755,7 @@ void SegmentPreCompactAsyncPinHandles(TableSegment *pSegment)
             _UNCHECKED_OBJECTREF value = *pValue;
             if (!HndIsNullOrDestroyedHandle(value))
             {
-                _ASSERTE (value->GetMethodTable() == g_pOverlappedDataClass);
-                OVERLAPPEDDATAREF overlapped = (OVERLAPPEDDATAREF)(ObjectToOBJECTREF((Object*)(value)));
-                if (overlapped->HasCompleted())
-                {
-                    // IO has finished.  We don't need to pin the user buffer any longer.
-                    overlapped->m_userObject = NULL;
-                }
-                BashMTForPinnedObject(ObjectToOBJECTREF(value));
+                clearIfComplete((Object*)value);
             }
             else
             {
@@ -842,7 +829,7 @@ BOOL SegmentCopyAsyncPinHandle(TableSegment *pSegment, _UNCHECKED_OBJECTREF *h)
     return TRUE;
 }
 
-void SegmentCompactAsyncPinHandles(TableSegment *pSegment, TableSegment **ppWorkerSegment)
+void SegmentCompactAsyncPinHandles(TableSegment *pSegment, TableSegment **ppWorkerSegment, void (*clearIfComplete)(Object*))
 {
     CONTRACTL
     {
@@ -876,14 +863,7 @@ void SegmentCompactAsyncPinHandles(TableSegment *pSegment, TableSegment **ppWork
             _UNCHECKED_OBJECTREF value = *pValue;
             if (!HndIsNullOrDestroyedHandle(value))
             {
-                _ASSERTE (value->GetMethodTable() == g_pOverlappedDataClass);
-                OVERLAPPEDDATAREF overlapped = (OVERLAPPEDDATAREF)(ObjectToOBJECTREF((Object*)value));
-                if (overlapped->HasCompleted())
-                {
-                    // IO has finished.  We don't need to pin the user buffer any longer.
-                    overlapped->m_userObject = NULL;
-                }
-                BashMTForPinnedObject(ObjectToOBJECTREF(value));
+                clearIfComplete((Object*)value);
                 fNeedNewSegment = !SegmentCopyAsyncPinHandle(*ppWorkerSegment,pValue);
             }
             if (fNeedNewSegment)
@@ -891,7 +871,7 @@ void SegmentCompactAsyncPinHandles(TableSegment *pSegment, TableSegment **ppWork
                 _ASSERTE ((*ppWorkerSegment)->rgFreeCount[HNDTYPE_ASYNCPINNED] == 0 &&
                           (*ppWorkerSegment)->bFreeList == BLOCK_INVALID);
                 TableSegment *pNextSegment = (*ppWorkerSegment)->pNextSegment;
-                SegmentPreCompactAsyncPinHandles(pNextSegment);
+                SegmentPreCompactAsyncPinHandles(pNextSegment, clearIfComplete);
                 *ppWorkerSegment = pNextSegment;
                 if (pNextSegment == pSegment)
                 {
@@ -961,7 +941,10 @@ BOOL SegmentHandleAsyncPinHandles (TableSegment *pSegment, const AsyncPinCallbac
 }
 
 // Replace an async pin handle with one from default domain
-bool SegmentRelocateAsyncPinHandles (TableSegment *pSegment, HandleTable *pTargetTable)
+bool SegmentRelocateAsyncPinHandles (TableSegment *pSegment,
+    HandleTable *pTargetTable,
+    void (*clearIfComplete)(Object*),
+    void (*setHandle)(Object*, OBJECTHANDLE))
 {
     CONTRACTL
     {
@@ -995,22 +978,15 @@ bool SegmentRelocateAsyncPinHandles (TableSegment *pSegment, HandleTable *pTarge
             _UNCHECKED_OBJECTREF value = *pValue;
             if (!HndIsNullOrDestroyedHandle(value))
             {
-                _ASSERTE (value->GetMethodTable() == g_pOverlappedDataClass);
-                OVERLAPPEDDATAREF overlapped = (OVERLAPPEDDATAREF)(ObjectToOBJECTREF((Object*)value));
-                if (overlapped->HasCompleted())
-                {
-                    // IO has finished.  We don't need to pin the user buffer any longer.
-                    overlapped->m_userObject = NULL;
-                }
-                BashMTForPinnedObject(ObjectToOBJECTREF(value));
-
-                overlapped->m_pinSelf = HndCreateHandle((HHANDLETABLE)pTargetTable, HNDTYPE_ASYNCPINNED, ObjectToOBJECTREF(value));
-                if (!overlapped->m_pinSelf)
+                clearIfComplete((Object*)value);
+                OBJECTHANDLE selfHandle = HndCreateHandle((HHANDLETABLE)pTargetTable, HNDTYPE_ASYNCPINNED, ObjectToOBJECTREF(value));
+                if (!selfHandle)
                 {
                     // failed to allocate a new handle - callers have to handle this.
                     return false;
                 }
 
+                setHandle((Object*)value, selfHandle);
                 *pValue = NULL;
             }
             pValue ++;
@@ -1034,8 +1010,6 @@ BOOL TableHandleAsyncPinHandles(HandleTable *pTable, const AsyncPinCallbackConte
     }
     CONTRACTL_END;
 
-    _ASSERTE (pTable->uADIndex.m_dwIndex == DefaultADID);
-
     BOOL result = FALSE;
     TableSegment *pSegment = pTable->pSegmentList;
 
@@ -1062,7 +1036,10 @@ BOOL TableHandleAsyncPinHandles(HandleTable *pTable, const AsyncPinCallbackConte
 //       from a again.
 //    c. After copying all handles to worker segments, move the segments to default domain.
 // It is very important that in step 2, we should not fail for OOM, which means no memory allocation.
-void TableRelocateAsyncPinHandles(HandleTable *pTable, HandleTable *pTargetTable)
+void TableRelocateAsyncPinHandles(HandleTable *pTable,
+    HandleTable *pTargetTable,
+    void (*clearIfComplete)(Object*),
+    void (*setHandle)(Object*, OBJECTHANDLE))
 {
     CONTRACTL
     {
@@ -1087,7 +1064,7 @@ void TableRelocateAsyncPinHandles(HandleTable *pTable, HandleTable *pTargetTable
     // Step 1: replace pinning handles with ones from default domain
     while (pSegment)
     {
-        wasSuccessful = wasSuccessful && SegmentRelocateAsyncPinHandles (pSegment, pTargetTable);
+        wasSuccessful = wasSuccessful && SegmentRelocateAsyncPinHandles (pSegment, pTargetTable, clearIfComplete, setHandle);
         if (!wasSuccessful)
         {
             break;
@@ -1139,12 +1116,12 @@ SLOW_PATH:
         // Compact async pinning handles into the smallest number of leading segments we can (the worker
         // segments).
         TableSegment *pWorkerSegment = pTable->pSegmentList;
-        SegmentPreCompactAsyncPinHandles (pWorkerSegment);
+        SegmentPreCompactAsyncPinHandles (pWorkerSegment, clearIfComplete);
 
         pSegment = pWorkerSegment->pNextSegment;
         while (pSegment)
         {
-            SegmentCompactAsyncPinHandles (pSegment, &pWorkerSegment);
+            SegmentCompactAsyncPinHandles (pSegment, &pWorkerSegment, clearIfComplete);
             pSegment= pSegment->pNextSegment;
         }
 
@@ -1193,7 +1170,6 @@ SLOW_PATH:
         break;
     }
 }
-#endif // !FEATURE_REDHAWK
 
 /*
  * Check if a handle is part of a HandleTable
diff --git a/src/gc/handletablepriv.h b/src/gc/handletablepriv.h
index f33a547a23..e0ed4b80e3 100644
--- a/src/gc/handletablepriv.h
+++ b/src/gc/handletablepriv.h
@@ -795,7 +795,7 @@ BOOL TableHandleAsyncPinHandles(HandleTable *pTable, const AsyncPinCallbackConte
  * Replaces async pin handles with ones in default domain.
  *
  */
-void TableRelocateAsyncPinHandles(HandleTable *pTable, HandleTable *pTargetTable);
+void TableRelocateAsyncPinHandles(HandleTable *pTable, HandleTable *pTargetTable, void (*clearIfComplete)(Object*), void (*setHandle)(Object*, OBJECTHANDLE));
 
 /*
  * Check if a handle is part of a HandleTable
diff --git a/src/gc/handletablescan.cpp b/src/gc/handletablescan.cpp
index b071f33729..fb08d37f35 100644
--- a/src/gc/handletablescan.cpp
+++ b/src/gc/handletablescan.cpp
@@ -20,10 +20,6 @@
 #include "objecthandle.h"
 #include "handletablepriv.h"
 
-#ifndef FEATURE_REDHAWK
-#include "nativeoverlapped.h"
-#endif // FEATURE_REDHAWK
-
 
 /****************************************************************************
  *
@@ -822,33 +818,17 @@ void BlockResetAgeMapForBlocksWorker(uint32_t *pdwGen, uint32_t dwClumpMask, Sca
                     if (minAge > thisAge)
                         minAge = thisAge;
 
-#ifndef FEATURE_REDHAWK
-                    if ((*pValue)->GetGCSafeMethodTable() == g_pOverlappedDataClass)
-                    {
-                        // reporting the pinned user objects
-                        OverlappedDataObject *pOverlapped = (OverlappedDataObject *)(*pValue);
-                        if (pOverlapped->m_userObject != NULL)
+                    GCToEEInterface::WalkAsyncPinned(*pValue, &minAge,
+                        [](Object*, Object* to, void* ctx)
                         {
-                            Object * pUserObject = OBJECTREFToObject(pOverlapped->m_userObject);
-                            thisAge = g_theGCHeap->WhichGeneration(pUserObject);
-                            if (minAge > thisAge)
-                                minAge = thisAge;
-                            if (pOverlapped->m_isArray)
+                            int* minAge = reinterpret_cast<int*>(ctx);
+                            int generation = g_theGCHeap->WhichGeneration(to);
+                            if (*minAge > generation)
                             {
-                                ArrayBase* pUserArrayObject = (ArrayBase*)pUserObject;
-                                Object **pObj = (Object**)pUserArrayObject->GetDataPtr(TRUE);
-                                size_t num = pUserArrayObject->GetNumComponents();
-                                for (size_t i = 0; i < num; i ++)
-                                {
-                                     thisAge = g_theGCHeap->WhichGeneration(pObj[i]);
-                                     if (minAge > thisAge)
-                                         minAge = thisAge;
-                                 }                                    
-                            }                            
-                        }
-                    }
-#endif // !FEATURE_REDHAWK                    
-                }
+                                *minAge = generation;
+                            }
+                        });
+               }
             }
             _ASSERTE(FitsInU1(minAge));
             ((uint8_t *)pSegment->rgGeneration)[uClump] = static_cast<uint8_t>(minAge);
@@ -920,9 +900,8 @@ static void VerifyObject(_UNCHECKED_OBJECTREF from, _UNCHECKED_OBJECTREF obj)
 #endif // FEATURE_REDHAWK
 }
 
-static void VerifyObjectAndAge(_UNCHECKED_OBJECTREF *pValue, _UNCHECKED_OBJECTREF from, _UNCHECKED_OBJECTREF obj, uint8_t minAge)
+static void VerifyObjectAndAge(_UNCHECKED_OBJECTREF from, _UNCHECKED_OBJECTREF obj, uint8_t minAge)
 {
-    UNREFERENCED_PARAMETER(pValue);
     VerifyObject(from, obj);
 
     int thisAge = g_theGCHeap->WhichGeneration(obj);
@@ -989,29 +968,13 @@ void BlockVerifyAgeMapForBlocksWorker(uint32_t *pdwGen, uint32_t dwClumpMask, Sc
             {
                 if (!HndIsNullOrDestroyedHandle(*pValue))
                 {
-                    VerifyObjectAndAge(pValue, (*pValue), (*pValue), minAge);
-#ifndef FEATURE_REDHAWK
-                    if ((*pValue)->GetGCSafeMethodTable() == g_pOverlappedDataClass)
-                    {
-                        // reporting the pinned user objects
-                        OverlappedDataObject *pOverlapped = (OverlappedDataObject *)(*pValue);
-                        if (pOverlapped->m_userObject != NULL)
+                    VerifyObjectAndAge((*pValue), (*pValue), minAge);
+                    GCToEEInterface::WalkAsyncPinned(*pValue, &minAge,
+                        [](Object* from, Object* object, void* age)
                         {
-                            Object * pUserObject = OBJECTREFToObject(pOverlapped->m_userObject);
-                            VerifyObjectAndAge(pValue, (*pValue), pUserObject, minAge);
-                            if (pOverlapped->m_isArray)
-                            {
-                                ArrayBase* pUserArrayObject = (ArrayBase*)pUserObject;
-                                Object **pObj = (Object**)pUserArrayObject->GetDataPtr(TRUE);
-                                size_t num = pUserArrayObject->GetNumComponents();
-                                for (size_t i = 0; i < num; i ++)
-                                {
-                                     VerifyObjectAndAge(pValue, pUserObject, pObj[i], minAge);
-                                }                                    
-                            }                            
-                        }
-                    }
-#endif // !FEATURE_REDHAWK
+                            uint8_t* minAge = reinterpret_cast<uint8_t*>(age);
+                            VerifyObjectAndAge(from, object, *minAge);
+                        });
 
                     if (uType == HNDTYPE_DEPENDENT)
                     {
diff --git a/src/gc/objecthandle.cpp b/src/gc/objecthandle.cpp
index 24db07dfb1..de30757131 100644
--- a/src/gc/objecthandle.cpp
+++ b/src/gc/objecthandle.cpp
@@ -25,9 +25,6 @@
 #ifdef FEATURE_COMINTEROP
 #include "comcallablewrapper.h"
 #endif // FEATURE_COMINTEROP
-#ifndef FEATURE_REDHAWK
-#include "nativeoverlapped.h"
-#endif // FEATURE_REDHAWK
 #endif // BUILD_AS_STANDALONE
 
 HandleTableMap g_HandleTableMap;
@@ -277,40 +274,23 @@ void CALLBACK PinObject(_UNCHECKED_OBJECTREF *pObjRef, uintptr_t *pExtraInfo, ui
     _ASSERTE(lp2);
     promote_func* callback = (promote_func*) lp2;
     callback(pRef, (ScanContext *)lp1, GC_CALL_PINNED);
+}
 
-#ifndef FEATURE_REDHAWK
-    Object * pPinnedObj = *pRef;
+void CALLBACK AsyncPinObject(_UNCHECKED_OBJECTREF *pObjRef, uintptr_t *pExtraInfo, uintptr_t lp1, uintptr_t lp2)
+{
+    UNREFERENCED_PARAMETER(pExtraInfo);
 
-    if (!HndIsNullOrDestroyedHandle(pPinnedObj) && pPinnedObj->GetGCSafeMethodTable() == g_pOverlappedDataClass)
-    {
-        // reporting the pinned user objects
-        OverlappedDataObject *pOverlapped = (OverlappedDataObject *)pPinnedObj;
-        if (pOverlapped->m_userObject != NULL)
-        {
-            //callback(OBJECTREF_TO_UNCHECKED_OBJECTREF(pOverlapped->m_userObject), (ScanContext *)lp1, GC_CALL_PINNED);
-            if (pOverlapped->m_isArray)
-            {
-                pOverlapped->m_userObjectInternal = static_cast<void*>(OBJECTREFToObject(pOverlapped->m_userObject));
-                ArrayBase* pUserObject = (ArrayBase*)OBJECTREFToObject(pOverlapped->m_userObject);
-                Object **ppObj = (Object**)pUserObject->GetDataPtr(TRUE);
-                size_t num = pUserObject->GetNumComponents();
-                for (size_t i = 0; i < num; i ++)
-                {
-                    callback(ppObj + i, (ScanContext *)lp1, GC_CALL_PINNED);
-                }
-            }
-            else
-            {
-                callback(&OBJECTREF_TO_UNCHECKED_OBJECTREF(pOverlapped->m_userObject), (ScanContext *)lp1, GC_CALL_PINNED);
-            }
-        }
+    LOG((LF_GC, LL_WARNING, LOG_HANDLE_OBJECT_CLASS("WARNING: ", pObjRef, "causes (async) pinning of ", *pObjRef)));
 
-        if (pOverlapped->GetAppDomainId() !=  DefaultADID && pOverlapped->GetAppDomainIndex().m_dwIndex == DefaultADID)
-        {
-            OverlappedDataObject::MarkCleanupNeededFromGC();
-        }
+    Object **pRef = (Object **)pObjRef;
+    _ASSERTE(lp2);
+    promote_func* callback = (promote_func*)lp2;
+    callback(pRef, (ScanContext *)lp2, GC_CALL_PINNED);
+    Object* pPinnedObj = *pRef;
+    if (!HndIsNullOrDestroyedHandle(pPinnedObj))
+    {
+        GCToEEInterface::WalkAsyncPinnedForPromotion(pPinnedObj, (ScanContext *)lp1, callback);
     }
-#endif // !FEATURE_REDHAWK
 }
 
 
@@ -424,14 +404,12 @@ void CALLBACK UpdatePointer(_UNCHECKED_OBJECTREF *pObjRef, uintptr_t *pExtraInfo
  */
 void CALLBACK ScanPointerForProfilerAndETW(_UNCHECKED_OBJECTREF *pObjRef, uintptr_t *pExtraInfo, uintptr_t lp1, uintptr_t lp2)
 {
-#ifndef FEATURE_REDHAWK
     CONTRACTL
     {
         NOTHROW;
         GC_NOTRIGGER;
     }
     CONTRACTL_END;
-#endif // FEATURE_REDHAWK
     UNREFERENCED_PARAMETER(pExtraInfo);
     handle_scan_fn fn = (handle_scan_fn)lp2;
 
@@ -1095,7 +1073,12 @@ void Ref_TracePinningRoots(uint32_t condemned, uint32_t maxgen, ScanContext* sc,
                         sc->pCurrentDomain = SystemDomain::GetAppDomainAtIndex(HndGetHandleTableADIndex(hTable));
                     }
 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
-                    HndScanHandlesForGC(hTable, PinObject, uintptr_t(sc), uintptr_t(fn), types, _countof(types), condemned, maxgen, flags);
+
+                    // Pinned handles and async pinned handles are scanned in separate passes, since async pinned
+                    // handles may require a callback into the EE in order to fully trace an async pinned
+                    // object's object graph.
+                    HndScanHandlesForGC(hTable, PinObject, uintptr_t(sc), uintptr_t(fn), &types[0], 1, condemned, maxgen, flags);
+                    HndScanHandlesForGC(hTable, AsyncPinObject, uintptr_t(sc), uintptr_t(fn), &types[1], 1, condemned, maxgen, flags);
                 }
             }
         walk = walk->pNext;
diff --git a/src/gc/objecthandle.h b/src/gc/objecthandle.h
index a6d2259009..6563d9661b 100644
--- a/src/gc/objecthandle.h
+++ b/src/gc/objecthandle.h
@@ -81,7 +81,7 @@ void Ref_Shutdown();
 HandleTableBucket* Ref_CreateHandleTableBucket(void* context);
 bool Ref_InitializeHandleTableBucket(HandleTableBucket* bucket, void* context);
 BOOL Ref_HandleAsyncPinHandles(async_pin_enum_fn callback, void* context);
-void Ref_RelocateAsyncPinHandles(HandleTableBucket *pSource, HandleTableBucket *pTarget);
+void Ref_RelocateAsyncPinHandles(HandleTableBucket *pSource, HandleTableBucket *pTarget, void (*clearIfComplete)(Object*), void (*setHandle)(Object*, OBJECTHANDLE));
 void Ref_RemoveHandleTableBucket(HandleTableBucket *pBucket);
 void Ref_DestroyHandleTableBucket(HandleTableBucket *pBucket);
 
diff --git a/src/gc/sample/gcenv.ee.cpp b/src/gc/sample/gcenv.ee.cpp
index 72ef9b5574..3d0303205e 100644
--- a/src/gc/sample/gcenv.ee.cpp
+++ b/src/gc/sample/gcenv.ee.cpp
@@ -329,3 +329,11 @@ bool GCToEEInterface::CreateThread(void (*threadStart)(void*), void* arg, bool i
 {
     return false;
 }
+
+void GCToEEInterface::WalkAsyncPinnedForPromotion(Object* object, ScanContext* sc, promote_func* callback)
+{
+}
+
+void GCToEEInterface::WalkAsyncPinned(Object* object, void* context, void (*callback)(Object*, Object*, void*))
+{
+}
diff --git a/src/vm/appdomain.cpp b/src/vm/appdomain.cpp
index 370847c9a2..07e3280342 100644
--- a/src/vm/appdomain.cpp
+++ b/src/vm/appdomain.cpp
@@ -9053,7 +9053,43 @@ void AppDomain::HandleAsyncPinHandles()
     // 4. Then we can delete all AsyncPinHandle marked with READYTOCLEAN.
     IGCHandleStore *pBucketInDefault = SystemDomain::System()->DefaultDomain()->m_handleStore;
 
-    pBucket->RelocateAsyncPinnedHandles(pBucketInDefault);
+    auto clearIfComplete = [](Object* object)
+    {
+        LIMITED_METHOD_CONTRACT;
+
+        assert(object != nullptr);
+        if (object->GetGCSafeMethodTable() != g_pOverlappedDataClass)
+        {
+            return;
+        }
+
+        OVERLAPPEDDATAREF overlapped = (OVERLAPPEDDATAREF)(ObjectToOBJECTREF((Object*)object));
+        if (overlapped->HasCompleted())
+        {
+            // IO has finished.  We don't need to pin the user buffer any longer.
+            overlapped->m_userObject = NULL;
+        }
+
+        BashMTForPinnedObject(ObjectToOBJECTREF(object));
+    };
+
+    auto setHandle = [](Object* object, OBJECTHANDLE handle)
+    {
+        LIMITED_METHOD_CONTRACT;
+
+        assert(object != nullptr);
+        assert(handle);
+
+        if (object->GetGCSafeMethodTable() != g_pOverlappedDataClass)
+        {
+            return;
+        }
+
+        OverlappedDataObject* overlapped = (OverlappedDataObject*)object;
+        overlapped->m_pinSelf = handle;
+    };
+
+    pBucket->RelocateAsyncPinnedHandles(pBucketInDefault, clearIfComplete, setHandle);
 
     OverlappedDataObject::RequestCleanup();
 }
diff --git a/src/vm/gcenv.ee.cpp b/src/vm/gcenv.ee.cpp
index e5da889a64..a880fdbca6 100644
--- a/src/vm/gcenv.ee.cpp
+++ b/src/vm/gcenv.ee.cpp
@@ -1201,7 +1201,7 @@ namespace
         };
 
         InlineSString<MaxThreadNameSize> wideName;
-        const WCHAR* namePtr;
+        const WCHAR* namePtr = nullptr;
         EX_TRY
         {
             if (name != nullptr)
@@ -1214,7 +1214,6 @@ namespace
         {
             // we're not obligated to provide a name - if it's not valid,
             // just report nullptr as the name.
-            namePtr = nullptr;
         }
         EX_END_CATCH(SwallowAllExceptions)
 
@@ -1307,3 +1306,81 @@ bool GCToEEInterface::CreateThread(void (*threadStart)(void*), void* arg, bool i
         return CreateNonSuspendableThread(threadStart, arg, name);
     }
 }
+
+void GCToEEInterface::WalkAsyncPinnedForPromotion(Object* object, ScanContext* sc, promote_func* callback)
+{
+    LIMITED_METHOD_CONTRACT;
+
+    assert(object != nullptr);
+    assert(sc != nullptr);
+    assert(callback != nullptr);
+    if (object->GetGCSafeMethodTable() != g_pOverlappedDataClass)
+    {
+        // not an overlapped data object - nothing to do.
+        return;
+    }
+
+    // reporting the pinned user objects
+    OverlappedDataObject *pOverlapped = (OverlappedDataObject *)object;
+    if (pOverlapped->m_userObject != NULL)
+    {
+        //callback(OBJECTREF_TO_UNCHECKED_OBJECTREF(pOverlapped->m_userObject), (ScanContext *)lp1, GC_CALL_PINNED);
+        if (pOverlapped->m_isArray)
+        {
+            // OverlappedDataObject is very special.  An async pin handle keeps it alive.
+            // During GC, we also make sure
+            // 1. m_userObject itself does not move if m_userObject is not array
+            // 2. Every object pointed by m_userObject does not move if m_userObject is array
+            // We do not want to pin m_userObject if it is array.  But m_userObject may be updated
+            // during relocation phase before OverlappedDataObject is doing relocation.
+            // m_userObjectInternal is used to track the location of the m_userObject before it is updated.
+            pOverlapped->m_userObjectInternal = static_cast<void*>(OBJECTREFToObject(pOverlapped->m_userObject));
+            ArrayBase* pUserObject = (ArrayBase*)OBJECTREFToObject(pOverlapped->m_userObject);
+            Object **ppObj = (Object**)pUserObject->GetDataPtr(TRUE);
+            size_t num = pUserObject->GetNumComponents();
+            for (size_t i = 0; i < num; i++)
+            {
+                callback(ppObj + i, sc, GC_CALL_PINNED);
+            }
+        }
+        else
+        {
+            callback(&OBJECTREF_TO_UNCHECKED_OBJECTREF(pOverlapped->m_userObject), (ScanContext *)sc, GC_CALL_PINNED);
+        }
+    }
+
+    if (pOverlapped->GetAppDomainId() != DefaultADID && pOverlapped->GetAppDomainIndex().m_dwIndex == DefaultADID)
+    {
+        OverlappedDataObject::MarkCleanupNeededFromGC();
+    }
+}
+
+void GCToEEInterface::WalkAsyncPinned(Object* object, void* context, void (*callback)(Object*, Object*, void*))
+{
+    LIMITED_METHOD_CONTRACT;
+
+    assert(object != nullptr);
+    assert(callback != nullptr);
+
+    if (object->GetGCSafeMethodTable() != g_pOverlappedDataClass)
+    {
+        return;
+    }
+
+    OverlappedDataObject *pOverlapped = (OverlappedDataObject *)(object);
+    if (pOverlapped->m_userObject != NULL)
+    {
+        Object * pUserObject = OBJECTREFToObject(pOverlapped->m_userObject);
+        callback(object, pUserObject, context);
+        if (pOverlapped->m_isArray)
+        {
+            ArrayBase* pUserArrayObject = (ArrayBase*)pUserObject;
+            Object **pObj = (Object**)pUserArrayObject->GetDataPtr(TRUE);
+            size_t num = pUserArrayObject->GetNumComponents();
+            for (size_t i = 0; i < num; i ++)
+            {
+                callback(pUserObject, pObj[i], context);
+            }
+        }
+    }
+}
diff --git a/src/vm/gcenv.ee.h b/src/vm/gcenv.ee.h
index b2ada36bcd..e3867b7e98 100644
--- a/src/vm/gcenv.ee.h
+++ b/src/vm/gcenv.ee.h
@@ -60,6 +60,8 @@ public:
     bool IsGCThread();
     bool WasCurrentThreadCreatedByGC();
     bool CreateThread(void (*threadStart)(void*), void* arg, bool is_suspendable, const char* name);
+    void WalkAsyncPinnedForPromotion(Object* object, ScanContext* sc, promote_func* callback);
+    void WalkAsyncPinned(Object* object, void* context, void(*callback)(Object*, Object*, void*));
 };
 
 } // namespace standalone
diff --git a/src/vm/gcenv.ee.standalone.cpp b/src/vm/gcenv.ee.standalone.cpp
index 5ba2aca812..be8ceca0c1 100644
--- a/src/vm/gcenv.ee.standalone.cpp
+++ b/src/vm/gcenv.ee.standalone.cpp
@@ -6,6 +6,7 @@
 #include "gcenv.h"
 #include "gcenv.ee.h"
 #include "threadsuspend.h"
+#include "nativeoverlapped.h"
 
 #ifdef FEATURE_COMINTEROP
 #include "runtimecallablewrapper.h"
@@ -27,4 +28,4 @@ namespace standalone
 
 #include "gcenv.ee.cpp"
 
-} // namespace standalone
\ No newline at end of file
+} // namespace standalone
diff --git a/src/vm/gcenv.ee.static.cpp b/src/vm/gcenv.ee.static.cpp
index 240e325a9e..975decadf4 100644
--- a/src/vm/gcenv.ee.static.cpp
+++ b/src/vm/gcenv.ee.static.cpp
@@ -6,6 +6,7 @@
 #include "gcenv.h"
 #include "../gc/env/gcenv.ee.h"
 #include "threadsuspend.h"
+#include "nativeoverlapped.h"
 
 #ifdef FEATURE_COMINTEROP
 #include "runtimecallablewrapper.h"
@@ -22,4 +23,4 @@ extern MethodTable* pWeakReferenceOfTCanonMT;
 // Finalizes a weak reference directly.
 extern void FinalizeWeakReference(Object* obj);
 
-#include "gcenv.ee.cpp"
\ No newline at end of file
+#include "gcenv.ee.cpp"