Move common ThreadPool code to shared CoreLib partition. (#22115)

* Move common ThreadPool code to shared CoreLib partition.

* Move numWorkingThreads counting out of shared code.

* Rename ThreadPoolGlobals.vmTpInitialized to threadPoolInitialized.
Remove exception handling from ThreadPoolWorkQueue.Dispatch.
Remove obsolete code from ThreadPoolWorkQueue.Dispatch.
Rename ThreadPool.EnsureVMInitialized to ThreadPool.EnsureInitialized.

7 files changed, 433 insertions, 397 deletions

diff --git a/src/System.Private.CoreLib/System.Private.CoreLib.csproj b/src/System.Private.CoreLib/System.Private.CoreLib.csproj
index d03b496b0f..dd607e2ae1 100644
--- a/src/System.Private.CoreLib/System.Private.CoreLib.csproj
+++ b/src/System.Private.CoreLib/System.Private.CoreLib.csproj
@@ -278,7 +278,7 @@
     <Compile Include="$(BclSourcesRoot)\System\Threading\SynchronizationContext.cs" />
     <Compile Include="$(BclSourcesRoot)\System\Threading\Tasks\TaskContinuation.cs" />
     <Compile Include="$(BclSourcesRoot)\System\Threading\Thread.cs" />
-    <Compile Include="$(BclSourcesRoot)\System\Threading\ThreadPool.cs" />
+    <Compile Include="$(BclSourcesRoot)\System\Threading\ThreadPool.CoreCLR.cs" />
     <Compile Include="$(BclSourcesRoot)\System\Threading\Timer.cs" />
     <Compile Include="$(BclSourcesRoot)\System\Threading\Volatile.cs" />
     <Compile Include="$(BclSourcesRoot)\System\Threading\WaitHandle.cs" />
diff --git a/src/System.Private.CoreLib/shared/System.Private.CoreLib.Shared.projitems b/src/System.Private.CoreLib/shared/System.Private.CoreLib.Shared.projitems
index bb2fbc9720..9ba6e27500 100644
--- a/src/System.Private.CoreLib/shared/System.Private.CoreLib.Shared.projitems
+++ b/src/System.Private.CoreLib/shared/System.Private.CoreLib.Shared.projitems
@@ -796,6 +796,7 @@
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\Tasks\Sources\IValueTaskSource.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\ThreadAbortException.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\ThreadInterruptedException.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\Threading\ThreadPool.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\ThreadPriority.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\ThreadStart.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\ThreadStartException.cs" />
diff --git a/src/System.Private.CoreLib/src/System/Threading/ThreadPool.cs b/src/System.Private.CoreLib/shared/System/Threading/ThreadPool.cs
index 518194aa74..e0447c5599 100644
--- a/src/System.Private.CoreLib/src/System/Threading/ThreadPool.cs
+++ b/src/System.Private.CoreLib/shared/System/Threading/ThreadPool.cs
@@ -21,19 +21,16 @@ using System.Runtime.ConstrainedExecution;
 using System.Runtime.InteropServices;
 using System.Threading.Tasks;
 using Internal.Runtime.CompilerServices;
-using Microsoft.Win32;
+
+using Thread = Internal.Runtime.Augments.RuntimeThread;
 
 namespace System.Threading
 {
     internal static class ThreadPoolGlobals
     {
-        //Per-appDomain quantum (in ms) for which the thread keeps processing
-        //requests in the current domain.
-        public const uint TP_QUANTUM = 30U;
-
         public static readonly int processorCount = Environment.ProcessorCount;
 
-        public static volatile bool vmTpInitialized;
+        public static volatile bool threadPoolInitialized;
         public static bool enableWorkerTracking;
 
         public static readonly ThreadPoolWorkQueue workQueue = new ThreadPoolWorkQueue();
@@ -52,7 +49,7 @@ namespace System.Threading
     }
 
     [StructLayout(LayoutKind.Sequential)] // enforce layout so that padding reduces false sharing
-    internal sealed class ThreadPoolWorkQueue
+    internal sealed partial class ThreadPoolWorkQueue
     {
         internal static class WorkStealingQueueList
         {
@@ -416,11 +413,10 @@ namespace System.Threading
         internal void EnsureThreadRequested()
         {
             //
-            // If we have not yet requested #procs threads from the VM, then request a new thread
-            // as needed
+            // If we have not yet requested #procs threads, then request a new thread.
             //
-            // Note that there is a separate count in the VM which will also be incremented in this case, 
-            // which is handled by RequestWorkerThread.
+            // CoreCLR: Note that there is a separate count in the VM which has already been incremented
+            // by the VM by the time we reach this point.
             //
             int count = numOutstandingThreadRequests;
             while (count < ThreadPoolGlobals.processorCount)
@@ -438,10 +434,11 @@ namespace System.Threading
         internal void MarkThreadRequestSatisfied()
         {
             //
-            // The VM has called us, so one of our outstanding thread requests has been satisfied.
+            // One of our outstanding thread requests has been satisfied.
             // Decrement the count so that future calls to EnsureThreadRequested will succeed.
-            // Note that there is a separate count in the VM which has already been decremented by the VM
-            // by the time we reach this point.
+            //
+            // CoreCLR: Note that there is a separate count in the VM which has already been decremented
+            // by the VM by the time we reach this point.
             //
             int count = numOutstandingThreadRequests;
             while (count > 0)
@@ -517,21 +514,28 @@ namespace System.Threading
             return callback;
         }
 
+        /// <summary>
+        /// Dispatches work items to this thread.
+        /// </summary>
+        /// <returns>
+        /// <c>true</c> if this thread did as much work as was available or its quantum expired.
+        /// <c>false</c> if this thread stopped working early.
+        /// </returns>
         internal static bool Dispatch()
         {
             ThreadPoolWorkQueue outerWorkQueue = ThreadPoolGlobals.workQueue;
+
             //
-            // The clock is ticking!  We have ThreadPoolGlobals.TP_QUANTUM milliseconds to get some work done, and then
-            // we need to return to the VM.
+            // Save the start time
             //
-            int quantumStartTime = Environment.TickCount;
+            int startTickCount = Environment.TickCount;
 
             //
             // Update our records to indicate that an outstanding request for a thread has now been fulfilled.
             // From this point on, we are responsible for requesting another thread if we stop working for any
             // reason, and we believe there might still be work in the queue.
             //
-            // Note that if this thread is aborted before we get a chance to request another one, the VM will
+            // CoreCLR: Note that if this thread is aborted before we get a chance to request another one, the VM will
             // record a thread request on our behalf.  So we don't need to worry about getting aborted right here.
             //
             outerWorkQueue.MarkThreadRequestSatisfied();
@@ -560,9 +564,9 @@ namespace System.Threading
                 currentThread.SynchronizationContext = null;
 
                 //
-                // Loop until our quantum expires.
+                // Loop until our quantum expires or there is no work.
                 //
-                while ((Environment.TickCount - quantumStartTime) < ThreadPoolGlobals.TP_QUANTUM)
+                while (ThreadPool.KeepDispatching(startTickCount))
                 {
                     bool missedSteal = false;
                     // Use operate on workItem local to try block so it can be enregistered 
@@ -571,10 +575,9 @@ namespace System.Threading
                     if (workItem == null)
                     {
                         //
-                        // No work.  We're going to return to the VM once we leave this protected region.
+                        // No work.
                         // If we missed a steal, though, there may be more work in the queue.
-                        // Instead of looping around and trying again, we'll just request another thread.  This way
-                        // we won't starve other AppDomains while we spin trying to get locks, and hopefully the thread
+                        // Instead of looping around and trying again, we'll just request another thread.  Hopefully the thread
                         // that owns the contended work-stealing queue will pick up its own workitems in the meantime, 
                         // which will be more efficient than this thread doing it anyway.
                         //
@@ -633,6 +636,8 @@ namespace System.Threading
                         Unsafe.As<IThreadPoolWorkItem>(workItem).Execute();
                     }
 
+                    currentThread.ResetThreadPoolThread();
+
                     // Release refs
                     outerWorkItem = workItem = null;
 
@@ -650,13 +655,6 @@ namespace System.Threading
                 // If we get here, it's because our quantum expired.  Tell the VM we're returning normally.
                 return true;
             }
-            catch (ThreadAbortException tae)
-            {
-                //
-                // In this case, the VM is going to request another thread on our behalf.  No need to do it twice.
-                //
-                needAnotherThread = false;
-            }
             finally
             {
                 //
@@ -666,10 +664,6 @@ namespace System.Threading
                 if (needAnotherThread)
                     outerWorkQueue.EnsureThreadRequested();
             }
-
-            // we can never reach this point, but the C# compiler doesn't know that, because it doesn't know the ThreadAbortException will be reraised above.
-            Debug.Fail("Should never reach this point");
-            return true;
         }
     }
 
@@ -747,188 +741,16 @@ namespace System.Threading
         }
     }
 
-    internal sealed class RegisteredWaitHandleSafe : CriticalFinalizerObject
-    {
-        private static IntPtr InvalidHandle => Win32Native.INVALID_HANDLE_VALUE;
-        private IntPtr registeredWaitHandle = InvalidHandle;
-        private WaitHandle m_internalWaitObject;
-        private bool bReleaseNeeded = false;
-        private volatile int m_lock = 0;
-
-        internal IntPtr GetHandle() => registeredWaitHandle;
-
-        internal void SetHandle(IntPtr handle)
-        {
-            registeredWaitHandle = handle;
-        }
-
-        internal void SetWaitObject(WaitHandle waitObject)
-        {
-            // needed for DangerousAddRef
-            RuntimeHelpers.PrepareConstrainedRegions();
-
-            m_internalWaitObject = waitObject;
-            if (waitObject != null)
-            {
-                m_internalWaitObject.SafeWaitHandle.DangerousAddRef(ref bReleaseNeeded);
-            }
-        }
-
-        internal bool Unregister(
-             WaitHandle waitObject          // object to be notified when all callbacks to delegates have completed
-             )
-        {
-            bool result = false;
-            // needed for DangerousRelease
-            RuntimeHelpers.PrepareConstrainedRegions();
-
-            // lock(this) cannot be used reliably in Cer since thin lock could be
-            // promoted to syncblock and that is not a guaranteed operation
-            bool bLockTaken = false;
-            do
-            {
-                if (Interlocked.CompareExchange(ref m_lock, 1, 0) == 0)
-                {
-                    bLockTaken = true;
-                    try
-                    {
-                        if (ValidHandle())
-                        {
-                            result = UnregisterWaitNative(GetHandle(), waitObject == null ? null : waitObject.SafeWaitHandle);
-                            if (result == true)
-                            {
-                                if (bReleaseNeeded)
-                                {
-                                    m_internalWaitObject.SafeWaitHandle.DangerousRelease();
-                                    bReleaseNeeded = false;
-                                }
-                                // if result not true don't release/suppress here so finalizer can make another attempt
-                                SetHandle(InvalidHandle);
-                                m_internalWaitObject = null;
-                                GC.SuppressFinalize(this);
-                            }
-                        }
-                    }
-                    finally
-                    {
-                        m_lock = 0;
-                    }
-                }
-                Thread.SpinWait(1);     // yield to processor
-            }
-            while (!bLockTaken);
-
-            return result;
-        }
-
-        private bool ValidHandle() =>
-            registeredWaitHandle != InvalidHandle && registeredWaitHandle != IntPtr.Zero;
-
-        ~RegisteredWaitHandleSafe()
-        {
-            // if the app has already unregistered the wait, there is nothing to cleanup
-            // we can detect this by checking the handle. Normally, there is no race condition here
-            // so no need to protect reading of handle. However, if this object gets 
-            // resurrected and then someone does an unregister, it would introduce a race condition
-            //
-            // PrepareConstrainedRegions call not needed since finalizer already in Cer
-            //
-            // lock(this) cannot be used reliably even in Cer since thin lock could be
-            // promoted to syncblock and that is not a guaranteed operation
-            //
-            // Note that we will not "spin" to get this lock.  We make only a single attempt;
-            // if we can't get the lock, it means some other thread is in the middle of a call
-            // to Unregister, which will do the work of the finalizer anyway.
-            //
-            // Further, it's actually critical that we *not* wait for the lock here, because
-            // the other thread that's in the middle of Unregister may be suspended for shutdown.
-            // Then, during the live-object finalization phase of shutdown, this thread would
-            // end up spinning forever, as the other thread would never release the lock.
-            // This will result in a "leak" of sorts (since the handle will not be cleaned up)
-            // but the process is exiting anyway.
-            //
-            // During AD-unload, we don�t finalize live objects until all threads have been 
-            // aborted out of the AD.  Since these locked regions are CERs, we won�t abort them 
-            // while the lock is held.  So there should be no leak on AD-unload.
-            //
-            if (Interlocked.CompareExchange(ref m_lock, 1, 0) == 0)
-            {
-                try
-                {
-                    if (ValidHandle())
-                    {
-                        WaitHandleCleanupNative(registeredWaitHandle);
-                        if (bReleaseNeeded)
-                        {
-                            m_internalWaitObject.SafeWaitHandle.DangerousRelease();
-                            bReleaseNeeded = false;
-                        }
-                        SetHandle(InvalidHandle);
-                        m_internalWaitObject = null;
-                    }
-                }
-                finally
-                {
-                    m_lock = 0;
-                }
-            }
-        }
-
-        [MethodImplAttribute(MethodImplOptions.InternalCall)]
-        private static extern void WaitHandleCleanupNative(IntPtr handle);
-
-        [MethodImplAttribute(MethodImplOptions.InternalCall)]
-        private static extern bool UnregisterWaitNative(IntPtr handle, SafeHandle waitObject);
-    }
-
-    public sealed class RegisteredWaitHandle : MarshalByRefObject
-    {
-        private readonly RegisteredWaitHandleSafe internalRegisteredWait;
-
-        internal RegisteredWaitHandle()
-        {
-            internalRegisteredWait = new RegisteredWaitHandleSafe();
-        }
-
-        internal void SetHandle(IntPtr handle)
-        {
-            internalRegisteredWait.SetHandle(handle);
-        }
-
-        internal void SetWaitObject(WaitHandle waitObject)
-        {
-            internalRegisteredWait.SetWaitObject(waitObject);
-        }
-
-        // This is the only public method on this class
-        public bool Unregister(
-             WaitHandle waitObject          // object to be notified when all callbacks to delegates have completed
-             )
-        {
-            return internalRegisteredWait.Unregister(waitObject);
-        }
-    }
-
     public delegate void WaitCallback(object state);
 
     public delegate void WaitOrTimerCallback(object state, bool timedOut);  // signaled or timed out
 
-    //
-    // This type is necessary because VS 2010's debugger looks for a method named _ThreadPoolWaitCallbacck.PerformWaitCallback
-    // on the stack to determine if a thread is a ThreadPool thread or not.  We have a better way to do this for .NET 4.5, but
-    // still need to maintain compatibility with VS 2010.  When compat with VS 2010 is no longer an issue, this type may be
-    // removed.
-    //
-    internal static class _ThreadPoolWaitCallback
-    {
-        internal static bool PerformWaitCallback() => ThreadPoolWorkQueue.Dispatch();
-    }
-
     internal abstract class QueueUserWorkItemCallbackBase : IThreadPoolWorkItem
     {
 #if DEBUG
         private volatile int executed;
 
+        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1821:RemoveEmptyFinalizers")]
         ~QueueUserWorkItemCallbackBase()
         {
             Debug.Assert(
@@ -1067,12 +889,12 @@ namespace System.Threading
         private static readonly ContextCallback _ccbt = new ContextCallback(WaitOrTimerCallback_Context_t);
         private static readonly ContextCallback _ccbf = new ContextCallback(WaitOrTimerCallback_Context_f);
 
-        internal _ThreadPoolWaitOrTimerCallback(WaitOrTimerCallback waitOrTimerCallback, object state, bool compressStack)
+        internal _ThreadPoolWaitOrTimerCallback(WaitOrTimerCallback waitOrTimerCallback, object state, bool flowExecutionContext)
         {
             _waitOrTimerCallback = waitOrTimerCallback;
             _state = state;
 
-            if (compressStack)
+            if (flowExecutionContext)
             {
                 // capture the exection context
                 _executionContext = ExecutionContext.Capture();
@@ -1092,9 +914,8 @@ namespace System.Threading
         }
 
         // call back helper
-        internal static void PerformWaitOrTimerCallback(object state, bool timedOut)
+        internal static void PerformWaitOrTimerCallback(_ThreadPoolWaitOrTimerCallback helper, bool timedOut)
         {
-            _ThreadPoolWaitOrTimerCallback helper = (_ThreadPoolWaitOrTimerCallback)state;
             Debug.Assert(helper != null, "Null state passed to PerformWaitOrTimerCallback!");
             // call directly if it is an unsafe call OR EC flow is suppressed
             ExecutionContext context = helper._executionContext;
@@ -1110,41 +931,10 @@ namespace System.Threading
         }
     }
 
-    [CLSCompliant(false)]
-    public unsafe delegate void IOCompletionCallback(uint errorCode, // Error code
-                                       uint numBytes, // No. of bytes transferred 
-                                       NativeOverlapped* pOVERLAP // ptr to OVERLAP structure
-                                       );
-
-    public static class ThreadPool
+    public static partial class ThreadPool
     {
-        public static bool SetMaxThreads(int workerThreads, int completionPortThreads)
-        {
-            return SetMaxThreadsNative(workerThreads, completionPortThreads);
-        }
-
-        public static void GetMaxThreads(out int workerThreads, out int completionPortThreads)
-        {
-            GetMaxThreadsNative(out workerThreads, out completionPortThreads);
-        }
-
-        public static bool SetMinThreads(int workerThreads, int completionPortThreads)
-        {
-            return SetMinThreadsNative(workerThreads, completionPortThreads);
-        }
-
-        public static void GetMinThreads(out int workerThreads, out int completionPortThreads)
-        {
-            GetMinThreadsNative(out workerThreads, out completionPortThreads);
-        }
-
-        public static void GetAvailableThreads(out int workerThreads, out int completionPortThreads)
-        {
-            GetAvailableThreadsNative(out workerThreads, out completionPortThreads);
-        }
-
         [CLSCompliant(false)]
-        public static RegisteredWaitHandle RegisterWaitForSingleObject(  // throws RegisterWaitException
+        public static RegisteredWaitHandle RegisterWaitForSingleObject(
              WaitHandle waitObject,
              WaitOrTimerCallback callBack,
              object state,
@@ -1152,11 +942,13 @@ namespace System.Threading
              bool executeOnlyOnce    // NOTE: we do not allow other options that allow the callback to be queued as an APC
              )
         {
+            if (millisecondsTimeOutInterval > (uint)int.MaxValue && millisecondsTimeOutInterval != uint.MaxValue)
+                throw new ArgumentOutOfRangeException(nameof(millisecondsTimeOutInterval), SR.ArgumentOutOfRange_NeedNonNegOrNegative1);
             return RegisterWaitForSingleObject(waitObject, callBack, state, millisecondsTimeOutInterval, executeOnlyOnce, true);
         }
 
         [CLSCompliant(false)]
-        public static RegisteredWaitHandle UnsafeRegisterWaitForSingleObject(  // throws RegisterWaitException
+        public static RegisteredWaitHandle UnsafeRegisterWaitForSingleObject(
              WaitHandle waitObject,
              WaitOrTimerCallback callBack,
              object state,
@@ -1164,44 +956,12 @@ namespace System.Threading
              bool executeOnlyOnce    // NOTE: we do not allow other options that allow the callback to be queued as an APC
              )
         {
+            if (millisecondsTimeOutInterval > (uint)int.MaxValue && millisecondsTimeOutInterval != uint.MaxValue)
+                throw new ArgumentOutOfRangeException(nameof(millisecondsTimeOutInterval), SR.ArgumentOutOfRange_NeedNonNegOrNegative1);
             return RegisterWaitForSingleObject(waitObject, callBack, state, millisecondsTimeOutInterval, executeOnlyOnce, false);
         }
 
-
-        private static RegisteredWaitHandle RegisterWaitForSingleObject(  // throws RegisterWaitException
-             WaitHandle waitObject,
-             WaitOrTimerCallback callBack,
-             object state,
-             uint millisecondsTimeOutInterval,
-             bool executeOnlyOnce,   // NOTE: we do not allow other options that allow the callback to be queued as an APC
-             bool compressStack
-             )
-        {
-            RegisteredWaitHandle registeredWaitHandle = new RegisteredWaitHandle();
-
-            if (callBack != null)
-            {
-                _ThreadPoolWaitOrTimerCallback callBackHelper = new _ThreadPoolWaitOrTimerCallback(callBack, state, compressStack);
-                state = (object)callBackHelper;
-                // call SetWaitObject before native call so that waitObject won't be closed before threadpoolmgr registration
-                // this could occur if callback were to fire before SetWaitObject does its addref
-                registeredWaitHandle.SetWaitObject(waitObject);
-                IntPtr nativeRegisteredWaitHandle = RegisterWaitForSingleObjectNative(waitObject,
-                                                                               state,
-                                                                               millisecondsTimeOutInterval,
-                                                                               executeOnlyOnce,
-                                                                               registeredWaitHandle);
-                registeredWaitHandle.SetHandle(nativeRegisteredWaitHandle);
-            }
-            else
-            {
-                throw new ArgumentNullException(nameof(WaitOrTimerCallback));
-            }
-            return registeredWaitHandle;
-        }
-
-
-        public static RegisteredWaitHandle RegisterWaitForSingleObject(  // throws RegisterWaitException
+        public static RegisteredWaitHandle RegisterWaitForSingleObject(
              WaitHandle waitObject,
              WaitOrTimerCallback callBack,
              object state,
@@ -1214,7 +974,7 @@ namespace System.Threading
             return RegisterWaitForSingleObject(waitObject, callBack, state, (uint)millisecondsTimeOutInterval, executeOnlyOnce, true);
         }
 
-        public static RegisteredWaitHandle UnsafeRegisterWaitForSingleObject(  // throws RegisterWaitException
+        public static RegisteredWaitHandle UnsafeRegisterWaitForSingleObject(
              WaitHandle waitObject,
              WaitOrTimerCallback callBack,
              object state,
@@ -1227,7 +987,7 @@ namespace System.Threading
             return RegisterWaitForSingleObject(waitObject, callBack, state, (uint)millisecondsTimeOutInterval, executeOnlyOnce, false);
         }
 
-        public static RegisteredWaitHandle RegisterWaitForSingleObject(  // throws RegisterWaitException
+        public static RegisteredWaitHandle RegisterWaitForSingleObject(
             WaitHandle waitObject,
             WaitOrTimerCallback callBack,
             object state,
@@ -1240,7 +1000,7 @@ namespace System.Threading
             return RegisterWaitForSingleObject(waitObject, callBack, state, (uint)millisecondsTimeOutInterval, executeOnlyOnce, true);
         }
 
-        public static RegisteredWaitHandle UnsafeRegisterWaitForSingleObject(  // throws RegisterWaitException
+        public static RegisteredWaitHandle UnsafeRegisterWaitForSingleObject(
             WaitHandle waitObject,
             WaitOrTimerCallback callBack,
             object state,
@@ -1295,7 +1055,7 @@ namespace System.Threading
                 ThrowHelper.ThrowArgumentNullException(ExceptionArgument.callBack);
             }
 
-            EnsureVMInitialized();
+            EnsureInitialized();
 
             ExecutionContext context = ExecutionContext.Capture();
 
@@ -1315,7 +1075,7 @@ namespace System.Threading
                 ThrowHelper.ThrowArgumentNullException(ExceptionArgument.callBack);
             }
 
-            EnsureVMInitialized();
+            EnsureInitialized();
 
             ExecutionContext context = ExecutionContext.Capture();
 
@@ -1353,7 +1113,7 @@ namespace System.Threading
                 return true;
             }
 
-            EnsureVMInitialized();
+            EnsureInitialized();
 
             ThreadPoolGlobals.workQueue.Enqueue(
                 new QueueUserWorkItemCallbackDefaultContext<TState>(callBack, state), forceGlobal: !preferLocal);
@@ -1368,7 +1128,7 @@ namespace System.Threading
                 ThrowHelper.ThrowArgumentNullException(ExceptionArgument.callBack);
             }
 
-            EnsureVMInitialized();
+            EnsureInitialized();
 
             object tpcallBack = new QueueUserWorkItemCallbackDefaultContext(callBack, state);
 
@@ -1398,7 +1158,7 @@ namespace System.Threading
         {
             Debug.Assert((callBack is IThreadPoolWorkItem) ^ (callBack is Task));
 
-            EnsureVMInitialized();
+            EnsureInitialized();
 
             ThreadPoolGlobals.workQueue.Enqueue(callBack, forceGlobal: !preferLocal);
         }
@@ -1408,7 +1168,7 @@ namespace System.Threading
         {
             Debug.Assert(null != workItem);
             return
-                ThreadPoolGlobals.vmTpInitialized && // if not initialized, so there's no way this workitem was ever queued.
+                ThreadPoolGlobals.threadPoolInitialized && // if not initialized, so there's no way this workitem was ever queued.
                 ThreadPoolGlobals.workQueue.LocalFindAndPop(workItem);
         }
 
@@ -1486,110 +1246,5 @@ namespace System.Threading
 
         internal static object[] GetLocallyQueuedWorkItemsForDebugger() =>
             ToObjectArray(GetLocallyQueuedWorkItems());
-
-        [DllImport(JitHelpers.QCall, CharSet = CharSet.Unicode)]
-        internal static extern bool RequestWorkerThread();
-
-        [MethodImplAttribute(MethodImplOptions.InternalCall)]
-        private static extern unsafe bool PostQueuedCompletionStatus(NativeOverlapped* overlapped);
-
-        [CLSCompliant(false)]
-        public static unsafe bool UnsafeQueueNativeOverlapped(NativeOverlapped* overlapped) =>
-            PostQueuedCompletionStatus(overlapped);
-
-        // The thread pool maintains a per-appdomain managed work queue.
-        // New thread pool entries are added in the managed queue.
-        // The VM is responsible for the actual growing/shrinking of 
-        // threads. 
-        private static void EnsureVMInitialized()
-        {
-            if (!ThreadPoolGlobals.vmTpInitialized)
-            {
-                EnsureVMInitializedCore(); // separate out to help with inlining
-            }
-        }
-
-        [MethodImpl(MethodImplOptions.NoInlining)]
-        private static void EnsureVMInitializedCore()
-        {
-            InitializeVMTp(ref ThreadPoolGlobals.enableWorkerTracking);
-            ThreadPoolGlobals.vmTpInitialized = true;
-        }
-
-        // Native methods: 
-
-        [MethodImplAttribute(MethodImplOptions.InternalCall)]
-        private static extern bool SetMinThreadsNative(int workerThreads, int completionPortThreads);
-
-        [MethodImplAttribute(MethodImplOptions.InternalCall)]
-        private static extern bool SetMaxThreadsNative(int workerThreads, int completionPortThreads);
-
-        [MethodImplAttribute(MethodImplOptions.InternalCall)]
-        private static extern void GetMinThreadsNative(out int workerThreads, out int completionPortThreads);
-
-        [MethodImplAttribute(MethodImplOptions.InternalCall)]
-        private static extern void GetMaxThreadsNative(out int workerThreads, out int completionPortThreads);
-
-        [MethodImplAttribute(MethodImplOptions.InternalCall)]
-        private static extern void GetAvailableThreadsNative(out int workerThreads, out int completionPortThreads);
-
-        [MethodImplAttribute(MethodImplOptions.InternalCall)]
-        internal static extern bool NotifyWorkItemComplete();
-
-        [MethodImplAttribute(MethodImplOptions.InternalCall)]
-        internal static extern void ReportThreadStatus(bool isWorking);
-
-        internal static void NotifyWorkItemProgress()
-        {
-            if (!ThreadPoolGlobals.vmTpInitialized)
-                ThreadPool.InitializeVMTp(ref ThreadPoolGlobals.enableWorkerTracking);
-            NotifyWorkItemProgressNative();
-        }
-
-        [MethodImplAttribute(MethodImplOptions.InternalCall)]
-        internal static extern void NotifyWorkItemProgressNative();
-
-        [DllImport(JitHelpers.QCall, CharSet = CharSet.Unicode)]
-        private static extern void InitializeVMTp(ref bool enableWorkerTracking);
-
-        [MethodImplAttribute(MethodImplOptions.InternalCall)]
-        private static extern IntPtr RegisterWaitForSingleObjectNative(
-             WaitHandle waitHandle,
-             object state,
-             uint timeOutInterval,
-             bool executeOnlyOnce,
-             RegisteredWaitHandle registeredWaitHandle
-             );
-
-
-        [Obsolete("ThreadPool.BindHandle(IntPtr) has been deprecated.  Please use ThreadPool.BindHandle(SafeHandle) instead.", false)]
-        public static bool BindHandle(IntPtr osHandle)
-        {
-            return BindIOCompletionCallbackNative(osHandle);
-        }
-
-        public static bool BindHandle(SafeHandle osHandle)
-        {
-            if (osHandle == null)
-                throw new ArgumentNullException(nameof(osHandle));
-
-            bool ret = false;
-            bool mustReleaseSafeHandle = false;
-            RuntimeHelpers.PrepareConstrainedRegions();
-            try
-            {
-                osHandle.DangerousAddRef(ref mustReleaseSafeHandle);
-                ret = BindIOCompletionCallbackNative(osHandle.DangerousGetHandle());
-            }
-            finally
-            {
-                if (mustReleaseSafeHandle)
-                    osHandle.DangerousRelease();
-            }
-            return ret;
-        }
-
-        [MethodImplAttribute(MethodImplOptions.InternalCall)]
-        private static extern bool BindIOCompletionCallbackNative(IntPtr fileHandle);
     }
 }
diff --git a/src/System.Private.CoreLib/src/Internal/Runtime/Augments/RuntimeThread.cs b/src/System.Private.CoreLib/src/Internal/Runtime/Augments/RuntimeThread.cs
index c613b2c01b..89e63968f0 100644
--- a/src/System.Private.CoreLib/src/Internal/Runtime/Augments/RuntimeThread.cs
+++ b/src/System.Private.CoreLib/src/Internal/Runtime/Augments/RuntimeThread.cs
@@ -277,5 +277,11 @@ namespace Internal.Runtime.Augments
 
         public void Start() => AsThread().Start();
         public void Start(object parameter) => AsThread().Start(parameter);
+
+        public void ResetThreadPoolThread()
+        {
+            // Currently implemented in unmanaged method Thread::InternalReset and
+            // called internally from the ThreadPool in NotifyWorkItemComplete.
+        }
     }
 }
diff --git a/src/System.Private.CoreLib/src/System/Threading/ThreadPool.CoreCLR.cs b/src/System.Private.CoreLib/src/System/Threading/ThreadPool.CoreCLR.cs
new file mode 100644
index 0000000000..812279f50d
--- /dev/null
+++ b/src/System.Private.CoreLib/src/System/Threading/ThreadPool.CoreCLR.cs
@@ -0,0 +1,374 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+/*=============================================================================
+**
+**
+**
+** Purpose: Class for creating and managing a threadpool
+**
+**
+=============================================================================*/
+
+using System.Runtime.CompilerServices;
+using System.Runtime.ConstrainedExecution;
+using System.Runtime.InteropServices;
+using Microsoft.Win32;
+
+namespace System.Threading
+{
+    //
+    // This type is necessary because VS 2010's debugger looks for a method named _ThreadPoolWaitCallbacck.PerformWaitCallback
+    // on the stack to determine if a thread is a ThreadPool thread or not.  We have a better way to do this for .NET 4.5, but
+    // still need to maintain compatibility with VS 2010.  When compat with VS 2010 is no longer an issue, this type may be
+    // removed.
+    //
+    internal static class _ThreadPoolWaitCallback
+    {
+        internal static bool PerformWaitCallback() => ThreadPoolWorkQueue.Dispatch();
+    }
+
+    internal sealed class RegisteredWaitHandleSafe : CriticalFinalizerObject
+    {
+        private static IntPtr InvalidHandle => Win32Native.INVALID_HANDLE_VALUE;
+        private IntPtr registeredWaitHandle = InvalidHandle;
+        private WaitHandle m_internalWaitObject;
+        private bool bReleaseNeeded = false;
+        private volatile int m_lock = 0;
+
+        internal IntPtr GetHandle() => registeredWaitHandle;
+
+        internal void SetHandle(IntPtr handle)
+        {
+            registeredWaitHandle = handle;
+        }
+
+        internal void SetWaitObject(WaitHandle waitObject)
+        {
+            // needed for DangerousAddRef
+            RuntimeHelpers.PrepareConstrainedRegions();
+
+            m_internalWaitObject = waitObject;
+            if (waitObject != null)
+            {
+                m_internalWaitObject.SafeWaitHandle.DangerousAddRef(ref bReleaseNeeded);
+            }
+        }
+
+        internal bool Unregister(
+             WaitHandle waitObject          // object to be notified when all callbacks to delegates have completed
+             )
+        {
+            bool result = false;
+            // needed for DangerousRelease
+            RuntimeHelpers.PrepareConstrainedRegions();
+
+            // lock(this) cannot be used reliably in Cer since thin lock could be
+            // promoted to syncblock and that is not a guaranteed operation
+            bool bLockTaken = false;
+            do
+            {
+                if (Interlocked.CompareExchange(ref m_lock, 1, 0) == 0)
+                {
+                    bLockTaken = true;
+                    try
+                    {
+                        if (ValidHandle())
+                        {
+                            result = UnregisterWaitNative(GetHandle(), waitObject == null ? null : waitObject.SafeWaitHandle);
+                            if (result == true)
+                            {
+                                if (bReleaseNeeded)
+                                {
+                                    m_internalWaitObject.SafeWaitHandle.DangerousRelease();
+                                    bReleaseNeeded = false;
+                                }
+                                // if result not true don't release/suppress here so finalizer can make another attempt
+                                SetHandle(InvalidHandle);
+                                m_internalWaitObject = null;
+                                GC.SuppressFinalize(this);
+                            }
+                        }
+                    }
+                    finally
+                    {
+                        m_lock = 0;
+                    }
+                }
+                Thread.SpinWait(1);     // yield to processor
+            }
+            while (!bLockTaken);
+
+            return result;
+        }
+
+        private bool ValidHandle() =>
+            registeredWaitHandle != InvalidHandle && registeredWaitHandle != IntPtr.Zero;
+
+        ~RegisteredWaitHandleSafe()
+        {
+            // if the app has already unregistered the wait, there is nothing to cleanup
+            // we can detect this by checking the handle. Normally, there is no race condition here
+            // so no need to protect reading of handle. However, if this object gets 
+            // resurrected and then someone does an unregister, it would introduce a race condition
+            //
+            // PrepareConstrainedRegions call not needed since finalizer already in Cer
+            //
+            // lock(this) cannot be used reliably even in Cer since thin lock could be
+            // promoted to syncblock and that is not a guaranteed operation
+            //
+            // Note that we will not "spin" to get this lock.  We make only a single attempt;
+            // if we can't get the lock, it means some other thread is in the middle of a call
+            // to Unregister, which will do the work of the finalizer anyway.
+            //
+            // Further, it's actually critical that we *not* wait for the lock here, because
+            // the other thread that's in the middle of Unregister may be suspended for shutdown.
+            // Then, during the live-object finalization phase of shutdown, this thread would
+            // end up spinning forever, as the other thread would never release the lock.
+            // This will result in a "leak" of sorts (since the handle will not be cleaned up)
+            // but the process is exiting anyway.
+            //
+            // During AD-unload, we don't finalize live objects until all threads have been 
+            // aborted out of the AD.  Since these locked regions are CERs, we won't abort them 
+            // while the lock is held.  So there should be no leak on AD-unload.
+            //
+            if (Interlocked.CompareExchange(ref m_lock, 1, 0) == 0)
+            {
+                try
+                {
+                    if (ValidHandle())
+                    {
+                        WaitHandleCleanupNative(registeredWaitHandle);
+                        if (bReleaseNeeded)
+                        {
+                            m_internalWaitObject.SafeWaitHandle.DangerousRelease();
+                            bReleaseNeeded = false;
+                        }
+                        SetHandle(InvalidHandle);
+                        m_internalWaitObject = null;
+                    }
+                }
+                finally
+                {
+                    m_lock = 0;
+                }
+            }
+        }
+
+        [MethodImplAttribute(MethodImplOptions.InternalCall)]
+        private static extern void WaitHandleCleanupNative(IntPtr handle);
+
+        [MethodImplAttribute(MethodImplOptions.InternalCall)]
+        private static extern bool UnregisterWaitNative(IntPtr handle, SafeHandle waitObject);
+    }
+
+    public sealed class RegisteredWaitHandle : MarshalByRefObject
+    {
+        private readonly RegisteredWaitHandleSafe internalRegisteredWait;
+
+        internal RegisteredWaitHandle()
+        {
+            internalRegisteredWait = new RegisteredWaitHandleSafe();
+        }
+
+        internal void SetHandle(IntPtr handle)
+        {
+            internalRegisteredWait.SetHandle(handle);
+        }
+
+        internal void SetWaitObject(WaitHandle waitObject)
+        {
+            internalRegisteredWait.SetWaitObject(waitObject);
+        }
+
+        // This is the only public method on this class
+        public bool Unregister(
+             WaitHandle waitObject          // object to be notified when all callbacks to delegates have completed
+             )
+        {
+            return internalRegisteredWait.Unregister(waitObject);
+        }
+    }
+
+    [CLSCompliant(false)]
+    public unsafe delegate void IOCompletionCallback(uint errorCode, // Error code
+                                       uint numBytes, // No. of bytes transferred 
+                                       NativeOverlapped* pOVERLAP // ptr to OVERLAP structure
+                                       );
+
+    public static partial class ThreadPool
+    {
+        // Time in ms for which ThreadPoolWorkQueue.Dispatch keeps executing work items before returning to the OS
+        private const uint DispatchQuantum = 30;
+
+        internal static bool KeepDispatching(int startTickCount)
+        {
+            // Note: this function may incorrectly return false due to TickCount overflow
+            // if work item execution took around a multiple of 2^32 milliseconds (~49.7 days),
+            // which is improbable.
+            return ((uint)(Environment.TickCount - startTickCount) < DispatchQuantum);
+        }
+
+        public static bool SetMaxThreads(int workerThreads, int completionPortThreads)
+        {
+            return SetMaxThreadsNative(workerThreads, completionPortThreads);
+        }
+
+        public static void GetMaxThreads(out int workerThreads, out int completionPortThreads)
+        {
+            GetMaxThreadsNative(out workerThreads, out completionPortThreads);
+        }
+
+        public static bool SetMinThreads(int workerThreads, int completionPortThreads)
+        {
+            return SetMinThreadsNative(workerThreads, completionPortThreads);
+        }
+
+        public static void GetMinThreads(out int workerThreads, out int completionPortThreads)
+        {
+            GetMinThreadsNative(out workerThreads, out completionPortThreads);
+        }
+
+        public static void GetAvailableThreads(out int workerThreads, out int completionPortThreads)
+        {
+            GetAvailableThreadsNative(out workerThreads, out completionPortThreads);
+        }
+
+        private static RegisteredWaitHandle RegisterWaitForSingleObject(  // throws RegisterWaitException
+             WaitHandle waitObject,
+             WaitOrTimerCallback callBack,
+             object state,
+             uint millisecondsTimeOutInterval,
+             bool executeOnlyOnce,   // NOTE: we do not allow other options that allow the callback to be queued as an APC
+             bool compressStack
+             )
+        {
+            RegisteredWaitHandle registeredWaitHandle = new RegisteredWaitHandle();
+
+            if (callBack != null)
+            {
+                _ThreadPoolWaitOrTimerCallback callBackHelper = new _ThreadPoolWaitOrTimerCallback(callBack, state, compressStack);
+                state = (object)callBackHelper;
+                // call SetWaitObject before native call so that waitObject won't be closed before threadpoolmgr registration
+                // this could occur if callback were to fire before SetWaitObject does its addref
+                registeredWaitHandle.SetWaitObject(waitObject);
+                IntPtr nativeRegisteredWaitHandle = RegisterWaitForSingleObjectNative(waitObject,
+                                                                               state,
+                                                                               millisecondsTimeOutInterval,
+                                                                               executeOnlyOnce,
+                                                                               registeredWaitHandle);
+                registeredWaitHandle.SetHandle(nativeRegisteredWaitHandle);
+            }
+            else
+            {
+                throw new ArgumentNullException(nameof(WaitOrTimerCallback));
+            }
+            return registeredWaitHandle;
+        }
+
+        [DllImport(JitHelpers.QCall, CharSet = CharSet.Unicode)]
+        internal static extern bool RequestWorkerThread();
+
+        [MethodImplAttribute(MethodImplOptions.InternalCall)]
+        private static extern unsafe bool PostQueuedCompletionStatus(NativeOverlapped* overlapped);
+
+        [CLSCompliant(false)]
+        public static unsafe bool UnsafeQueueNativeOverlapped(NativeOverlapped* overlapped) =>
+            PostQueuedCompletionStatus(overlapped);
+
+        // The thread pool maintains a per-appdomain managed work queue.
+        // New thread pool entries are added in the managed queue.
+        // The VM is responsible for the actual growing/shrinking of 
+        // threads. 
+        private static void EnsureInitialized()
+        {
+            if (!ThreadPoolGlobals.threadPoolInitialized)
+            {
+                EnsureVMInitializedCore(); // separate out to help with inlining
+            }
+        }
+
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        private static void EnsureVMInitializedCore()
+        {
+            InitializeVMTp(ref ThreadPoolGlobals.enableWorkerTracking);
+            ThreadPoolGlobals.threadPoolInitialized = true;
+        }
+
+        // Native methods: 
+
+        [MethodImplAttribute(MethodImplOptions.InternalCall)]
+        private static extern bool SetMinThreadsNative(int workerThreads, int completionPortThreads);
+
+        [MethodImplAttribute(MethodImplOptions.InternalCall)]
+        private static extern bool SetMaxThreadsNative(int workerThreads, int completionPortThreads);
+
+        [MethodImplAttribute(MethodImplOptions.InternalCall)]
+        private static extern void GetMinThreadsNative(out int workerThreads, out int completionPortThreads);
+
+        [MethodImplAttribute(MethodImplOptions.InternalCall)]
+        private static extern void GetMaxThreadsNative(out int workerThreads, out int completionPortThreads);
+
+        [MethodImplAttribute(MethodImplOptions.InternalCall)]
+        private static extern void GetAvailableThreadsNative(out int workerThreads, out int completionPortThreads);
+
+        [MethodImplAttribute(MethodImplOptions.InternalCall)]
+        internal static extern bool NotifyWorkItemComplete();
+
+        [MethodImplAttribute(MethodImplOptions.InternalCall)]
+        internal static extern void ReportThreadStatus(bool isWorking);
+
+        internal static void NotifyWorkItemProgress()
+        {
+            EnsureInitialized();
+            NotifyWorkItemProgressNative();
+        }
+
+        [MethodImplAttribute(MethodImplOptions.InternalCall)]
+        internal static extern void NotifyWorkItemProgressNative();
+
+        [DllImport(JitHelpers.QCall, CharSet = CharSet.Unicode)]
+        private static extern void InitializeVMTp(ref bool enableWorkerTracking);
+
+        [MethodImplAttribute(MethodImplOptions.InternalCall)]
+        private static extern IntPtr RegisterWaitForSingleObjectNative(
+             WaitHandle waitHandle,
+             object state,
+             uint timeOutInterval,
+             bool executeOnlyOnce,
+             RegisteredWaitHandle registeredWaitHandle
+             );
+
+
+        [Obsolete("ThreadPool.BindHandle(IntPtr) has been deprecated.  Please use ThreadPool.BindHandle(SafeHandle) instead.", false)]
+        public static bool BindHandle(IntPtr osHandle)
+        {
+            return BindIOCompletionCallbackNative(osHandle);
+        }
+
+        public static bool BindHandle(SafeHandle osHandle)
+        {
+            if (osHandle == null)
+                throw new ArgumentNullException(nameof(osHandle));
+
+            bool ret = false;
+            bool mustReleaseSafeHandle = false;
+            RuntimeHelpers.PrepareConstrainedRegions();
+            try
+            {
+                osHandle.DangerousAddRef(ref mustReleaseSafeHandle);
+                ret = BindIOCompletionCallbackNative(osHandle.DangerousGetHandle());
+            }
+            finally
+            {
+                if (mustReleaseSafeHandle)
+                    osHandle.DangerousRelease();
+            }
+            return ret;
+        }
+
+        [MethodImplAttribute(MethodImplOptions.InternalCall)]
+        private static extern bool BindIOCompletionCallbackNative(IntPtr fileHandle);
+    }
+}
diff --git a/src/vm/metasig.h b/src/vm/metasig.h
index e8327fc020..7700c0a5b4 100644
--- a/src/vm/metasig.h
+++ b/src/vm/metasig.h
@@ -555,7 +555,7 @@ DEFINE_METASIG_T(SM(Str_AssemblyBase_IntPtr_RetIntPtr, s C(ASSEMBLYBASE) I, I))
 DEFINE_METASIG_T(SM(Str_AssemblyBase_Bool_UInt_RetIntPtr, s C(ASSEMBLYBASE) F K, I))
 
 // ThreadPool
-DEFINE_METASIG(SM(Obj_Bool_RetVoid, j F, v))
+DEFINE_METASIG_T(SM(_ThreadPoolWaitOrTimerCallback_Bool_RetVoid, C(TPWAITORTIMER_HELPER) F, v))
 
 // For FailFast
 DEFINE_METASIG(SM(Str_RetVoid, s, v))
diff --git a/src/vm/mscorlib.h b/src/vm/mscorlib.h
index 95819d95b6..81c21f2bde 100644
--- a/src/vm/mscorlib.h
+++ b/src/vm/mscorlib.h
@@ -846,7 +846,7 @@ DEFINE_CLASS(IOCB_HELPER,              Threading,            _IOCompletionCallba
 DEFINE_METHOD(IOCB_HELPER,             PERFORM_IOCOMPLETION_CALLBACK,        PerformIOCompletionCallback,          SM_UInt_UInt_PtrNativeOverlapped_RetVoid)
 
 DEFINE_CLASS(TPWAITORTIMER_HELPER,              Threading,            _ThreadPoolWaitOrTimerCallback)
-DEFINE_METHOD(TPWAITORTIMER_HELPER,             PERFORM_WAITORTIMER_CALLBACK,        PerformWaitOrTimerCallback,          SM_Obj_Bool_RetVoid)
+DEFINE_METHOD(TPWAITORTIMER_HELPER,             PERFORM_WAITORTIMER_CALLBACK,        PerformWaitOrTimerCallback,          SM__ThreadPoolWaitOrTimerCallback_Bool_RetVoid)
 
 DEFINE_CLASS(TP_WAIT_CALLBACK,         Threading,              _ThreadPoolWaitCallback)
 DEFINE_METHOD(TP_WAIT_CALLBACK,        PERFORM_WAIT_CALLBACK,               PerformWaitCallback,                   SM_RetBool)