Jitted Code Pitching Feature implemented

15 files changed, 638 insertions, 36 deletions

diff --git a/src/inc/CMakeLists.txt b/src/inc/CMakeLists.txt
index 40499b44ea..4c82f157bc 100644
--- a/src/inc/CMakeLists.txt
+++ b/src/inc/CMakeLists.txt
@@ -64,6 +64,10 @@ endforeach(IDL_SOURCE)
 add_compile_options(-fPIC)
 endif(WIN32)
 
+if(FEATURE_JIT_PITCHING)
+   add_definitions(-DFEATURE_JIT_PITCHING)
+endif(FEATURE_JIT_PITCHING)
+
 # Compile *_i.cpp to lib
 _add_library(corguids ${CORGUIDS_SOURCES})
 
diff --git a/src/inc/clrconfigvalues.h b/src/inc/clrconfigvalues.h
index 8a21a9d8fd..ce5a95d508 100644
--- a/src/inc/clrconfigvalues.h
+++ b/src/inc/clrconfigvalues.h
@@ -134,6 +134,17 @@ RETAIL_CONFIG_DWORD_INFO(EXTERNAL_FinalizeOnShutdown, W("FinalizeOnShutdown"), D
 RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_ARMEnabled, W("ARMEnabled"), (DWORD)0, "Set it to 1 to enable ARM")
 
 //
+// Jit Pitching
+//
+RETAIL_CONFIG_DWORD_INFO(INTERNAL_JitPitchEnabled, W("JitPitchEnabled"), (DWORD)0, "Set it to 1 to enable Jit Pitching")
+RETAIL_CONFIG_DWORD_INFO(INTERNAL_JitPitchMemThreshold, W("JitPitchMemThreshold"), (DWORD)0, "Do Jit Pitching when code heap usage is larger than this (in bytes)")
+RETAIL_CONFIG_DWORD_INFO(INTERNAL_JitPitchMethodSizeThreshold, W("JitPitchMethodSizeThreshold"), (DWORD)0, "Do Jit Pitching for methods whose native code size larger than this (in bytes)")
+RETAIL_CONFIG_DWORD_INFO(INTERNAL_JitPitchTimeInterval, W("JitPitchTimeInterval"), (DWORD)0, "Time interval between Jit Pitchings in ms")
+RETAIL_CONFIG_DWORD_INFO(INTERNAL_JitPitchPrintStat, W("JitPitchPrintStat"), (DWORD)0, "Print statistics about Jit Pitching")
+RETAIL_CONFIG_DWORD_INFO(INTERNAL_JitPitchMinVal, W("JitPitchMinVal"), (DWORD)0, "Do Jit Pitching if the value of the inner counter greater than this value (for debugging purpose only)")
+RETAIL_CONFIG_DWORD_INFO(INTERNAL_JitPitchMaxVal, W("JitPitchMaxVal"), (DWORD)0xffffffff, "Do Jit Pitching the value of the inner counter less then this value (for debuggin purpose only)")
+
+//
 // Assembly Loader
 //
 RETAIL_CONFIG_DWORD_INFO_EX(EXTERNAL_DesignerNamespaceResolutionEnabled, W("designerNamespaceResolution"), FALSE, "Set it to 1 to enable DesignerNamespaceResolve event for WinRT types", CLRConfig::IgnoreEnv | CLRConfig::IgnoreHKLM | CLRConfig::IgnoreHKCU | CLRConfig::FavorConfigFile)
diff --git a/src/vm/CMakeLists.txt b/src/vm/CMakeLists.txt
index adb8409558..8bb2292816 100644
--- a/src/vm/CMakeLists.txt
+++ b/src/vm/CMakeLists.txt
@@ -33,6 +33,10 @@ if(FEATURE_GDBJIT)
     add_definitions(-DFEATURE_GDBJIT)
 endif(FEATURE_GDBJIT)
 
+if(FEATURE_JIT_PITCHING)
+   add_definitions(-DFEATURE_JIT_PITCHING)
+endif(FEATURE_JIT_PITCHING)
+
 set(VM_SOURCES_DAC_AND_WKS_COMMON
     appdomain.cpp
     array.cpp
@@ -137,6 +141,12 @@ if(FEATURE_READYTORUN)
     )
 endif(FEATURE_READYTORUN)
 
+if(FEATURE_JIT_PITCHING)
+    list(APPEND VM_SOURCES_DAC_AND_WKS_COMMON
+        codepitchingmanager.cpp
+    )
+endif(FEATURE_JIT_PITCHING)
+
 set(VM_SOURCES_DAC
     ${VM_SOURCES_DAC_AND_WKS_COMMON}
     contexts.cpp
diff --git a/src/vm/codeman.cpp b/src/vm/codeman.cpp
index d934b824f6..1e0cebef72 100644
--- a/src/vm/codeman.cpp
+++ b/src/vm/codeman.cpp
@@ -2186,7 +2186,7 @@ HeapList* EEJitManager::NewCodeHeap(CodeHeapRequestInfo *pInfo, DomainCodeHeapLi
     } CONTRACT_END;
 
     size_t initialRequestSize = pInfo->getRequestSize();
-    size_t minReserveSize = VIRTUAL_ALLOC_RESERVE_GRANULARITY; //     ( 64 KB)           
+    size_t minReserveSize = VIRTUAL_ALLOC_RESERVE_GRANULARITY; //     ( 64 KB)
 
 #ifdef _WIN64
     if (pInfo->m_hiAddr == 0)
@@ -2390,12 +2390,12 @@ void* EEJitManager::allocCodeRaw(CodeHeapRequestInfo *pInfo,
     {
         // Let us create a new heap.
 
-        DomainCodeHeapList *pList = GetCodeHeapList(pInfo->m_pMD, pInfo->m_pAllocator);
+        DomainCodeHeapList *pList = GetCodeHeapList(pInfo, pInfo->m_pAllocator);
         if (pList == NULL)
         {
             // not found so need to create the first one
             pList = CreateCodeHeapList(pInfo);
-            _ASSERTE(pList == GetCodeHeapList(pInfo->m_pMD, pInfo->m_pAllocator));
+            _ASSERTE(pList == GetCodeHeapList(pInfo, pInfo->m_pAllocator));
         }
         _ASSERTE(pList);
 
@@ -2478,23 +2478,29 @@ CodeHeader* EEJitManager::allocCode(MethodDesc* pMD, size_t blockSize, CorJitAll
 
     SIZE_T totalSize = blockSize;
 
+    CodeHeader * pCodeHdr = NULL;
+
+    CodeHeapRequestInfo requestInfo(pMD);
+#if defined(FEATURE_JIT_PITCHING)
+    if (pMD && pMD->IsPitchable() && CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchMethodSizeThreshold) < blockSize)
+    {
+        requestInfo.SetDynamicDomain();
+    }
+#endif
+
 #if defined(USE_INDIRECT_CODEHEADER)
     SIZE_T realHeaderSize = offsetof(RealCodeHeader, unwindInfos[0]) + (sizeof(T_RUNTIME_FUNCTION) * nUnwindInfos); 
 
     // if this is a LCG method then we will be allocating the RealCodeHeader
     // following the code so that the code block can be removed easily by 
     // the LCG code heap.
-    if (pMD->IsLCGMethod())
+    if (requestInfo.IsDynamicDomain())
     {
         totalSize = ALIGN_UP(totalSize, sizeof(void*)) + realHeaderSize;
         static_assert_no_msg(CODE_SIZE_ALIGN >= sizeof(void*));
     }
 #endif  // USE_INDIRECT_CODEHEADER
 
-    CodeHeader * pCodeHdr = NULL;
-
-    CodeHeapRequestInfo requestInfo(pMD);
-
     // Scope the lock
     {
         CrstHolder ch(&m_CodeHeapCritSec);
@@ -2521,7 +2527,7 @@ CodeHeader* EEJitManager::allocCode(MethodDesc* pMD, size_t blockSize, CorJitAll
         pCodeHdr = ((CodeHeader *)pCode) - 1;
 
 #ifdef USE_INDIRECT_CODEHEADER
-        if (pMD->IsLCGMethod())
+        if (requestInfo.IsDynamicDomain())
         {
             pCodeHdr->SetRealCodeHeader((BYTE*)pCode + ALIGN_UP(blockSize, sizeof(void*)));
         }
@@ -2550,7 +2556,7 @@ CodeHeader* EEJitManager::allocCode(MethodDesc* pMD, size_t blockSize, CorJitAll
     RETURN(pCodeHdr);
 }
 
-EEJitManager::DomainCodeHeapList *EEJitManager::GetCodeHeapList(MethodDesc *pMD, LoaderAllocator *pAllocator, BOOL fDynamicOnly)
+EEJitManager::DomainCodeHeapList *EEJitManager::GetCodeHeapList(CodeHeapRequestInfo *pInfo, LoaderAllocator *pAllocator, BOOL fDynamicOnly)
 {
     CONTRACTL {
         NOTHROW;
@@ -2564,7 +2570,7 @@ EEJitManager::DomainCodeHeapList *EEJitManager::GetCodeHeapList(MethodDesc *pMD,
 
     // get the appropriate list of heaps
     // pMD is NULL for NGen modules during Module::LoadTokenTables
-    if (fDynamicOnly || (pMD != NULL && pMD->IsLCGMethod()))
+    if (fDynamicOnly || (pInfo != NULL && pInfo->IsDynamicDomain()))
     {
         ppList = m_DynamicDomainCodeHeaps.Table();
         count = m_DynamicDomainCodeHeaps.Count();
@@ -2605,7 +2611,7 @@ HeapList* EEJitManager::GetCodeHeap(CodeHeapRequestInfo *pInfo)
 
     // loop through the m_DomainCodeHeaps to find the AppDomain
     // if not found, then create it
-    DomainCodeHeapList *pList = GetCodeHeapList(pInfo->m_pMD, pInfo->m_pAllocator);
+    DomainCodeHeapList *pList = GetCodeHeapList(pInfo, pInfo->m_pAllocator);
     if (pList)
     {
         // Set pResult to the largest non-full HeapList
@@ -2726,7 +2732,7 @@ bool EEJitManager::CanUseCodeHeap(CodeHeapRequestInfo *pInfo, HeapList *pCodeHea
        }
    }
 
-   return retVal; 
+   return retVal;
 }
 
 EEJitManager::DomainCodeHeapList * EEJitManager::CreateCodeHeapList(CodeHeapRequestInfo *pInfo)
diff --git a/src/vm/codeman.h b/src/vm/codeman.h
index f85eeb59db..afef682e2a 100644
--- a/src/vm/codeman.h
+++ b/src/vm/codeman.h
@@ -369,6 +369,8 @@ struct CodeHeapRequestInfo
     bool         m_isCollectible;
     
     bool   IsDynamicDomain()                    { return m_isDynamicDomain;    }
+    void   SetDynamicDomain()                   { m_isDynamicDomain = true;    }
+
     bool   IsCollectible()                      { return m_isCollectible;      }
     
     size_t getRequestSize()                     { return m_requestSize;        }
@@ -1095,7 +1097,7 @@ private :
                              size_t header, size_t blockSize, unsigned align,
                              HeapList ** ppCodeHeap /* Writeback, Can be null */ );
 
-    DomainCodeHeapList *GetCodeHeapList(MethodDesc *pMD, LoaderAllocator *pAllocator, BOOL fDynamicOnly = FALSE);
+    DomainCodeHeapList *GetCodeHeapList(CodeHeapRequestInfo *pInfo, LoaderAllocator *pAllocator, BOOL fDynamicOnly = FALSE);
     DomainCodeHeapList *CreateCodeHeapList(CodeHeapRequestInfo *pInfo);
     LoaderHeap* GetJitMetaHeap(MethodDesc *pMD);
 #endif // !CROSSGEN_COMPILE
diff --git a/src/vm/codepitchingmanager.cpp b/src/vm/codepitchingmanager.cpp
new file mode 100644
index 0000000000..521c101b0f
--- /dev/null
+++ b/src/vm/codepitchingmanager.cpp
@@ -0,0 +1,522 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+// ===========================================================================
+// File: CodePitchingManager.cpp
+//
+
+// ===========================================================================
+// This file contains the implementation for code pitching.
+// Its distinctive features and algorithm are:
+//
+// 1. All its code is under #if defined(FEATURE_JIT_PITCHING) and doesn't mess up with other code
+// 2. This feature is working only if the options INTERNAL_JitPitchEnabled != 0 and INTERNAL_JitPitchMemThreshold > 0
+// 3. Jitted code can be pitched only for methods that are not Dynamic, FCall or Virtual
+// 4. If the size of the generated native code exceeds the value of INTERNAL_JitDPitchMethodSizeThreshold this code is
+//    placed in the special heap code list. Each heap block in this list stores the code for only one method and has the
+//    sufficient size for the code of a method aligned to 4K. The pointers to such methods are stored in the
+//    "PitchingCandidateMethods" hash map.
+// 5. If the entrypoint of a method is backpatched this method is excluded from the "PitchingCandidateMethods" hash map
+//    and stored in "NotForPitchingMethods" hashmap.
+// 6. When the total size of the generated native code exceeds the value of INTERNAL_JitPitchMemThreshold option, the
+//    execution of the program is stopped and stack frames for all the threads are inspected and pointers to methods
+//    being executed are stored in the "ExecutedMethods" hash map
+// 7. The code for all the methods from the "PitchingCandidateMethods" that are not in the "ExecutedMethods" is pitched.
+//    (All heap blocks for these methods are set in the initial state and can be reused for newly compiled methods, pointers
+//     to the code for non-executed methods are set to nullptr).
+// 8. If the code for the given method is pitched once, this method is stored in the "NotForPitchingMethods" hashmap. Thus,
+//    if this method is compiled the second time, it is considered as called repeatedly, therefore, pitching for it is inexpedient,
+//    and the newly compiled code stored in the usual heap.
+// 9. The coreclr code with this feature is built by the option
+//     ./build.sh cmakeargs -DFEATURE_JIT_PITCHING=true
+// ===========================================================================
+
+#include "common.h"
+
+#ifndef DACCESS_COMPILE
+
+#if defined(FEATURE_JIT_PITCHING)
+
+#include "nibblemapmacros.h"
+#include "threadsuspend.h"
+
+static PtrHashMap* s_pPitchingCandidateMethods = nullptr;
+static PtrHashMap* s_pPitchingCandidateSizes = nullptr;
+static SimpleRWLock* s_pPitchingCandidateMethodsLock = nullptr;
+
+static PtrHashMap* s_pExecutedMethods = nullptr;
+static SimpleRWLock* s_pExecutedMethodsLock = nullptr;
+
+static PtrHashMap* s_pNotForPitchingMethods = nullptr;
+static SimpleRWLock* s_pNotForPitchingMethodsLock = nullptr;
+
+#ifdef _DEBUG
+static PtrHashMap* s_pPitchedMethods = nullptr;
+static SimpleRWLock* s_pPitchedMethodsLock = nullptr;
+#endif
+
+static ULONG s_totalNCSize = 0;
+static SimpleRWLock* s_totalNCSizeLock = nullptr;
+
+static ULONG s_jitPitchedBytes = 0;
+
+static INT64 s_JitPitchLastTick = 0;
+
+static bool s_JitPitchInitialized = false;
+
+
+static BOOL IsOwnerOfRWLock(LPVOID lock)
+{
+    // @TODO - SimpleRWLock does not have knowledge of which thread gets the writer
+    // lock, so no way to verify
+    return TRUE;
+}
+
+static void CreateRWLock(SimpleRWLock** lock)
+{
+    if (*lock == nullptr)
+    {
+        void *pLockSpace = SystemDomain::GetGlobalLoaderAllocator()->GetLowFrequencyHeap()->AllocMem(S_SIZE_T(sizeof(SimpleRWLock)));
+        SimpleRWLock *pLock = new (pLockSpace) SimpleRWLock(COOPERATIVE_OR_PREEMPTIVE, LOCK_TYPE_DEFAULT);
+
+        if (FastInterlockCompareExchangePointer(lock, pLock, NULL) != NULL)
+            SystemDomain::GetGlobalLoaderAllocator()->GetLowFrequencyHeap()->BackoutMem(pLockSpace, sizeof(SimpleRWLock));
+    }
+}
+
+static PtrHashMap* CreateHashMap(SimpleRWLock* rwLock)
+{
+    PtrHashMap *pMap = new (SystemDomain::GetGlobalLoaderAllocator()->GetLowFrequencyHeap()) PtrHashMap();
+    LockOwner lock = {rwLock, IsOwnerOfRWLock};
+    pMap->Init(32, nullptr, FALSE, &lock);
+    return pMap;
+}
+
+static void InitializeJitPitching()
+{
+    if (!s_JitPitchInitialized)
+    {
+        CreateRWLock(&s_pNotForPitchingMethodsLock);
+        CreateRWLock(&s_pPitchingCandidateMethodsLock);
+        CreateRWLock(&s_totalNCSizeLock);
+
+        {
+            SimpleReadLockHolder srlh(s_pNotForPitchingMethodsLock);
+            if (s_pNotForPitchingMethods == nullptr)
+            {
+                s_pNotForPitchingMethods = CreateHashMap(s_pNotForPitchingMethodsLock);
+            }
+        }
+
+        {
+            SimpleReadLockHolder srlh(s_pPitchingCandidateMethodsLock);
+            if (s_pPitchingCandidateMethods == nullptr)
+            {
+                s_pPitchingCandidateMethods = CreateHashMap(s_pPitchingCandidateMethodsLock);
+                s_pPitchingCandidateSizes = CreateHashMap(s_pPitchingCandidateMethodsLock);
+            }
+        }
+
+        s_JitPitchInitialized = true;
+    }
+}
+
+static COUNT_T GetFullHash(MethodDesc* pMD)
+{
+    const char *moduleName = pMD->GetModule()->GetSimpleName();
+
+    COUNT_T hash = HashStringA(moduleName);         // Start the hash with the Module name
+
+    SString className, methodName, methodSig;
+
+    pMD->GetMethodInfo(className, methodName, methodSig);
+
+    hash = HashCOUNT_T(hash, className.Hash());   // Hash in the name of the Class name
+    hash = HashCOUNT_T(hash, methodName.Hash());  // Hash in the name of the Method name
+    hash = HashCOUNT_T(hash, 0xffffffff & (ULONGLONG)pMD);
+
+    return hash;
+}
+
+bool MethodDesc::IsPitchable()
+{
+    if ((CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchEnabled) == 0) ||
+        (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchMemThreshold) == 0))
+        return FALSE;
+
+    InitializeJitPitching();
+
+    if (IsLCGMethod() || IsVtableMethod() || IsInterface() || IsVirtual())
+        return FALSE;
+
+    _ASSERTE(s_pNotForPitchingMethodsLock != nullptr && s_pNotForPitchingMethods != nullptr);
+
+    {
+        SimpleReadLockHolder srlh(s_pNotForPitchingMethodsLock);
+        UPTR key = (UPTR)GetFullHash(this);
+        MethodDesc *pFound = (MethodDesc *)s_pNotForPitchingMethods->LookupValue(key, (LPVOID)this);
+        if (pFound != (MethodDesc *)INVALIDENTRY)
+        {
+            return FALSE;
+        }
+    }
+    return TRUE;
+}
+
+EXTERN_C bool LookupOrCreateInNotForPitching(MethodDesc* pMD)
+{
+    CONTRACTL
+    {
+        MODE_COOPERATIVE;
+        GC_TRIGGERS;
+        THROWS;
+    }
+    CONTRACTL_END;
+
+    if (pMD != nullptr && pMD->IsPitchable())
+    {
+        UPTR key = (UPTR)GetFullHash(pMD);
+
+        _ASSERTE(s_pNotForPitchingMethodsLock != nullptr && s_pNotForPitchingMethods != nullptr);
+
+        {
+             SimpleReadLockHolder srlh(s_pNotForPitchingMethodsLock);
+             MethodDesc *pFound = (MethodDesc *)s_pNotForPitchingMethods->LookupValue(key, (LPVOID)pMD);
+             if (pFound != (MethodDesc *)INVALIDENTRY)
+                 return TRUE;
+        }
+
+        {
+             SimpleWriteLockHolder swlh(s_pNotForPitchingMethodsLock);
+             s_pNotForPitchingMethods->InsertValue(key, (LPVOID)pMD);
+        }
+    }
+    return FALSE;
+}
+
+static void LookupOrCreateInPitchingCandidate(MethodDesc* pMD, ULONG sizeOfCode)
+{
+    CONTRACTL
+    {
+        MODE_COOPERATIVE;
+        GC_TRIGGERS;
+        THROWS;
+    }
+    CONTRACTL_END;
+
+    if (pMD == nullptr || !pMD->IsPitchable())
+        return;
+
+    PCODE prCode = pMD->GetPreImplementedCode();
+    if (prCode)
+        return;
+
+    if (!pMD->HasPrecode())
+        return;
+
+    UPTR key = (UPTR)GetFullHash(pMD);
+
+    _ASSERTE(s_pPitchingCandidateMethodsLock != nullptr && s_pPitchingCandidateMethods != nullptr);
+    _ASSERTE(s_pPitchingCandidateSizes);
+
+    {
+        // Try getting an existing value first.
+        SimpleReadLockHolder srlh(s_pPitchingCandidateMethodsLock);
+        MethodDesc *pFound = (MethodDesc *)s_pPitchingCandidateMethods->LookupValue(key, (LPVOID)pMD);
+        if (pFound != (MethodDesc *)INVALIDENTRY)
+            return;
+    }
+
+    {
+        SimpleWriteLockHolder swlh(s_pPitchingCandidateMethodsLock);
+        s_pPitchingCandidateMethods->InsertValue(key, (LPVOID)pMD);
+        s_pPitchingCandidateSizes->InsertValue(key, (LPVOID)((ULONGLONG)(sizeOfCode << 1)));
+#ifdef _DEBUG
+        if (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchPrintStat) != 0)
+        {
+            SString className, methodName, methodSig;
+            pMD->GetMethodInfo(className, methodName, methodSig);
+
+            StackScratchBuffer scratch;
+            const char* szClassName = className.GetUTF8(scratch);
+            const char* szMethodSig = methodSig.GetUTF8(scratch);
+
+            printf("Candidate %lld %s :: %s %s\n",
+                   sizeOfCode, szClassName, pMD->GetName(), szMethodSig);
+        }
+#endif
+    }
+}
+
+EXTERN_C void DeleteFromPitchingCandidate(MethodDesc* pMD)
+{
+    CONTRACTL
+    {
+        MODE_COOPERATIVE;
+        GC_TRIGGERS;
+        THROWS;
+    }
+    CONTRACTL_END;
+
+    if (pMD != nullptr && pMD->IsPitchable())
+    {
+        PCODE pCode = pMD->GetPreImplementedCode();
+
+        if (pCode)
+           return;
+
+        _ASSERTE(s_pPitchingCandidateMethodsLock != nullptr && s_pPitchingCandidateMethods != nullptr);
+        _ASSERTE(s_pPitchingCandidateSizes != nullptr);
+
+        UPTR key = (UPTR)GetFullHash(pMD);
+        {
+            SimpleReadLockHolder srlh(s_pPitchingCandidateMethodsLock);
+            MethodDesc *pFound = (MethodDesc *)s_pPitchingCandidateMethods->LookupValue(key, (LPVOID)pMD);
+            if (pFound == (MethodDesc *)INVALIDENTRY)
+                return;
+        }
+
+        {
+            SimpleWriteLockHolder swlh(s_pPitchingCandidateMethodsLock);
+            s_pPitchingCandidateMethods->DeleteValue(key, (LPVOID)pMD);
+        }
+
+        LPVOID pitchedBytes;
+        {
+            SimpleReadLockHolder srlh(s_pPitchingCandidateMethodsLock);
+            pitchedBytes = s_pPitchingCandidateSizes->LookupValue(key, nullptr);
+            _ASSERTE(pitchedBytes != (LPVOID)INVALIDENTRY);
+        }
+        {
+            SimpleWriteLockHolder swlh(s_pPitchingCandidateMethodsLock);
+            s_pPitchingCandidateSizes->DeleteValue(key, pitchedBytes);
+        }
+    }
+}
+
+EXTERN_C void MarkMethodNotPitchingCandidate(MethodDesc* pMD)
+{
+
+    DeleteFromPitchingCandidate(pMD);
+    (void)LookupOrCreateInNotForPitching(pMD);
+}
+
+StackWalkAction CrawlFrameVisitor(CrawlFrame* pCf, Thread* pMdThread)
+{
+    CONTRACTL
+    {
+        THROWS;
+        GC_NOTRIGGER;
+        SO_TOLERANT;
+        MODE_ANY;
+    }
+    CONTRACTL_END;
+
+    MethodDesc* pMD = pCf->GetFunction();
+
+    // Filter out methods we don't care about
+    if (pMD == nullptr || !pMD->IsPitchable())
+    {
+        return SWA_CONTINUE;
+    }
+
+    if (s_pExecutedMethods == nullptr)
+    {
+        PtrHashMap *pMap = new (SystemDomain::GetGlobalLoaderAllocator()->GetLowFrequencyHeap()) PtrHashMap();
+        pMap->Init(TRUE, nullptr);
+        s_pExecutedMethods = pMap;
+    }
+
+    UPTR key = (UPTR)GetFullHash(pMD);
+    MethodDesc *pFound = (MethodDesc *)s_pExecutedMethods->LookupValue(key, (LPVOID)pMD);
+    if (pFound == (MethodDesc *)INVALIDENTRY)
+    {
+        s_pExecutedMethods->InsertValue(key, (LPVOID)pMD);
+    }
+
+    return SWA_CONTINUE;
+}
+
+// Visitor for stack walk callback.
+StackWalkAction StackWalkCallback(CrawlFrame* pCf, VOID* data)
+{
+    WRAPPER_NO_CONTRACT;
+
+    // WalkInfo* info = (WalkInfo*) data;
+    return CrawlFrameVisitor(pCf, (Thread *)data);
+}
+
+static ULONGLONG s_PitchedMethodCounter = 0;
+void MethodDesc::PitchNativeCode()
+{
+    WRAPPER_NO_CONTRACT;
+    SUPPORTS_DAC;
+
+    g_IBCLogger.LogMethodDescAccess(this);
+
+    if (!IsPitchable())
+        return;
+
+    PCODE pCode = GetNativeCode();
+
+    if (!pCode)
+        return;
+
+    _ASSERTE(HasPrecode());
+
+    _ASSERTE(HasNativeCode());
+
+    ++s_PitchedMethodCounter;
+
+    if (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchMinVal) > s_PitchedMethodCounter)
+    {
+        return;
+    }
+    if (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchMaxVal) < s_PitchedMethodCounter)
+    {
+        return;
+    }
+
+    if (LookupOrCreateInNotForPitching(this))
+        return;
+
+    MethodTable * pMT = GetMethodTable();
+    _ASSERTE(pMT != nullptr);
+
+    CodeHeader* pCH = ((CodeHeader*)(pCode & ~1)) - 1;
+    _ASSERTE(pCH->GetMethodDesc() == this);
+
+    HostCodeHeap* pHeap = HostCodeHeap::GetCodeHeap((TADDR)pCode);
+    pHeap->GetJitManager()->FreeCodeMemory(pHeap, (void*)pCode);
+
+    ClearFlagsOnUpdate();
+
+    _ASSERTE(HasPrecode());
+    GetPrecode()->Reset();
+
+    if (HasNativeCodeSlot())
+    {
+        RelativePointer<TADDR> *pRelPtr = (RelativePointer<TADDR> *)GetAddrOfNativeCodeSlot();
+        pRelPtr->SetValueMaybeNull(NULL);
+    }
+    else
+    {
+#ifdef FEATURE_INTERPRETER
+        SetNativeCodeInterlocked(NULL, NULL, FALSE);
+#else
+        SetNativeCodeInterlocked(NULL, NULL);
+#endif
+    }
+
+    _ASSERTE(!HasNativeCode());
+
+    UPTR key = (UPTR)GetFullHash(this);
+    ULONGLONG pitchedBytes;
+    {
+        SimpleReadLockHolder srlh(s_pPitchingCandidateMethodsLock);
+        pitchedBytes = (ULONGLONG)s_pPitchingCandidateSizes->LookupValue(key, nullptr);
+        _ASSERTE(pitchedBytes != (ULONGLONG)INVALIDENTRY);
+        if (pitchedBytes == (ULONGLONG)INVALIDENTRY)
+            pitchedBytes = 0;
+        s_jitPitchedBytes += (pitchedBytes >> 1);
+    }
+    {
+        SimpleWriteLockHolder swlh(s_pPitchingCandidateMethodsLock);
+        s_pPitchingCandidateMethods->DeleteValue(key, (LPVOID)this);
+        if (pitchedBytes != 0)
+            s_pPitchingCandidateSizes->DeleteValue(key, (LPVOID)pitchedBytes);
+    }
+
+    if (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchPrintStat) != 0)
+    {
+        SString className, methodName, methodSig;
+        GetMethodInfo(className, methodName, methodSig);
+
+        StackScratchBuffer scratch;
+        const char* szClassName = className.GetUTF8(scratch);
+        const char* szMethodSig = methodSig.GetUTF8(scratch);
+
+        if (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchPrintStat) != 0)
+        {
+            printf("Pitched %lld %lld %s :: %s %s\n",
+                   s_PitchedMethodCounter, pitchedBytes, szClassName, GetName(), szMethodSig);
+        }
+    }
+
+    DACNotify::DoJITPitchingNotification(this);
+}
+
+EXTERN_C void CheckStacksAndPitch()
+{
+    if ((CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchEnabled) != 0) &&
+        (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchMemThreshold) != 0) &&
+        (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchTimeInterval) == 0 ||
+         ((::GetTickCount64() - s_JitPitchLastTick) > CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchTimeInterval))))
+    {
+        SimpleReadLockHolder srlh(s_totalNCSizeLock);
+
+        if ((s_totalNCSize - s_jitPitchedBytes) > CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchMemThreshold) &&
+            s_pPitchingCandidateMethods != nullptr)
+        {
+            EX_TRY
+            {
+                // Suspend the runtime.
+                ThreadSuspend::SuspendEE(ThreadSuspend::SUSPEND_OTHER);
+
+                // Walk all other threads.
+                Thread* pThread = nullptr;
+                while ((pThread = ThreadStore::GetThreadList(pThread)) != nullptr)
+                {
+                    pThread->StackWalkFrames(StackWalkCallback, (VOID *)pThread, ALLOW_ASYNC_STACK_WALK);
+                }
+
+                if (s_pExecutedMethods)
+                {
+                    PtrHashMap::PtrIterator i = s_pPitchingCandidateMethods->begin();
+                    while (!i.end())
+                    {
+                        MethodDesc *pMD = (MethodDesc *) i.GetValue();
+                        UPTR key = (UPTR)GetFullHash(pMD);
+                        MethodDesc *pFound = (MethodDesc *)s_pExecutedMethods->LookupValue(key, (LPVOID)pMD);
+                        ++i;
+                        if (pFound == (MethodDesc *)INVALIDENTRY)
+                        {
+                            pMD->PitchNativeCode();
+                        }
+                    }
+                    s_pExecutedMethods->Clear();
+                    delete s_pExecutedMethods;
+                    s_pExecutedMethods = nullptr;
+                    s_pPitchingCandidateMethods->Compact();
+                    s_pPitchingCandidateSizes->Compact();
+                }
+
+                s_JitPitchLastTick = ::GetTickCount64();
+
+                ThreadSuspend::RestartEE(FALSE, TRUE);
+            }
+            EX_CATCH
+            {
+            }
+            EX_END_CATCH(SwallowAllExceptions);
+        }
+    }
+}
+
+EXTERN_C void SavePitchingCandidate(MethodDesc* pMD, ULONG sizeOfCode)
+{
+    if (pMD && pMD->IsPitchable() && CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchMethodSizeThreshold) < sizeOfCode)
+    {
+        LookupOrCreateInPitchingCandidate(pMD, sizeOfCode);
+    }
+    if (sizeOfCode > 0)
+    {
+        SimpleWriteLockHolder swlh(s_totalNCSizeLock);
+        s_totalNCSize += sizeOfCode;
+        if (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchPrintStat) != 0)
+            printf("jitted %lld (bytes) pitched %lld (bytes)\n", s_totalNCSize, s_jitPitchedBytes);
+    }
+}
+#endif
+
+#endif
diff --git a/src/vm/dynamicmethod.h b/src/vm/dynamicmethod.h
index f9a92b0af0..7fd63e59b9 100644
--- a/src/vm/dynamicmethod.h
+++ b/src/vm/dynamicmethod.h
@@ -287,7 +287,7 @@ private:
 public:
     // Space for header is reserved immediately before. It is not included in size.
     virtual void* AllocMemForCode_NoThrow(size_t header, size_t size, DWORD alignment) DAC_EMPTY_RET(NULL);
-    
+
     virtual ~HostCodeHeap() DAC_EMPTY();
 
     LoaderAllocator* GetAllocator() { return m_pAllocator; }
@@ -307,6 +307,11 @@ protected:
 
     void FreeMemForCode(void * codeStart);
 
+#if defined(FEATURE_JIT_PITCHING)
+public:
+    PTR_EEJitManager GetJitManager() { return m_pJitManager; }
+#endif
+
 }; // class HostCodeHeap
 
 //---------------------------------------------------------------------------------------
diff --git a/src/vm/gdbjit.cpp b/src/vm/gdbjit.cpp
index 1857f60407..8e2d1e8279 100644
--- a/src/vm/gdbjit.cpp
+++ b/src/vm/gdbjit.cpp
@@ -2153,7 +2153,7 @@ void NotifyGdb::OnMethodCompiled(MethodDesc* methodDescPtr)
     __jit_debug_register_code();
 }
 
-void NotifyGdb::MethodDropped(MethodDesc* methodDescPtr)
+void NotifyGdb::MethodPitched(MethodDesc* methodDescPtr)
 {
     static const int textSectionIndex = GetSectionIndex(".text");
 
diff --git a/src/vm/gdbjit.h b/src/vm/gdbjit.h
index 6bde3f27ba..be7249f22a 100644
--- a/src/vm/gdbjit.h
+++ b/src/vm/gdbjit.h
@@ -331,7 +331,7 @@ class NotifyGdb
 {
 public:
     static void MethodCompiled(MethodDesc* methodDescPtr);
-    static void MethodDropped(MethodDesc* methodDescPtr);
+    static void MethodPitched(MethodDesc* methodDescPtr);
     template <typename PARENT_TRAITS>
     class DeleteValuesOnDestructSHashTraits : public PARENT_TRAITS
     {
diff --git a/src/vm/hosting.cpp b/src/vm/hosting.cpp
index d47bc28238..035fff8812 100644
--- a/src/vm/hosting.cpp
+++ b/src/vm/hosting.cpp
@@ -480,12 +480,15 @@ BOOL EEHeapFree(HANDLE hHeap, DWORD dwFlags, LPVOID lpMem)
 #ifdef _DEBUG
         GlobalAllocStore::RemoveAlloc (lpMem);
 
-        // Check the heap handle to detect heap contamination
-        lpMem = (BYTE*)lpMem - OS_HEAP_ALIGN;
-        HANDLE storedHeapHandle = *((HANDLE*)lpMem);
-        if(storedHeapHandle != hHeap)
-            _ASSERTE(!"Heap contamination detected! HeapFree was called on a heap other than the one that memory was allocated from.\n"
-                      "Possible cause: you used new (executable) to allocate the memory, but didn't use DeleteExecutable() to free it.");
+        if (lpMem != NULL)
+        {
+            // Check the heap handle to detect heap contamination
+            lpMem = (BYTE*)lpMem - OS_HEAP_ALIGN;
+            HANDLE storedHeapHandle = *((HANDLE*)lpMem);
+            if(storedHeapHandle != hHeap)
+                _ASSERTE(!"Heap contamination detected! HeapFree was called on a heap other than the one that memory was allocated from.\n"
+                         "Possible cause: you used new (executable) to allocate the memory, but didn't use DeleteExecutable() to free it.");
+        }
 #endif
         // DON'T REMOVE THIS SEEMINGLY USELESS CAST
         //
diff --git a/src/vm/method.hpp b/src/vm/method.hpp
index 71e838a94a..36a23716b4 100644
--- a/src/vm/method.hpp
+++ b/src/vm/method.hpp
@@ -1318,6 +1318,12 @@ public:
     BOOL IsNativeCodeStableAfterInit()
     {
         LIMITED_METHOD_DAC_CONTRACT;
+
+#if defined(FEATURE_JIT_PITCHING)
+        if (IsPitchable())
+            return false;
+#endif
+
         return 
 #ifdef FEATURE_TIERED_COMPILATION
             !IsEligibleForTieredCompilation() &&
@@ -1434,6 +1440,11 @@ public:
     // - ngened code if IsPreImplemented()
     PCODE GetNativeCode();
 
+#if defined(FEATURE_JIT_PITCHING)
+    bool IsPitchable();
+    void PitchNativeCode();
+#endif
+
     //================================================================
     // FindOrCreateAssociatedMethodDesc
     //
diff --git a/src/vm/methodtablebuilder.cpp b/src/vm/methodtablebuilder.cpp
index bd9dd24e2e..c185ba112a 100644
--- a/src/vm/methodtablebuilder.cpp
+++ b/src/vm/methodtablebuilder.cpp
@@ -6834,6 +6834,12 @@ MethodTableBuilder::NeedsNativeCodeSlot(bmtMDMethod * pMDMethod)
     }
 #endif
 
+#if defined(FEATURE_JIT_PITCHING)
+    if ((CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchEnabled) != 0) &&
+        (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchMemThreshold) != 0))
+        return TRUE;
+#endif
+
     return GetModule()->IsEditAndContinueEnabled();
 }
 
diff --git a/src/vm/prestub.cpp b/src/vm/prestub.cpp
index 31a5670e00..0c01efd64e 100644
--- a/src/vm/prestub.cpp
+++ b/src/vm/prestub.cpp
@@ -52,7 +52,14 @@
 #include "callcounter.h"
 #endif
 
-#ifndef DACCESS_COMPILE 
+#ifndef DACCESS_COMPILE
+
+#if defined(FEATURE_JIT_PITCHING)
+EXTERN_C void CheckStacksAndPitch();
+EXTERN_C void SavePitchingCandidate(MethodDesc* pMD, ULONG sizeOfCode);
+EXTERN_C void DeleteFromPitchingCandidate(MethodDesc* pMD);
+EXTERN_C void MarkMethodNotPitchingCandidate(MethodDesc* pMD);
+#endif
 
 EXTERN_C void STDCALL ThePreStub();
 
@@ -262,7 +269,6 @@ void DACNotifyCompilationFinished(MethodDesc *methodDesc)
 // This function creates a DeadlockAware list of methods being jitted
 // which prevents us from trying to JIT the same method more that once.
 
-
 PCODE MethodDesc::MakeJitWorker(COR_ILMETHOD_DECODER* ILHeader, CORJIT_FLAGS flags)
 {
     STANDARD_VM_CONTRACT;
@@ -276,6 +282,10 @@ PCODE MethodDesc::MakeJitWorker(COR_ILMETHOD_DECODER* ILHeader, CORJIT_FLAGS fla
          GetMethodTable()->GetDebugClassName(),
          m_pszDebugMethodName));
 
+#if defined(FEATURE_JIT_PITCHING)
+    CheckStacksAndPitch();
+#endif
+
     PCODE pCode = NULL;
     ULONG sizeOfCode = 0;
 #if defined(FEATURE_INTERPRETER) || defined(FEATURE_TIERED_COMPILATION)
@@ -548,7 +558,7 @@ PCODE MethodDesc::MakeJitWorker(COR_ILMETHOD_DECODER* ILHeader, CORJIT_FLAGS fla
 #endif // FEATURE_INTERPRETER
 
 #ifdef FEATURE_MULTICOREJIT
-            
+
             // If called from multi-core JIT background thread, store code under lock, delay patching until code is queried from application threads
             if (fBackgroundThread)
             {
@@ -596,6 +606,12 @@ GotNewCode:
                     pCode = GetNativeCode();
                     goto Done;
                 }
+#if defined(FEATURE_JIT_PITCHING)
+                else
+                {
+                    SavePitchingCandidate(this, sizeOfCode);
+                }
+#endif
             }
 
 #ifdef FEATURE_INTERPRETER
@@ -1343,7 +1359,9 @@ PCODE MethodDesc::DoPrestub(MethodTable *pDispatchingMT)
 #endif
         LOG((LF_CLASSLOADER, LL_INFO10000,
                 "    In PreStubWorker, method already jitted, backpatching call point\n"));
-
+#if defined(FEATURE_JIT_PITCHING)
+        MarkMethodNotPitchingCandidate(this);
+#endif
         RETURN DoBackpatch(pMT, pDispatchingMT, TRUE);
     }
 
@@ -2127,6 +2145,10 @@ EXTERN_C PCODE STDCALL ExternalMethodFixupWorker(TransitionBlock * pTransitionBl
                 pCode = PatchNonVirtualExternalMethod(pMD, pCode, pImportSection, pIndirection);
             }
         }
+
+#if defined (FEATURE_JIT_PITCHING)
+        DeleteFromPitchingCandidate(pMD);
+#endif
     }
 
     // Force a GC on every jit if the stress level is high enough
diff --git a/src/vm/util.cpp b/src/vm/util.cpp
index 260e0daa38..724536a231 100644
--- a/src/vm/util.cpp
+++ b/src/vm/util.cpp
@@ -2871,7 +2871,7 @@ void DACNotify::DoJITNotification(MethodDesc *MethodDescPtr)
     DACNotifyExceptionHelper(Args, 2);
 }
 
-void DACNotify::DoJITDiscardNotification(MethodDesc *MethodDescPtr)
+void DACNotify::DoJITPitchingNotification(MethodDesc *MethodDescPtr)
 {
     CONTRACTL
     {
@@ -2883,9 +2883,9 @@ void DACNotify::DoJITDiscardNotification(MethodDesc *MethodDescPtr)
     CONTRACTL_END;
 
 #if defined(FEATURE_GDBJIT) && defined(FEATURE_PAL) && !defined(CROSSGEN_COMPILE)
-    NotifyGdb::MethodDropped(MethodDescPtr);
+    NotifyGdb::MethodPitched(MethodDescPtr);
 #endif    
-    TADDR Args[2] = { JIT_DISCARD_NOTIFICATION, (TADDR) MethodDescPtr };
+    TADDR Args[2] = { JIT_PITCHING_NOTIFICATION, (TADDR) MethodDescPtr };
     DACNotifyExceptionHelper(Args, 2);
 }    
    
@@ -3007,10 +3007,10 @@ BOOL DACNotify::ParseJITNotification(TADDR Args[], TADDR& MethodDescPtr)
     return TRUE;
 }
 
-BOOL DACNotify::ParseJITDiscardNotification(TADDR Args[], TADDR& MethodDescPtr)
+BOOL DACNotify::ParseJITPitchingNotification(TADDR Args[], TADDR& MethodDescPtr)
 {
-    _ASSERTE(Args[0] == JIT_DISCARD_NOTIFICATION);
-    if (Args[0] != JIT_DISCARD_NOTIFICATION)
+    _ASSERTE(Args[0] == JIT_PITCHING_NOTIFICATION);
+    if (Args[0] != JIT_PITCHING_NOTIFICATION)
     {
         return FALSE;
     }
diff --git a/src/vm/util.hpp b/src/vm/util.hpp
index 1f86d6c2d5..80cf97055b 100644
--- a/src/vm/util.hpp
+++ b/src/vm/util.hpp
@@ -1064,7 +1064,7 @@ public:
         MODULE_LOAD_NOTIFICATION=1,
         MODULE_UNLOAD_NOTIFICATION=2,
         JIT_NOTIFICATION=3,
-        JIT_DISCARD_NOTIFICATION=4,
+        JIT_PITCHING_NOTIFICATION=4,
         EXCEPTION_NOTIFICATION=5,
         GC_NOTIFICATION= 6,
         CATCH_ENTER_NOTIFICATION = 7,
@@ -1072,7 +1072,7 @@ public:
     
     // called from the runtime
     static void DoJITNotification(MethodDesc *MethodDescPtr);
-    static void DoJITDiscardNotification(MethodDesc *MethodDescPtr);    
+    static void DoJITPitchingNotification(MethodDesc *MethodDescPtr);
     static void DoModuleLoadNotification(Module *Module);
     static void DoModuleUnloadNotification(Module *Module);
     static void DoExceptionNotification(class Thread* ThreadPtr);
@@ -1082,7 +1082,7 @@ public:
     // called from the DAC
     static int GetType(TADDR Args[]);
     static BOOL ParseJITNotification(TADDR Args[], TADDR& MethodDescPtr);
-    static BOOL ParseJITDiscardNotification(TADDR Args[], TADDR& MethodDescPtr);
+    static BOOL ParseJITPitchingNotification(TADDR Args[], TADDR& MethodDescPtr);
     static BOOL ParseModuleLoadNotification(TADDR Args[], TADDR& ModulePtr);
     static BOOL ParseModuleUnloadNotification(TADDR Args[], TADDR& ModulePtr);
     static BOOL ParseExceptionNotification(TADDR Args[], TADDR& ThreadPtr);