Add pooling for JIT scratch memory (#18924)

Fixes #3408

24 files changed, 289 insertions, 353 deletions

diff --git a/src/ToolBox/superpmi/superpmi-shared/icorjithostimpl.h b/src/ToolBox/superpmi/superpmi-shared/icorjithostimpl.h
index aac68421e7..b05046f04c 100644
--- a/src/ToolBox/superpmi/superpmi-shared/icorjithostimpl.h
+++ b/src/ToolBox/superpmi/superpmi-shared/icorjithostimpl.h
@@ -23,15 +23,11 @@
 // against the interface declaration.
 
 public:
-// Allocate memory of the given size in bytes. All bytes of the returned block
-// must be initialized to zero. If `usePageAllocator` is true, the implementation
-// should use an allocator that deals in OS pages if one exists.
-void* allocateMemory(size_t size, bool usePageAllocator = false);
-
-// Frees memory previous obtained by a call to `ICorJitHost::allocateMemory`. The
-// value of the `usePageAllocator` parameter must match the value that was
-// provided to the call to used to allocate the memory.
-void freeMemory(void* block, bool usePageAllocator = false);
+// Allocate memory of the given size in bytes.
+void* allocateMemory(size_t size);
+
+// Frees memory previous obtained by a call to `ICorJitHost::allocateMemory`.
+void freeMemory(void* block);
 
 // Return an integer config value for the given key, if any exists.
 int getIntConfigValue(const wchar_t* name, int defaultValue);
diff --git a/src/ToolBox/superpmi/superpmi-shim-collector/jithost.cpp b/src/ToolBox/superpmi/superpmi-shim-collector/jithost.cpp
index fd5c46aeee..c3df837034 100644
--- a/src/ToolBox/superpmi/superpmi-shim-collector/jithost.cpp
+++ b/src/ToolBox/superpmi/superpmi-shim-collector/jithost.cpp
@@ -19,14 +19,14 @@ void JitHost::setMethodContext(MethodContext* methodContext)
     this->mc = methodContext;
 }
 
-void* JitHost::allocateMemory(size_t size, bool usePageAllocator)
+void* JitHost::allocateMemory(size_t size)
 {
-    return wrappedHost->allocateMemory(size, usePageAllocator);
+    return wrappedHost->allocateMemory(size);
 }
 
-void JitHost::freeMemory(void* block, bool usePageAllocator)
+void JitHost::freeMemory(void* block)
 {
-    return wrappedHost->freeMemory(block, usePageAllocator);
+    return wrappedHost->freeMemory(block);
 }
 
 int JitHost::getIntConfigValue(const wchar_t* key, int defaultValue)
diff --git a/src/ToolBox/superpmi/superpmi-shim-counter/jithost.cpp b/src/ToolBox/superpmi/superpmi-shim-counter/jithost.cpp
index d4efc33c69..4b46e0f9ff 100644
--- a/src/ToolBox/superpmi/superpmi-shim-counter/jithost.cpp
+++ b/src/ToolBox/superpmi/superpmi-shim-counter/jithost.cpp
@@ -20,14 +20,14 @@ void JitHost::setMethodCallSummarizer(MethodCallSummarizer* methodCallSummarizer
     this->mcs = methodCallSummarizer;
 }
 
-void* JitHost::allocateMemory(size_t size, bool usePageAllocator)
+void* JitHost::allocateMemory(size_t size)
 {
-    return wrappedHost->allocateMemory(size, usePageAllocator);
+    return wrappedHost->allocateMemory(size);
 }
 
-void JitHost::freeMemory(void* block, bool usePageAllocator)
+void JitHost::freeMemory(void* block)
 {
-    return wrappedHost->freeMemory(block, usePageAllocator);
+    return wrappedHost->freeMemory(block);
 }
 
 int JitHost::getIntConfigValue(const wchar_t* key, int defaultValue)
diff --git a/src/ToolBox/superpmi/superpmi-shim-simple/jithost.cpp b/src/ToolBox/superpmi/superpmi-shim-simple/jithost.cpp
index 01bff37a01..d864c8149b 100644
--- a/src/ToolBox/superpmi/superpmi-shim-simple/jithost.cpp
+++ b/src/ToolBox/superpmi/superpmi-shim-simple/jithost.cpp
@@ -14,14 +14,14 @@ JitHost::JitHost(ICorJitHost* wrappedHost) : wrappedHost(wrappedHost)
 {
 }
 
-void* JitHost::allocateMemory(size_t size, bool usePageAllocator)
+void* JitHost::allocateMemory(size_t size)
 {
-    return wrappedHost->allocateMemory(size, usePageAllocator);
+    return wrappedHost->allocateMemory(size);
 }
 
-void JitHost::freeMemory(void* block, bool usePageAllocator)
+void JitHost::freeMemory(void* block)
 {
-    return wrappedHost->freeMemory(block, usePageAllocator);
+    return wrappedHost->freeMemory(block);
 }
 
 int JitHost::getIntConfigValue(const wchar_t* key, int defaultValue)
diff --git a/src/ToolBox/superpmi/superpmi/jithost.cpp b/src/ToolBox/superpmi/superpmi/jithost.cpp
index 1b9b9a338d..ef2d46e744 100644
--- a/src/ToolBox/superpmi/superpmi/jithost.cpp
+++ b/src/ToolBox/superpmi/superpmi/jithost.cpp
@@ -51,12 +51,12 @@ JitHost::JitHost(JitInstance& jitInstance) : jitInstance(jitInstance)
 {
 }
 
-void* JitHost::allocateMemory(size_t size, bool usePageAllocator)
+void* JitHost::allocateMemory(size_t size)
 {
     return InitIEEMemoryManager(&jitInstance)->ClrVirtualAlloc(nullptr, size, 0, 0);
 }
 
-void JitHost::freeMemory(void* block, bool usePageAllocator)
+void JitHost::freeMemory(void* block)
 {
     InitIEEMemoryManager(&jitInstance)->ClrVirtualFree(block, 0, 0);
 }
diff --git a/src/inc/corcompile.h b/src/inc/corcompile.h
index 03dd981cc5..bc78c33787 100644
--- a/src/inc/corcompile.h
+++ b/src/inc/corcompile.h
@@ -1746,6 +1746,8 @@ class ICorCompileInfo
             IN  CORINFO_METHOD_HANDLE   hMethod,
             OUT CORJIT_FLAGS           *pFlags) = 0;
 
+    virtual ICorJitHost* GetJitHost() = 0;
+
     // needed for stubs to obtain the number of bytes to copy into the native image
     // return the beginning of the stub and the size to copy (in bytes)
     virtual void* GetStubSize(void *pStubAddress, DWORD *pSizeToCopy) = 0;
diff --git a/src/inc/corjithost.h b/src/inc/corjithost.h
index 8242fab2b8..b2ab80646b 100644
--- a/src/inc/corjithost.h
+++ b/src/inc/corjithost.h
@@ -15,15 +15,11 @@
 class ICorJitHost
 {
 public:
-    // Allocate memory of the given size in bytes. All bytes of the returned block
-    // must be initialized to zero. If `usePageAllocator` is true, the implementation
-    // should use an allocator that deals in OS pages if one exists.
-    virtual void* allocateMemory(size_t size, bool usePageAllocator = false) = 0;
+    // Allocate memory of the given size in bytes.
+    virtual void* allocateMemory(size_t size) = 0;
 
-    // Frees memory previous obtained by a call to `ICorJitHost::allocateMemory`. The
-    // value of the `usePageAllocator` parameter must match the value that was
-    // provided to the call to used to allocate the memory.
-    virtual void freeMemory(void* block, bool usePageAllocator = false) = 0;
+    // Frees memory previous obtained by a call to `ICorJitHost::allocateMemory`.
+    virtual void freeMemory(void* block) = 0;
 
     // Return an integer config value for the given key, if any exists.
     virtual int getIntConfigValue(
@@ -43,6 +39,20 @@ public:
     virtual void freeStringConfigValue(
         const wchar_t* value
         ) = 0;
+
+    // Allocate memory slab of the given size in bytes. The host is expected to pool
+    // these for a good performance.
+    virtual void* allocateSlab(size_t size, size_t* pActualSize)
+    {
+        *pActualSize = size;
+        return allocateMemory(size);
+    }
+
+    // Free memory slab of the given size in bytes.
+    virtual void freeSlab(void* slab, size_t actualSize)
+    {
+        freeMemory(slab);
+    }
 };
 
 #endif
diff --git a/src/jit/alloc.cpp b/src/jit/alloc.cpp
index d2c58e66c9..9a9d4ff2f4 100644
--- a/src/jit/alloc.cpp
+++ b/src/jit/alloc.cpp
@@ -9,110 +9,6 @@
 #endif // defined(_MSC_VER)
 
 //------------------------------------------------------------------------
-// SinglePagePool: Manage a single, default-sized page pool for ArenaAllocator.
-//
-//    Allocating a page is slightly costly as it involves the JIT host and
-//    possibly the operating system as well. This pool avoids allocation
-//    in many cases (i.e. for all non-concurrent method compilations).
-//
-class ArenaAllocator::SinglePagePool
-{
-    // The page maintained by this pool
-    PageDescriptor* m_page;
-    // The page available for allocation (either m_page or &m_shutdownPage if shutdown was called)
-    PageDescriptor* m_availablePage;
-    // A dummy page that is made available during shutdown
-    PageDescriptor m_shutdownPage;
-
-public:
-    // Attempt to acquire the page managed by this pool.
-    PageDescriptor* tryAcquirePage(IEEMemoryManager* memoryManager)
-    {
-        assert(memoryManager != nullptr);
-
-        PageDescriptor* page = InterlockedExchangeT(&m_availablePage, nullptr);
-        if ((page != nullptr) && (page->m_memoryManager != memoryManager))
-        {
-            // The pool page belongs to a different memory manager, release it.
-            releasePage(page, page->m_memoryManager);
-            page = nullptr;
-        }
-
-        assert((page == nullptr) || isPoolPage(page));
-
-        return page;
-    }
-
-    // Attempt to pool the specified page.
-    void tryPoolPage(PageDescriptor* page)
-    {
-        assert(page != &m_shutdownPage);
-
-        // Try to pool this page, give up if another thread has already pooled a page.
-        InterlockedCompareExchangeT(&m_page, page, nullptr);
-    }
-
-    // Check if a page is pooled.
-    bool isEmpty()
-    {
-        return (m_page == nullptr);
-    }
-
-    // Check if the specified page is pooled.
-    bool isPoolPage(PageDescriptor* page)
-    {
-        return (m_page == page);
-    }
-
-    // Release the specified page.
-    PageDescriptor* releasePage(PageDescriptor* page, IEEMemoryManager* memoryManager)
-    {
-        // tryAcquirePage may end up releasing the shutdown page if shutdown was called.
-        assert((page == &m_shutdownPage) || isPoolPage(page));
-        assert((page == &m_shutdownPage) || (memoryManager != nullptr));
-
-        // Normally m_availablePage should be null when releasePage is called but it can
-        // be the shutdown page if shutdown is called while the pool page is in use.
-        assert((m_availablePage == nullptr) || (m_availablePage == &m_shutdownPage));
-
-        PageDescriptor* next = page->m_next;
-        // Update the page's memory manager (replaces m_next that's not needed in this state).
-        page->m_memoryManager = memoryManager;
-        // Try to make the page available. This will fail if the pool was shutdown
-        // and then we need to free the page here.
-        PageDescriptor* shutdownPage = InterlockedCompareExchangeT(&m_availablePage, page, nullptr);
-        if (shutdownPage != nullptr)
-        {
-            assert(shutdownPage == &m_shutdownPage);
-            freeHostMemory(memoryManager, page);
-        }
-
-        // Return the next page for caller's convenience.
-        return next;
-    }
-
-    // Free the pooled page.
-    void shutdown()
-    {
-        // If the pool page is available then acquire it now so it can be freed.
-        // Also make the shutdown page available so that:
-        // - tryAcquirePage won't be return it because it has a null memory manager
-        // - releasePage won't be able to make the pool page available and instead will free it
-        PageDescriptor* page = InterlockedExchangeT(&m_availablePage, &m_shutdownPage);
-
-        assert(page != &m_shutdownPage);
-        assert((page == nullptr) || isPoolPage(page));
-
-        if ((page != nullptr) && (page->m_memoryManager != nullptr))
-        {
-            freeHostMemory(page->m_memoryManager, page);
-        }
-    }
-};
-
-ArenaAllocator::SinglePagePool ArenaAllocator::s_pagePool = {};
-
-//------------------------------------------------------------------------
 // ArenaAllocator::bypassHostAllocator:
 //    Indicates whether or not the ArenaAllocator should bypass the JIT
 //    host when allocating memory for arena pages.
@@ -147,39 +43,14 @@ size_t ArenaAllocator::getDefaultPageSize()
 // ArenaAllocator::ArenaAllocator:
 //    Default-constructs an arena allocator.
 ArenaAllocator::ArenaAllocator()
-    : m_memoryManager(nullptr)
-    , m_firstPage(nullptr)
-    , m_lastPage(nullptr)
-    , m_nextFreeByte(nullptr)
-    , m_lastFreeByte(nullptr)
+    : m_firstPage(nullptr), m_lastPage(nullptr), m_nextFreeByte(nullptr), m_lastFreeByte(nullptr)
 {
-    assert(!isInitialized());
-}
-
-//------------------------------------------------------------------------
-// ArenaAllocator::initialize:
-//    Initializes the arena allocator.
-//
-// Arguments:
-//    memoryManager - The `IEEMemoryManager` instance that will be used to
-//                    allocate memory for arena pages.
-void ArenaAllocator::initialize(IEEMemoryManager* memoryManager)
-{
-    assert(!isInitialized());
-    m_memoryManager = memoryManager;
-    assert(isInitialized());
-
 #if MEASURE_MEM_ALLOC
     memset(&m_stats, 0, sizeof(m_stats));
     memset(&m_statsAllocators, 0, sizeof(m_statsAllocators));
 #endif // MEASURE_MEM_ALLOC
 }
 
-bool ArenaAllocator::isInitialized()
-{
-    return m_memoryManager != nullptr;
-}
-
 //------------------------------------------------------------------------
 // ArenaAllocator::allocateNewPage:
 //    Allocates a new arena page.
@@ -192,8 +63,6 @@ bool ArenaAllocator::isInitialized()
 //    A pointer to the first usable byte of the newly allocated page.
 void* ArenaAllocator::allocateNewPage(size_t size)
 {
-    assert(isInitialized());
-
     size_t pageSize = sizeof(PageDescriptor) + size;
 
     // Check for integer overflow
@@ -212,46 +81,23 @@ void* ArenaAllocator::allocateNewPage(size_t size)
         m_lastPage->m_usedBytes = m_nextFreeByte - m_lastPage->m_contents;
     }
 
-    PageDescriptor* newPage        = nullptr;
-    bool            tryPoolNewPage = false;
+    PageDescriptor* newPage = nullptr;
 
     if (!bypassHostAllocator())
     {
-        // Round to the nearest multiple of OS page size
+        // Round to the nearest multiple of default page size
         pageSize = roundUp(pageSize, DEFAULT_PAGE_SIZE);
-
-        // If this is the first time we allocate a page then try to use the pool page.
-        if ((m_firstPage == nullptr) && (pageSize == DEFAULT_PAGE_SIZE))
-        {
-            newPage = s_pagePool.tryAcquirePage(m_memoryManager);
-
-            if (newPage == nullptr)
-            {
-                // If there's no pool page yet then try to pool the newly allocated page.
-                tryPoolNewPage = s_pagePool.isEmpty();
-            }
-            else
-            {
-                assert(newPage->m_memoryManager == m_memoryManager);
-                assert(newPage->m_pageBytes == DEFAULT_PAGE_SIZE);
-            }
-        }
     }
 
     if (newPage == nullptr)
     {
         // Allocate the new page
-        newPage = static_cast<PageDescriptor*>(allocateHostMemory(m_memoryManager, pageSize));
+        newPage = static_cast<PageDescriptor*>(allocateHostMemory(pageSize, &pageSize));
 
         if (newPage == nullptr)
         {
             NOMEM();
         }
-
-        if (tryPoolNewPage)
-        {
-            s_pagePool.tryPoolPage(newPage);
-        }
     }
 
     // Append the new page to the end of the list
@@ -285,31 +131,20 @@ void* ArenaAllocator::allocateNewPage(size_t size)
 //    Performs any necessary teardown for an `ArenaAllocator`.
 void ArenaAllocator::destroy()
 {
-    assert(isInitialized());
-
     PageDescriptor* page = m_firstPage;
 
-    // If the first page is the pool page then return it to the pool.
-    if ((page != nullptr) && s_pagePool.isPoolPage(page))
-    {
-        page = s_pagePool.releasePage(page, m_memoryManager);
-    }
-
     // Free all of the allocated pages
     for (PageDescriptor* next; page != nullptr; page = next)
     {
-        assert(!s_pagePool.isPoolPage(page));
         next = page->m_next;
-        freeHostMemory(m_memoryManager, page);
+        freeHostMemory(page, page->m_pageBytes);
     }
 
     // Clear out the allocator's fields
-    m_memoryManager = nullptr;
-    m_firstPage     = nullptr;
-    m_lastPage      = nullptr;
-    m_nextFreeByte  = nullptr;
-    m_lastFreeByte  = nullptr;
-    assert(!isInitialized());
+    m_firstPage    = nullptr;
+    m_lastPage     = nullptr;
+    m_nextFreeByte = nullptr;
+    m_lastFreeByte = nullptr;
 }
 
 // The debug version of the allocator may allocate directly from the
@@ -329,25 +164,21 @@ void ArenaAllocator::destroy()
 //
 // Arguments:
 //    size - The number of bytes to allocate.
+//    pActualSize - The number of byte actually allocated.
 //
 // Return Value:
 //    A pointer to the allocated memory.
-void* ArenaAllocator::allocateHostMemory(IEEMemoryManager* memoryManager, size_t size)
+void* ArenaAllocator::allocateHostMemory(size_t size, size_t* pActualSize)
 {
-    assert(memoryManager != nullptr);
-
 #if defined(DEBUG)
     if (bypassHostAllocator())
     {
+        *pActualSize = size;
         return ::HeapAlloc(GetProcessHeap(), 0, size);
     }
-    else
-    {
-        return ClrAllocInProcessHeap(0, S_SIZE_T(size));
-    }
-#else  // defined(DEBUG)
-    return memoryManager->ClrVirtualAlloc(nullptr, size, MEM_COMMIT, PAGE_READWRITE);
 #endif // !defined(DEBUG)
+
+    return g_jitHost->allocateSlab(size, pActualSize);
 }
 
 //------------------------------------------------------------------------
@@ -356,22 +187,17 @@ void* ArenaAllocator::allocateHostMemory(IEEMemoryManager* memoryManager, size_t
 //
 // Arguments:
 //    block - A pointer to the memory to free.
-void ArenaAllocator::freeHostMemory(IEEMemoryManager* memoryManager, void* block)
+void ArenaAllocator::freeHostMemory(void* block, size_t size)
 {
-    assert(memoryManager != nullptr);
-
 #if defined(DEBUG)
     if (bypassHostAllocator())
     {
         ::HeapFree(GetProcessHeap(), 0, block);
+        return;
     }
-    else
-    {
-        ClrFreeInProcessHeap(0, block);
-    }
-#else  // defined(DEBUG)
-    memoryManager->ClrVirtualFree(block, 0, MEM_RELEASE);
 #endif // !defined(DEBUG)
+
+    g_jitHost->freeSlab(block, size);
 }
 
 //------------------------------------------------------------------------
@@ -383,8 +209,6 @@ void ArenaAllocator::freeHostMemory(IEEMemoryManager* memoryManager, void* block
 //    See above.
 size_t ArenaAllocator::getTotalBytesAllocated()
 {
-    assert(isInitialized());
-
     size_t bytes = 0;
     for (PageDescriptor* page = m_firstPage; page != nullptr; page = page->m_next)
     {
@@ -411,8 +235,6 @@ size_t ArenaAllocator::getTotalBytesAllocated()
 //    that are unused across all area pages.
 size_t ArenaAllocator::getTotalBytesUsed()
 {
-    assert(isInitialized());
-
     if (m_lastPage != nullptr)
     {
         m_lastPage->m_usedBytes = m_nextFreeByte - m_lastPage->m_contents;
@@ -427,14 +249,6 @@ size_t ArenaAllocator::getTotalBytesUsed()
     return bytes;
 }
 
-//------------------------------------------------------------------------
-// ArenaAllocator::shutdown:
-//    Performs any necessary teardown for the arena allocator subsystem.
-void ArenaAllocator::shutdown()
-{
-    s_pagePool.shutdown();
-}
-
 #if MEASURE_MEM_ALLOC
 CritSecObject                     ArenaAllocator::s_statsLock;
 ArenaAllocator::AggregateMemStats ArenaAllocator::s_aggStats;
diff --git a/src/jit/alloc.h b/src/jit/alloc.h
index 7090c7fa43..2945df9107 100644
--- a/src/jit/alloc.h
+++ b/src/jit/alloc.h
@@ -29,12 +29,7 @@ private:
 
     struct PageDescriptor
     {
-        union {
-            // Used when the page is allocated
-            PageDescriptor* m_next;
-            // Used by the pooled page when available
-            IEEMemoryManager* m_memoryManager;
-        };
+        PageDescriptor* m_next;
 
         size_t m_pageBytes; // # of bytes allocated
         size_t m_usedBytes; // # of bytes actually used. (This is only valid when we've allocated a new page.)
@@ -43,19 +38,11 @@ private:
         BYTE m_contents[];
     };
 
-    // Anything less than 64K leaves VM holes since the OS allocates address space in this size.
-    // Thus if we want to make this smaller, we need to do a reserve / commit scheme
     enum
     {
-        DEFAULT_PAGE_SIZE = 16 * OS_page_size,
+        DEFAULT_PAGE_SIZE = 0x10000,
     };
 
-    class SinglePagePool;
-
-    static SinglePagePool s_pagePool;
-
-    IEEMemoryManager* m_memoryManager;
-
     PageDescriptor* m_firstPage;
     PageDescriptor* m_lastPage;
 
@@ -63,12 +50,10 @@ private:
     BYTE* m_nextFreeByte;
     BYTE* m_lastFreeByte;
 
-    bool isInitialized();
-
     void* allocateNewPage(size_t size);
 
-    static void* allocateHostMemory(IEEMemoryManager* memoryManager, size_t size);
-    static void freeHostMemory(IEEMemoryManager* memoryManager, void* block);
+    static void* allocateHostMemory(size_t size, size_t* pActualSize);
+    static void freeHostMemory(void* block, size_t size);
 
 #if MEASURE_MEM_ALLOC
     struct MemStats
@@ -151,8 +136,6 @@ public:
 public:
     ArenaAllocator();
 
-    void initialize(IEEMemoryManager* memoryManager);
-
     // NOTE: it would be nice to have a destructor on this type to ensure that any value that
     //       goes out of scope is either uninitialized or has been torn down via a call to
     //       destroy(), but this interacts badly in methods that use SEH. #3058 tracks
@@ -168,8 +151,6 @@ public:
 
     static bool   bypassHostAllocator();
     static size_t getDefaultPageSize();
-
-    static void shutdown();
 };
 
 //------------------------------------------------------------------------
@@ -190,7 +171,6 @@ public:
 //
 inline void* ArenaAllocator::allocateMemory(size_t size)
 {
-    assert(isInitialized());
     assert(size != 0);
 
     // Ensure that we always allocate in pointer sized increments.
diff --git a/src/jit/compiler.cpp b/src/jit/compiler.cpp
index e54787690a..edfe431357 100644
--- a/src/jit/compiler.cpp
+++ b/src/jit/compiler.cpp
@@ -1449,8 +1449,6 @@ void Compiler::compShutdown()
     DisplayNowayAssertMap();
 #endif // MEASURE_NOWAY
 
-    ArenaAllocator::shutdown();
-
     /* Shut down the emitter */
 
     emitter::emitDone();
@@ -6623,7 +6621,6 @@ START:
     }
     else
     {
-        alloc.initialize(compHnd->getMemoryManager());
         pAlloc = &alloc;
     }
 
diff --git a/src/jit/host.h b/src/jit/host.h
index 87e13d4180..95894a5e32 100644
--- a/src/jit/host.h
+++ b/src/jit/host.h
@@ -52,8 +52,6 @@ extern "C" void __cdecl assertAbort(const char* why, const char* file, unsigned
 #define _HOST_H_
 /*****************************************************************************/
 
-const size_t OS_page_size = (4 * 1024);
-
 extern FILE* jitstdout;
 
 inline FILE* procstdout()
diff --git a/src/jit/hostallocator.cpp b/src/jit/hostallocator.cpp
index c9afd1a2bc..112bf1c107 100644
--- a/src/jit/hostallocator.cpp
+++ b/src/jit/hostallocator.cpp
@@ -8,11 +8,11 @@
 void* HostAllocator::allocateHostMemory(size_t size)
 {
     assert(g_jitHost != nullptr);
-    return g_jitHost->allocateMemory(size, false);
+    return g_jitHost->allocateMemory(size);
 }
 
 void HostAllocator::freeHostMemory(void* p)
 {
     assert(g_jitHost != nullptr);
-    g_jitHost->freeMemory(p, false);
+    g_jitHost->freeMemory(p);
 }
diff --git a/src/utilcode/CMakeLists.txt b/src/utilcode/CMakeLists.txt
index 9629e5140f..f591e7cbec 100644
--- a/src/utilcode/CMakeLists.txt
+++ b/src/utilcode/CMakeLists.txt
@@ -54,8 +54,7 @@ set(UTILCODE_COMMON_SOURCES
   debug.cpp
   pedecoder.cpp
   winfix.cpp
-  longfilepathwrappers.cpp 
-  jithost.cpp
+  longfilepathwrappers.cpp
 )
 
 # These source file do not yet compile on Linux.
diff --git a/src/utilcode/jithost.cpp b/src/utilcode/jithost.cpp
deleted file mode 100644
index 3174aa6188..0000000000
--- a/src/utilcode/jithost.cpp
+++ /dev/null
@@ -1,77 +0,0 @@
-// Licensed to the .NET Foundation under one or more agreements.
-// The .NET Foundation licenses this file to you under the MIT license.
-// See the LICENSE file in the project root for more information.
-
-#include "stdafx.h"
-
-#include "utilcode.h"
-#include "corjit.h"
-#include "jithost.h"
-
-void* JitHost::allocateMemory(size_t size, bool usePageAllocator)
-{
-    WRAPPER_NO_CONTRACT;
-
-    if (usePageAllocator)
-    {
-        return GetEEMemoryManager()->ClrVirtualAlloc(nullptr, size, MEM_COMMIT, PAGE_READWRITE);
-    }
-    else
-    {
-        return ClrAllocInProcessHeap(0, S_SIZE_T(size));
-    }
-}
-
-void JitHost::freeMemory(void* block, bool usePageAllocator)
-{
-    WRAPPER_NO_CONTRACT;
-
-    if (usePageAllocator)
-    {
-        GetEEMemoryManager()->ClrVirtualFree(block, 0, MEM_RELEASE);
-    }
-    else
-    {
-        ClrFreeInProcessHeap(0, block);
-    }
-}
-
-int JitHost::getIntConfigValue(const wchar_t* name, int defaultValue)
-{
-    WRAPPER_NO_CONTRACT;
-
-    // Translate JIT call into runtime configuration query
-    CLRConfig::ConfigDWORDInfo info{ name, defaultValue, CLRConfig::EEConfig_default };
-
-    // Perform a CLRConfig look up on behalf of the JIT.
-    return CLRConfig::GetConfigValue(info);
-}
-
-const wchar_t* JitHost::getStringConfigValue(const wchar_t* name)
-{
-    WRAPPER_NO_CONTRACT;
-
-    // Translate JIT call into runtime configuration query
-    CLRConfig::ConfigStringInfo info{ name, CLRConfig::EEConfig_default };
-
-    // Perform a CLRConfig look up on behalf of the JIT.
-    return CLRConfig::GetConfigValue(info);
-}
-
-void JitHost::freeStringConfigValue(const wchar_t* value)
-{
-    WRAPPER_NO_CONTRACT;
-
-    CLRConfig::FreeConfigString(const_cast<wchar_t*>(value));
-}
-
-JitHost JitHost::theJitHost;
-ICorJitHost* JitHost::getJitHost()
-{
-    STATIC_CONTRACT_SO_TOLERANT;
-    STATIC_CONTRACT_GC_NOTRIGGER;
-    STATIC_CONTRACT_NOTHROW;
-    STATIC_CONTRACT_CANNOT_TAKE_LOCK;
-
-    return &theJitHost;
-}
diff --git a/src/vm/CMakeLists.txt b/src/vm/CMakeLists.txt
index cb75a09adb..709c5e6f73 100644
--- a/src/vm/CMakeLists.txt
+++ b/src/vm/CMakeLists.txt
@@ -81,6 +81,7 @@ set(VM_SOURCES_DAC_AND_WKS_COMMON
     ilstubresolver.cpp
     inlinetracking.cpp
     instmethhash.cpp
+    jithost.cpp
     jitinterface.cpp
     loaderallocator.cpp
     memberload.cpp
diff --git a/src/vm/ceemain.cpp b/src/vm/ceemain.cpp
index fb1832a83f..da4c7c5f65 100644
--- a/src/vm/ceemain.cpp
+++ b/src/vm/ceemain.cpp
@@ -175,6 +175,7 @@
 #include "finalizerthread.h"
 #include "threadsuspend.h"
 #include "disassembler.h"
+#include "jithost.h"
 
 #ifndef FEATURE_PAL
 #include "dwreport.h"
@@ -908,6 +909,8 @@ void EEStartupHelper(COINITIEE fFlags)
 
         ExecutionManager::Init();
 
+        JitHost::Init();
+
 #ifndef CROSSGEN_COMPILE
 
 #ifndef FEATURE_PAL      
diff --git a/src/vm/compile.cpp b/src/vm/compile.cpp
index b1c3f8cadb..52440523bd 100644
--- a/src/vm/compile.cpp
+++ b/src/vm/compile.cpp
@@ -68,6 +68,7 @@
 
 #include "versionresilienthashcode.h"
 #include "inlinetracking.h"
+#include "jithost.h"
 
 #ifdef CROSSGEN_COMPILE
 CompilationDomain * theDomain;
@@ -1112,6 +1113,11 @@ void CEECompileInfo::CompressDebugInfo(
     CompressDebugInfo::CompressBoundariesAndVars(pOffsetMapping, iOffsetMapping, pNativeVarInfo, iNativeVarInfo, pDebugInfoBuffer, NULL);
 }
 
+ICorJitHost* CEECompileInfo::GetJitHost()
+{
+    return JitHost::getJitHost();
+}
+
 HRESULT CEECompileInfo::GetBaseJitFlags(
         IN  CORINFO_METHOD_HANDLE   hMethod,
         OUT CORJIT_FLAGS           *pFlags)
diff --git a/src/vm/compile.h b/src/vm/compile.h
index 5be78c27bb..fb664249c5 100644
--- a/src/vm/compile.h
+++ b/src/vm/compile.h
@@ -355,6 +355,8 @@ class CEECompileInfo : public ICorCompileInfo
             IN  CORINFO_METHOD_HANDLE    hMethod,
             OUT CORJIT_FLAGS            *pFlags);
 
+    ICorJitHost* GetJitHost();
+
     void* GetStubSize(void *pStubAddress, DWORD *pSizeToCopy);
 
     HRESULT GetStubClone(void *pStub, BYTE *pBuffer, DWORD dwBufferSize);
diff --git a/src/vm/crossgen/CMakeLists.txt b/src/vm/crossgen/CMakeLists.txt
index fe1f7a38c9..75742b599f 100644
--- a/src/vm/crossgen/CMakeLists.txt
+++ b/src/vm/crossgen/CMakeLists.txt
@@ -45,6 +45,7 @@ set(VM_CROSSGEN_SOURCES
     ../invokeutil.cpp
     ../inlinetracking.cpp
     ../contractimpl.cpp
+    ../jithost.cpp
     ../jitinterface.cpp
     ../loaderallocator.cpp
     ../memberload.cpp
diff --git a/src/vm/finalizerthread.cpp b/src/vm/finalizerthread.cpp
index 48164f6279..62816ebacb 100644
--- a/src/vm/finalizerthread.cpp
+++ b/src/vm/finalizerthread.cpp
@@ -7,12 +7,12 @@
 
 #include "finalizerthread.h"
 #include "threadsuspend.h"
+#include "jithost.h"
 
 #ifdef FEATURE_COMINTEROP
 #include "runtimecallablewrapper.h"
 #endif
 
-
 #ifdef FEATURE_PROFAPI_ATTACH_DETACH 
 #include "profattach.h"
 #endif // FEATURE_PROFAPI_ATTACH_DETACH 
@@ -583,6 +583,8 @@ VOID FinalizerThread::FinalizerThreadWorker(void *args)
                 bPriorityBoosted = TRUE;
         }
 
+        JitHost::Reclaim();
+
         GetFinalizerThread()->DisablePreemptiveGC();
 
 #ifdef _DEBUG
diff --git a/src/vm/jithost.cpp b/src/vm/jithost.cpp
new file mode 100644
index 0000000000..4a7783ef64
--- /dev/null
+++ b/src/vm/jithost.cpp
@@ -0,0 +1,182 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#include "common.h"
+
+#include "utilcode.h"
+#include "corjit.h"
+#include "jithost.h"
+
+void* JitHost::allocateMemory(size_t size)
+{
+    WRAPPER_NO_CONTRACT;
+
+    return ClrAllocInProcessHeap(0, S_SIZE_T(size));
+}
+
+void JitHost::freeMemory(void* block)
+{
+    WRAPPER_NO_CONTRACT;
+
+    ClrFreeInProcessHeap(0, block);
+}
+
+int JitHost::getIntConfigValue(const wchar_t* name, int defaultValue)
+{
+    WRAPPER_NO_CONTRACT;
+
+    // Translate JIT call into runtime configuration query
+    CLRConfig::ConfigDWORDInfo info{ name, defaultValue, CLRConfig::EEConfig_default };
+
+    // Perform a CLRConfig look up on behalf of the JIT.
+    return CLRConfig::GetConfigValue(info);
+}
+
+const wchar_t* JitHost::getStringConfigValue(const wchar_t* name)
+{
+    WRAPPER_NO_CONTRACT;
+
+    // Translate JIT call into runtime configuration query
+    CLRConfig::ConfigStringInfo info{ name, CLRConfig::EEConfig_default };
+
+    // Perform a CLRConfig look up on behalf of the JIT.
+    return CLRConfig::GetConfigValue(info);
+}
+
+void JitHost::freeStringConfigValue(const wchar_t* value)
+{
+    WRAPPER_NO_CONTRACT;
+
+    CLRConfig::FreeConfigString(const_cast<wchar_t*>(value));
+}
+
+//
+// Pool memory blocks for JIT to avoid frequent commit/decommit. The frequent commit/decommit has been 
+// shown to slow down the JIT significantly (10% or more). The memory blocks used by the JIT tend to be too big
+// to be covered by pooling done by the default malloc.
+//
+// - Keep up to some limit worth of memory, with loose affinization of memory blocks to threads.
+// - On finalizer thread, release the extra memory that was not used recently.
+//
+
+void* JitHost::allocateSlab(size_t size, size_t* pActualSize)
+{
+    size = max(size, sizeof(Slab));
+
+    Thread* pCurrentThread = GetThread();
+    if (m_pCurrentCachedList != NULL || m_pPreviousCachedList != NULL)
+    {
+        CrstHolder lock(&m_jitSlabAllocatorCrst);
+        Slab** ppCandidate = NULL;
+
+        for (Slab ** ppList = &m_pCurrentCachedList; *ppList != NULL; ppList = &(*ppList)->pNext)
+        {
+            Slab* p = *ppList;
+            if (p->size >= size && p->size <= 4 * size) // Avoid wasting more than 4x memory
+            {
+                ppCandidate = ppList;
+                if (p->affinity == pCurrentThread)
+                    break;
+            }
+        }
+
+        if (ppCandidate == NULL)
+        {
+            for (Slab ** ppList = &m_pPreviousCachedList; *ppList != NULL; ppList = &(*ppList)->pNext)
+            {
+                Slab* p = *ppList;
+                if (p->size == size) // Allocation from previous list requires exact match
+                {
+                    ppCandidate = ppList;
+                    if (p->affinity == pCurrentThread)
+                        break;
+                }
+            }
+        }
+
+        if (ppCandidate != NULL)
+        {
+            Slab* p = *ppCandidate;
+            *ppCandidate = p->pNext;
+
+            m_totalCached -= p->size;
+            *pActualSize = p->size;
+
+            return p;
+        }
+    }
+
+    *pActualSize = size;
+    return ClrAllocInProcessHeap(0, S_SIZE_T(size));
+}
+
+void JitHost::freeSlab(void* slab, size_t actualSize)
+{
+    _ASSERTE(actualSize >= sizeof(Slab));
+
+    if (actualSize < 0x100000) // Do not cache blocks that are more than 1MB
+    {
+        CrstHolder lock(&m_jitSlabAllocatorCrst);
+
+        if (m_totalCached < 0x1000000) // Do not cache more than 16MB
+        {
+            m_totalCached += actualSize;
+
+            Slab* pSlab = (Slab*)slab;
+            pSlab->size = actualSize;
+            pSlab->affinity = GetThread();
+            pSlab->pNext = m_pCurrentCachedList;
+            m_pCurrentCachedList = pSlab;
+            return;
+        }
+    }
+
+    ClrFreeInProcessHeap(0, slab);
+}
+
+void JitHost::init()
+{
+    m_jitSlabAllocatorCrst.Init(CrstLeafLock);
+}
+
+void JitHost::reclaim()
+{
+    if (m_pCurrentCachedList != NULL || m_pPreviousCachedList != NULL)
+    {
+        DWORD ticks = ::GetTickCount();
+
+        if (m_lastFlush == 0) // Just update m_lastFlush first time around
+        {
+            m_lastFlush = ticks;
+            return;
+        }
+
+        if ((DWORD)(ticks - m_lastFlush) < 2000) // Flush the free lists every 2 seconds
+            return;
+        m_lastFlush = ticks;
+
+        // Flush all slabs in m_pPreviousCachedList
+        for (;;)
+        {
+            Slab* slabToDelete = NULL;
+
+            {
+                CrstHolder lock(&m_jitSlabAllocatorCrst);
+                slabToDelete = m_pPreviousCachedList;
+                if (slabToDelete == NULL)
+                {
+                    m_pPreviousCachedList = m_pCurrentCachedList;
+                    m_pCurrentCachedList = NULL;
+                    break;
+                }
+                m_totalCached -= slabToDelete->size;
+                m_pPreviousCachedList = slabToDelete->pNext;
+            }
+
+            ClrFreeInProcessHeap(0, slabToDelete);
+        }
+    }
+}
+
+JitHost JitHost::s_theJitHost;
diff --git a/src/inc/jithost.h b/src/vm/jithost.h
index 73ad3343b5..b2ec67b3de 100644
--- a/src/inc/jithost.h
+++ b/src/vm/jithost.h
@@ -9,20 +9,41 @@
 class JitHost : public ICorJitHost
 {
 private:
-    static JitHost theJitHost;
+    static JitHost s_theJitHost;
+
+    struct Slab
+    {
+        Slab * pNext;
+        size_t size;
+        Thread* affinity;
+    };
+
+    CrstStatic m_jitSlabAllocatorCrst;
+    Slab* m_pCurrentCachedList;
+    Slab* m_pPreviousCachedList;
+    size_t m_totalCached;
+    DWORD m_lastFlush;
 
     JitHost() {}
     JitHost(const JitHost& other) = delete;
     JitHost& operator=(const JitHost& other) = delete;
 
+    void init();
+    void reclaim();
+
 public:
-    virtual void* allocateMemory(size_t size, bool usePageAllocator);
-    virtual void freeMemory(void* block, bool usePageAllocator);
+    virtual void* allocateMemory(size_t size);
+    virtual void freeMemory(void* block);
     virtual int getIntConfigValue(const wchar_t* name, int defaultValue);
     virtual const wchar_t* getStringConfigValue(const wchar_t* name);
     virtual void freeStringConfigValue(const wchar_t* value);
+    virtual void* allocateSlab(size_t size, size_t* pActualSize);
+    virtual void freeSlab(void* slab, size_t actualSize);
+
+    static void Init() { s_theJitHost.init(); }
+    static void Reclaim() { s_theJitHost.reclaim(); }
 
-    static ICorJitHost* getJitHost();
+    static ICorJitHost* getJitHost() { return &s_theJitHost; }
 };
 
 #endif // __JITHOST_H__
diff --git a/src/zap/common.h b/src/zap/common.h
index df5beacba0..b2dbfee7af 100644
--- a/src/zap/common.h
+++ b/src/zap/common.h
@@ -35,7 +35,6 @@ typedef unsigned int TARGET_POINTER_TYPE;
 
 #include "utilcode.h"
 #include "corjit.h"
-#include "jithost.h"
 #include "corcompile.h"
 #include "iceefilegen.h"
 #include "corpriv.h"
diff --git a/src/zap/zapper.cpp b/src/zap/zapper.cpp
index 683805eacf..090b545826 100644
--- a/src/zap/zapper.cpp
+++ b/src/zap/zapper.cpp
@@ -525,7 +525,7 @@ void Zapper::LoadAndInitializeJITForNgen(LPCWSTR pwzJitName, OUT HINSTANCE* phJi
     pJitStartup jitStartupFn = (pJitStartup)GetProcAddress(*phJit, "jitStartup");
     if (jitStartupFn != nullptr)
     {
-        jitStartupFn(JitHost::getJitHost());
+        jitStartupFn(m_pEECompileInfo->GetJitHost());
     }
 
     //get the appropriate compiler interface
@@ -599,7 +599,7 @@ void Zapper::InitEE(BOOL fForceDebug, BOOL fForceProfile, BOOL fForceInstrument)
     //
 
 #ifdef FEATURE_MERGE_JIT_AND_ENGINE
-    jitStartup(JitHost::getJitHost());
+    jitStartup(m_pEECompileInfo->GetJitHost());
     m_pJitCompiler = getJit();
 
     if (m_pJitCompiler == NULL)