Implement MEM_RESET handling in VirtualAlloc on Unix (#10939)

3 files changed, 88 insertions, 10 deletions

diff --git a/src/gc/gc.cpp b/src/gc/gc.cpp
index ecc13e38fd..1aba30bf4b 100644
--- a/src/gc/gc.cpp
+++ b/src/gc/gc.cpp
@@ -9266,12 +9266,10 @@ void gc_heap::delete_heap_segment (heap_segment* seg, BOOL consider_hoarding)
 
 void gc_heap::reset_heap_segment_pages (heap_segment* seg)
 {
-#ifndef FEATURE_PAL // No MEM_RESET support in PAL VirtualAlloc
     size_t page_start = align_on_page ((size_t)heap_segment_allocated (seg));
     size_t size = (size_t)heap_segment_committed (seg) - page_start;
     if (size != 0)
         GCToOSInterface::VirtualReset((void*)page_start, size, false /* unlock */);
-#endif //!FEATURE_PAL
 }
 
 void gc_heap::decommit_heap_segment_pages (heap_segment* seg,
@@ -30809,7 +30807,6 @@ CObjectHeader* gc_heap::allocate_large_object (size_t jsize, int64_t& alloc_byte
 
 void reset_memory (uint8_t* o, size_t sizeo)
 {
-#ifndef FEATURE_PAL
     if (sizeo > 128 * 1024)
     {
         // We cannot reset the memory for the useful part of a free object.
@@ -30824,7 +30821,6 @@ void reset_memory (uint8_t* o, size_t sizeo)
             reset_mm_p = GCToOSInterface::VirtualReset((void*)page_start, size, true /* unlock */);
         }
     }
-#endif //!FEATURE_PAL
 }
 
 void gc_heap::reset_large_object (uint8_t* o)
diff --git a/src/gc/unix/gcenv.unix.cpp b/src/gc/unix/gcenv.unix.cpp
index 45489c69a7..cad83a342b 100644
--- a/src/gc/unix/gcenv.unix.cpp
+++ b/src/gc/unix/gcenv.unix.cpp
@@ -348,8 +348,20 @@ bool GCToOSInterface::VirtualDecommit(void* address, size_t size)
 //  true if it has succeeded, false if it has failed
 bool GCToOSInterface::VirtualReset(void * address, size_t size, bool unlock)
 {
-    // TODO(CoreCLR#1259) pipe to madvise?
-    return false;
+    int st;
+#if HAVE_MADV_FREE
+    // Try to use MADV_FREE if supported. It tells the kernel that the application doesn't
+    // need the pages in the range. Freeing the pages can be delayed until a memory pressure
+    // occurs.
+    st = madvise(address, size, MADV_FREE);
+    if (st != 0)
+#endif    
+    {
+        // In case the MADV_FREE is not supported, use MADV_DONTNEED
+        st = madvise(address, size, MADV_DONTNEED);
+    }
+
+    return (st == 0);
 }
 
 // Check if the OS supports write watching
diff --git a/src/pal/src/map/virtual.cpp b/src/pal/src/map/virtual.cpp
index 4a55de9891..d52ba1e896 100644
--- a/src/pal/src/map/virtual.cpp
+++ b/src/pal/src/map/virtual.cpp
@@ -92,6 +92,7 @@ namespace VirtualMemoryLogging
         Commit = 0x30,
         Decommit = 0x40,
         Release = 0x50,
+        Reset = 0x60,
     };
 
     // Indicates that the attempted operation has failed
@@ -810,6 +811,58 @@ static BOOL VIRTUALStoreAllocationInfo(
 
 /******
  *
+ *  VIRTUALResetMemory() - Helper function that resets the memory 
+ *
+ *
+ */
+static LPVOID VIRTUALResetMemory(
+                IN CPalThread *pthrCurrent, /* Currently executing thread */
+                IN LPVOID lpAddress,        /* Region to reserve or commit */
+                IN SIZE_T dwSize)           /* Size of Region */
+{
+    LPVOID pRetVal = NULL;
+    UINT_PTR StartBoundary;
+    SIZE_T MemSize;
+
+    TRACE( "Resetting the memory now..\n");
+
+    StartBoundary = (UINT_PTR)lpAddress & ~VIRTUAL_PAGE_MASK;
+    // Add the sizes, and round down to the nearest page boundary. 
+    MemSize = ( ((UINT_PTR)lpAddress + dwSize + VIRTUAL_PAGE_MASK) & ~VIRTUAL_PAGE_MASK ) - 
+               StartBoundary;
+
+    int st;
+#if HAVE_MADV_FREE
+    // Try to use MADV_FREE if supported. It tells the kernel that the application doesn't
+    // need the pages in the range. Freeing the pages can be delayed until a memory pressure
+    // occurs.
+    st = madvise((LPVOID)StartBoundary, MemSize, MADV_FREE);
+    if (st != 0)
+#endif    
+    {
+        // In case the MADV_FREE is not supported, use MADV_DONTNEED
+        st = madvise((LPVOID)StartBoundary, MemSize, MADV_DONTNEED);
+    }
+
+    if (st == 0)
+    {
+        pRetVal = lpAddress;
+    }
+
+    LogVaOperation(
+        VirtualMemoryLogging::VirtualOperation::Reset,
+        lpAddress,
+        dwSize,
+        0,
+        0,
+        pRetVal,
+        pRetVal != NULL);
+
+    return pRetVal;
+}
+
+/******
+ *
  *  VIRTUALReserveMemory() - Helper function that actually reserves the memory.
  *
  *      NOTE: I call SetLastError in here, because many different error states
@@ -837,8 +890,6 @@ static LPVOID VIRTUALReserveMemory(
     MemSize = ( ((UINT_PTR)lpAddress + dwSize + VIRTUAL_PAGE_MASK) & ~VIRTUAL_PAGE_MASK ) - 
                StartBoundary;
 
-    InternalEnterCriticalSection(pthrCurrent, &virtual_critsec);
-
     // If this is a request for special executable (JIT'ed) memory then, first of all,
     // try to get memory from the executable memory allocator to satisfy the request.
     if (((flAllocationType & MEM_RESERVE_EXECUTABLE) != 0) && (lpAddress == NULL))
@@ -881,7 +932,6 @@ static LPVOID VIRTUALReserveMemory(
         pRetVal,
         pRetVal != NULL);
 
-    InternalLeaveCriticalSection(pthrCurrent, &virtual_critsec);
     return pRetVal;
 }
 
@@ -1211,7 +1261,7 @@ VirtualAlloc(
     }
 
     /* Test for un-supported flags. */
-    if ( ( flAllocationType & ~( MEM_COMMIT | MEM_RESERVE | MEM_TOP_DOWN | MEM_RESERVE_EXECUTABLE ) ) != 0 )
+    if ( ( flAllocationType & ~( MEM_COMMIT | MEM_RESERVE | MEM_RESET | MEM_TOP_DOWN | MEM_RESERVE_EXECUTABLE ) ) != 0 )
     {
         ASSERT( "flAllocationType can be one, or any combination of MEM_COMMIT, \
                MEM_RESERVE, MEM_TOP_DOWN, or MEM_RESERVE_EXECUTABLE.\n" );
@@ -1240,6 +1290,26 @@ VirtualAlloc(
         NULL,
         TRUE);
 
+    if ( flAllocationType & MEM_RESET ) 
+    {
+        if ( flAllocationType != MEM_RESET )
+        {
+            ASSERT( "MEM_RESET cannot be used with any other allocation flags in flAllocationType.\n" );
+            pthrCurrent->SetLastError( ERROR_INVALID_PARAMETER );
+            goto done;
+        }
+
+        InternalEnterCriticalSection(pthrCurrent, &virtual_critsec);
+        pRetVal = VIRTUALResetMemory( pthrCurrent, lpAddress, dwSize );
+        InternalLeaveCriticalSection(pthrCurrent, &virtual_critsec);
+
+        if ( !pRetVal )
+        {
+            /* Error messages are already displayed, just leave. */
+            goto done;
+        }
+    }
+
     if ( flAllocationType & MEM_RESERVE ) 
     {
         InternalEnterCriticalSection(pthrCurrent, &virtual_critsec);