[Local GC] Move Software Write Watch's write barrier updates to GCToEEInterface::StompWriteBarrier (#8605)

* Move Software Write Watch's write barrier updates to use the new
GCToEEInterface::StompWriteBarrier to stomp the EE's write barrier.

* Address code review feedback, move SetCardsAfterBulkCopy to EE side of the interface

7 files changed, 232 insertions, 33 deletions

diff --git a/src/vm/amd64/jitinterfaceamd64.cpp b/src/vm/amd64/jitinterfaceamd64.cpp
index d5dec8e6e8..53d8f74f1b 100644
--- a/src/vm/amd64/jitinterfaceamd64.cpp
+++ b/src/vm/amd64/jitinterfaceamd64.cpp
@@ -16,7 +16,6 @@
 #include "eeconfig.h"
 #include "excep.h"
 #include "threadsuspend.h"
-#include "../../gc/softwarewritewatch.h"
 
 extern uint8_t* g_ephemeral_low;
 extern uint8_t* g_ephemeral_high;
@@ -532,9 +531,9 @@ void WriteBarrierManager::UpdateWriteWatchAndCardTableLocations(bool isRuntimeSu
 #ifdef FEATURE_SVR_GC
         case WRITE_BARRIER_WRITE_WATCH_SVR64:
 #endif // FEATURE_SVR_GC
-            if (*(UINT64*)m_pWriteWatchTableImmediate != (size_t)SoftwareWriteWatch::GetTable())
+            if (*(UINT64*)m_pWriteWatchTableImmediate != (size_t)g_sw_ww_table)
             {
-                *(UINT64*)m_pWriteWatchTableImmediate = (size_t)SoftwareWriteWatch::GetTable();
+                *(UINT64*)m_pWriteWatchTableImmediate = (size_t)g_sw_ww_table;
                 fFlushCache = true;
             }
             break;
diff --git a/src/vm/gcenv.ee.cpp b/src/vm/gcenv.ee.cpp
index 3be8384b3d..5fb83bfa04 100644
--- a/src/vm/gcenv.ee.cpp
+++ b/src/vm/gcenv.ee.cpp
@@ -1226,6 +1226,14 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
         assert(args->lowest_address != nullptr);
         assert(args->highest_address != nullptr);
         g_card_table = args->card_table;
+#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+        if (args->write_watch_table != nullptr)
+        {
+            assert(args->is_runtime_suspended);
+            g_sw_ww_table = args->write_watch_table;
+        }
+#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+
         ::StompWriteBarrierResize(args->is_runtime_suspended, args->requires_upper_bounds_check);
 
         // We need to make sure that other threads executing checked write barriers
@@ -1241,10 +1249,10 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
         return;
     case WriteBarrierOp::StompEphemeral:
         // StompEphemeral requires a new ephemeral low and a new ephemeral high
-        assert(args->ephemeral_lo != nullptr);
-        assert(args->ephemeral_hi != nullptr);
-        g_ephemeral_low = args->ephemeral_lo;
-        g_ephemeral_high = args->ephemeral_hi;
+        assert(args->ephemeral_low != nullptr);
+        assert(args->ephemeral_high != nullptr);
+        g_ephemeral_low = args->ephemeral_low;
+        g_ephemeral_high = args->ephemeral_high;
         ::StompWriteBarrierEphemeral(args->is_runtime_suspended);
         return;
     case WriteBarrierOp::Initialize:
@@ -1255,6 +1263,8 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
         assert(args->card_table != nullptr);
         assert(args->lowest_address != nullptr);
         assert(args->highest_address != nullptr);
+        assert(args->ephemeral_low != nullptr);
+        assert(args->ephemeral_high != nullptr);
         assert(args->is_runtime_suspended && "the runtime must be suspended here!");
         assert(!args->requires_upper_bounds_check && "the ephemeral generation must be at the top of the heap!");
 
@@ -1263,6 +1273,31 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
         g_lowest_address = args->lowest_address;
         VolatileStore(&g_highest_address, args->highest_address);
         ::StompWriteBarrierResize(true, false);
+
+        // g_ephemeral_low/high aren't needed for the write barrier stomp, but they
+        // are needed in other places.
+        g_ephemeral_low = args->ephemeral_low;
+        g_ephemeral_high = args->ephemeral_high;
+        return;
+    case WriteBarrierOp::SwitchToWriteWatch:
+#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+        assert(args->write_watch_table != nullptr);
+        assert(args->is_runtime_suspended && "the runtime must be suspended here!");
+        g_sw_ww_table = args->write_watch_table;
+        g_sw_ww_enabled_for_gc_heap = true;
+        ::SwitchToWriteWatchBarrier(true);
+#else
+        assert(!"should never be called without FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP");
+#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+        return;
+    case WriteBarrierOp::SwitchToNonWriteWatch:
+#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+        assert(args->is_runtime_suspended && "the runtime must be suspended here!");
+        g_sw_ww_enabled_for_gc_heap = false;
+        ::SwitchToNonWriteWatchBarrier(true);
+#else
+        assert(!"should never be called without FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP");
+#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
         return;
     default:
         assert(!"unknown WriteBarrierOp enum");
diff --git a/src/vm/gcenv.h b/src/vm/gcenv.h
index ad5baa262e..2e23b270b5 100644
--- a/src/vm/gcenv.h
+++ b/src/vm/gcenv.h
@@ -51,8 +51,6 @@
 #include "gcenv.interlocked.h"
 #include "gcenv.interlocked.inl"
 
-#include "../gc/softwarewritewatch.h"
-
 namespace ETW
 {
     typedef  enum _GC_ROOT_KIND {
diff --git a/src/vm/gcheaputilities.cpp b/src/vm/gcheaputilities.cpp
index ac24fa34ce..91f259d275 100644
--- a/src/vm/gcheaputilities.cpp
+++ b/src/vm/gcheaputilities.cpp
@@ -16,4 +16,11 @@ uint8_t* g_ephemeral_low  = (uint8_t*)1;
 uint8_t* g_ephemeral_high = (uint8_t*)~0;
 
 // This is the global GC heap, maintained by the VM.
-GPTR_IMPL(IGCHeap, g_pGCHeap);
\ No newline at end of file
+GPTR_IMPL(IGCHeap, g_pGCHeap);
+
+#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+
+uint8_t* g_sw_ww_table = nullptr;
+bool g_sw_ww_enabled_for_gc_heap = false;
+
+#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
\ No newline at end of file
diff --git a/src/vm/gcheaputilities.h b/src/vm/gcheaputilities.h
index e5883fc919..e76a21173c 100644
--- a/src/vm/gcheaputilities.h
+++ b/src/vm/gcheaputilities.h
@@ -10,6 +10,31 @@
 // The singular heap instance.
 GPTR_DECL(IGCHeap, g_pGCHeap);
 
+#ifndef DACCESS_COMPILE
+extern "C" {
+#endif // !DACCESS_COMPILE
+GPTR_DECL(uint8_t,g_lowest_address);
+GPTR_DECL(uint8_t,g_highest_address);
+GPTR_DECL(uint32_t,g_card_table);
+#ifndef DACCESS_COMPILE
+}
+#endif // !DACCESS_COMPILE
+
+extern "C" uint8_t* g_ephemeral_low;
+extern "C" uint8_t* g_ephemeral_high;
+
+#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+
+// Table containing the dirty state. This table is translated to exclude the lowest address it represents, see
+// TranslateTableToExcludeHeapStartAddress.
+extern "C" uint8_t *g_sw_ww_table;
+
+// Write watch may be disabled when it is not needed (between GCs for instance). This indicates whether it is enabled.
+extern "C" bool g_sw_ww_enabled_for_gc_heap;
+
+#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+
+
 // GCHeapUtilities provides a number of static methods
 // that operate on the global heap instance. It can't be
 // instantiated.
@@ -108,22 +133,74 @@ public:
         return IGCHeap::maxGeneration;
     }
 
+#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+
+    // Returns True if software write watch is currently enabled for the GC Heap,
+    // or False if it is not.
+    inline static bool SoftwareWriteWatchIsEnabled()
+    {
+        WRAPPER_NO_CONTRACT;
+
+        return g_sw_ww_enabled_for_gc_heap;
+    }
+
+    // In accordance with the SoftwareWriteWatch scheme, marks a given address as
+    // "dirty" (e.g. has been written to).
+    inline static void SoftwareWriteWatchSetDirty(void* address, size_t write_size)
+    {
+        LIMITED_METHOD_CONTRACT;
+
+        // We presumably have just written something to this address, so it can't be null.
+        assert(address != nullptr);
+
+        // The implementation is limited to writes of a pointer size or less. Writes larger
+        // than pointer size may cross page boundaries and would require us to potentially
+        // set more than one entry in the SWW table, which can't be done atomically under
+        // the current scheme.
+        assert(write_size <= sizeof(void*));
+
+        size_t table_byte_index = reinterpret_cast<size_t>(address) >> SOFTWARE_WRITE_WATCH_AddressToTableByteIndexShift;
+
+        // The table byte index that we calculate for the address should be the same as the one
+        // calculated for a pointer to the end of the written region. If this were not the case,
+        // this write crossed a boundary and would dirty two pages.
+        uint8_t* end_of_write_ptr = reinterpret_cast<uint8_t*>(address) + (write_size - 1);
+        assert(table_byte_index == reinterpret_cast<size_t>(end_of_write_ptr) >> SOFTWARE_WRITE_WATCH_AddressToTableByteIndexShift);
+        uint8_t* table_address = &g_sw_ww_table[table_byte_index];
+        if (*table_address == 0)
+        {
+            *table_address = 0xFF;
+        }
+    }
+
+    // In accordance with the SoftwareWriteWatch scheme, marks a range of addresses
+    // as dirty, starting at the given address and with the given length.
+    inline static void SoftwareWriteWatchSetDirtyRegion(void* address, size_t length)
+    {
+        LIMITED_METHOD_CONTRACT;
+
+        // We presumably have just memcopied something to this address, so it can't be null.
+        assert(address != nullptr);
+
+        // The "base index" is the first index in the SWW table that covers the target
+        // region of memory.
+        size_t base_index = reinterpret_cast<size_t>(address) >> SOFTWARE_WRITE_WATCH_AddressToTableByteIndexShift;
+
+        // The "end_index" is the last index in the SWW table that covers the target
+        // region of memory.
+        uint8_t* end_pointer = reinterpret_cast<uint8_t*>(address) + length - 1;
+        size_t end_index = reinterpret_cast<size_t>(end_pointer) >> SOFTWARE_WRITE_WATCH_AddressToTableByteIndexShift;
+
+        // We'll mark the entire region of memory as dirty by memseting all entries in
+        // the SWW table between the start and end indexes.
+        memset(&g_sw_ww_table[base_index], ~0, end_index - base_index + 1);
+    }
+#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+
+
 private:
     // This class should never be instantiated.
     GCHeapUtilities() = delete;
 };
 
-#ifndef DACCESS_COMPILE
-extern "C" {
-#endif // !DACCESS_COMPILE
-GPTR_DECL(uint8_t,g_lowest_address);
-GPTR_DECL(uint8_t,g_highest_address);
-GPTR_DECL(uint32_t,g_card_table);
-#ifndef DACCESS_COMPILE
-} 
-#endif // !DACCESS_COMPILE
-
-extern "C" uint8_t* g_ephemeral_low;
-extern "C" uint8_t* g_ephemeral_high;
-
 #endif // _GCHEAPUTILITIES_H_
\ No newline at end of file
diff --git a/src/vm/gchelpers.cpp b/src/vm/gchelpers.cpp
index 20a3a29540..30f6dd0c81 100644
--- a/src/vm/gchelpers.cpp
+++ b/src/vm/gchelpers.cpp
@@ -35,7 +35,6 @@
 #endif // FEATURE_COMINTEROP
 
 #include "rcwwalker.h"
-#include "../gc/softwarewritewatch.h"
 
 //========================================================================
 //
@@ -1241,9 +1240,9 @@ extern "C" HCIMPL2_RAW(VOID, JIT_CheckedWriteBarrier, Object **dst, Object *ref)
 #endif
 
 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
-    if (SoftwareWriteWatch::IsEnabledForGCHeap())
+    if (GCHeapUtilities::SoftwareWriteWatchIsEnabled())
     {
-        SoftwareWriteWatch::SetDirty(dst, sizeof(*dst));
+        GCHeapUtilities::SoftwareWriteWatchSetDirty(dst, sizeof(*dst));
     }
 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
 
@@ -1298,9 +1297,9 @@ extern "C" HCIMPL2_RAW(VOID, JIT_WriteBarrier, Object **dst, Object *ref)
 #endif
     
 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
-    if (SoftwareWriteWatch::IsEnabledForGCHeap())
+    if (GCHeapUtilities::SoftwareWriteWatchIsEnabled())
     {
-        SoftwareWriteWatch::SetDirty(dst, sizeof(*dst));
+        GCHeapUtilities::SoftwareWriteWatchSetDirty(dst, sizeof(*dst));
     }
 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
 
@@ -1366,9 +1365,9 @@ void ErectWriteBarrier(OBJECTREF *dst, OBJECTREF ref)
 #endif
 
 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
-    if (SoftwareWriteWatch::IsEnabledForGCHeap())
+    if (GCHeapUtilities::SoftwareWriteWatchIsEnabled())
     {
-        SoftwareWriteWatch::SetDirty(dst, sizeof(*dst));
+        GCHeapUtilities::SoftwareWriteWatchSetDirty(dst, sizeof(*dst));
     }
 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
 
@@ -1399,10 +1398,11 @@ void ErectWriteBarrierForMT(MethodTable **dst, MethodTable *ref)
     if (ref->Collectible())
     {
 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
-        if (SoftwareWriteWatch::IsEnabledForGCHeap())
+        if (GCHeapUtilities::SoftwareWriteWatchIsEnabled())
         {
-            SoftwareWriteWatch::SetDirty(dst, sizeof(*dst));
+            GCHeapUtilities::SoftwareWriteWatchSetDirty(dst, sizeof(*dst));
         }
+
 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
 
         BYTE *refObject = *(BYTE **)((MethodTable*)ref)->GetLoaderAllocatorObjectHandle();
@@ -1417,3 +1417,84 @@ void ErectWriteBarrierForMT(MethodTable **dst, MethodTable *ref)
         }
     }
 }
+
+//----------------------------------------------------------------------------
+//
+// Write Barrier Support for bulk copy ("Clone") operations
+//
+// StartPoint is the target bulk copy start point
+// len is the length of the bulk copy (in bytes)
+//
+//
+// Performance Note:
+//
+// This is implemented somewhat "conservatively", that is we
+// assume that all the contents of the bulk copy are object
+// references.  If they are not, and the value lies in the
+// ephemeral range, we will set false positives in the card table.
+//
+// We could use the pointer maps and do this more accurately if necessary
+
+#if defined(_MSC_VER) && defined(_TARGET_X86_)
+#pragma optimize("y", on)        // Small critical routines, don't put in EBP frame
+#endif //_MSC_VER && _TARGET_X86_
+
+void
+SetCardsAfterBulkCopy(Object **start, size_t len)
+{
+    // Check whether the writes were even into the heap. If not there's no card update required.
+    // Also if the size is smaller than a pointer, no write barrier is required.
+    if ((BYTE*)start < g_lowest_address || (BYTE*)start >= g_highest_address || len < sizeof(uintptr_t))
+    {
+        return;
+    }
+
+
+    // Don't optimize the Generation 0 case if we are checking for write barrier violations
+    // since we need to update the shadow heap even in the generation 0 case.
+#if defined (WRITE_BARRIER_CHECK) && !defined (SERVER_GC)
+    if (g_pConfig->GetHeapVerifyLevel() & EEConfig::HEAPVERIFY_BARRIERCHECK)
+    {
+        for(unsigned i=0; i < len / sizeof(Object*); i++)
+        {
+            updateGCShadow(&start[i], start[i]);
+        }
+    }
+#endif //WRITE_BARRIER_CHECK && !SERVER_GC
+
+#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+    if (GCHeapUtilities::SoftwareWriteWatchIsEnabled())
+    {
+        GCHeapUtilities::SoftwareWriteWatchSetDirtyRegion(start, len);
+    }
+#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+
+    size_t startAddress = (size_t)start;
+    size_t endAddress = startAddress + len;
+    size_t startingClump = startAddress >> card_byte_shift;
+    size_t endingClump = (endAddress + (1 << card_byte_shift) - 1) >> card_byte_shift;
+
+    // calculate the number of clumps to mark (round_up(end) - start)
+    size_t clumpCount = endingClump - startingClump;
+    // VolatileLoadWithoutBarrier() is used here to prevent fetch of g_card_table from being reordered
+    // with g_lowest/highest_address check at the beginning of this function.
+    uint8_t* card = ((uint8_t*)VolatileLoadWithoutBarrier(&g_card_table)) + startingClump;
+
+    // Fill the cards. To avoid cache line thrashing we check whether the cards have already been set before
+    // writing.
+    do
+    {
+        if (*card != 0xff)
+        {
+            *card = 0xff;
+        }
+
+        card++;
+        clumpCount--;
+    }
+    while (clumpCount != 0);
+}
+
+#if defined(_MSC_VER) && defined(_TARGET_X86_)
+#pragma optimize("", on)        // Go back to command line default optimizations
+#endif //_MSC_VER && _TARGET_X86_
\ No newline at end of file
diff --git a/src/vm/gchelpers.h b/src/vm/gchelpers.h
index f5590beebe..449524aa9a 100644
--- a/src/vm/gchelpers.h
+++ b/src/vm/gchelpers.h
@@ -109,6 +109,8 @@ OBJECTREF AllocateObject(MethodTable *pMT
 
 extern void StompWriteBarrierEphemeral(bool isRuntimeSuspended);
 extern void StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck);
+extern void SwitchToWriteWatchBarrier(bool isRuntimeSuspended);
+extern void SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended);
 
 extern void ThrowOutOfMemoryDimensionsExceeded();
 
@@ -119,5 +121,5 @@ extern void ThrowOutOfMemoryDimensionsExceeded();
 //========================================================================
 
 void ErectWriteBarrier(OBJECTREF* dst, OBJECTREF ref);
-
+void SetCardsAfterBulkCopy(Object **start, size_t len);
 #endif // _GCHELPERS_H_