diff options
author | Sean Gillespie <sean@swgillespie.me> | 2017-01-06 16:21:11 -0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-01-06 16:21:11 -0800 |
commit | c10c1ff8e3237689212606c9aa5153beec8a1778 (patch) | |
tree | 53041ba883979389d5eb22b43ce33df4312d5a68 /src/vm | |
parent | 548aec353e72117f875d6c08ffe52e2a43f01ad4 (diff) | |
download | coreclr-c10c1ff8e3237689212606c9aa5153beec8a1778.tar.gz coreclr-c10c1ff8e3237689212606c9aa5153beec8a1778.tar.bz2 coreclr-c10c1ff8e3237689212606c9aa5153beec8a1778.zip |
[Local GC] Move Software Write Watch's write barrier updates to GCToEEInterface::StompWriteBarrier (#8605)
* Move Software Write Watch's write barrier updates to use the new
GCToEEInterface::StompWriteBarrier to stomp the EE's write barrier.
* Address code review feedback, move SetCardsAfterBulkCopy to EE side of the interface
Diffstat (limited to 'src/vm')
-rw-r--r-- | src/vm/amd64/jitinterfaceamd64.cpp | 5 | ||||
-rw-r--r-- | src/vm/gcenv.ee.cpp | 43 | ||||
-rw-r--r-- | src/vm/gcenv.h | 2 | ||||
-rw-r--r-- | src/vm/gcheaputilities.cpp | 9 | ||||
-rw-r--r-- | src/vm/gcheaputilities.h | 103 | ||||
-rw-r--r-- | src/vm/gchelpers.cpp | 99 | ||||
-rw-r--r-- | src/vm/gchelpers.h | 4 |
7 files changed, 232 insertions, 33 deletions
diff --git a/src/vm/amd64/jitinterfaceamd64.cpp b/src/vm/amd64/jitinterfaceamd64.cpp index d5dec8e6e8..53d8f74f1b 100644 --- a/src/vm/amd64/jitinterfaceamd64.cpp +++ b/src/vm/amd64/jitinterfaceamd64.cpp @@ -16,7 +16,6 @@ #include "eeconfig.h" #include "excep.h" #include "threadsuspend.h" -#include "../../gc/softwarewritewatch.h" extern uint8_t* g_ephemeral_low; extern uint8_t* g_ephemeral_high; @@ -532,9 +531,9 @@ void WriteBarrierManager::UpdateWriteWatchAndCardTableLocations(bool isRuntimeSu #ifdef FEATURE_SVR_GC case WRITE_BARRIER_WRITE_WATCH_SVR64: #endif // FEATURE_SVR_GC - if (*(UINT64*)m_pWriteWatchTableImmediate != (size_t)SoftwareWriteWatch::GetTable()) + if (*(UINT64*)m_pWriteWatchTableImmediate != (size_t)g_sw_ww_table) { - *(UINT64*)m_pWriteWatchTableImmediate = (size_t)SoftwareWriteWatch::GetTable(); + *(UINT64*)m_pWriteWatchTableImmediate = (size_t)g_sw_ww_table; fFlushCache = true; } break; diff --git a/src/vm/gcenv.ee.cpp b/src/vm/gcenv.ee.cpp index 3be8384b3d..5fb83bfa04 100644 --- a/src/vm/gcenv.ee.cpp +++ b/src/vm/gcenv.ee.cpp @@ -1226,6 +1226,14 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) assert(args->lowest_address != nullptr); assert(args->highest_address != nullptr); g_card_table = args->card_table; +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + if (args->write_watch_table != nullptr) + { + assert(args->is_runtime_suspended); + g_sw_ww_table = args->write_watch_table; + } +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + ::StompWriteBarrierResize(args->is_runtime_suspended, args->requires_upper_bounds_check); // We need to make sure that other threads executing checked write barriers @@ -1241,10 +1249,10 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) return; case WriteBarrierOp::StompEphemeral: // StompEphemeral requires a new ephemeral low and a new ephemeral high - assert(args->ephemeral_lo != nullptr); - assert(args->ephemeral_hi != nullptr); - g_ephemeral_low = args->ephemeral_lo; - g_ephemeral_high = args->ephemeral_hi; + assert(args->ephemeral_low != nullptr); + assert(args->ephemeral_high != nullptr); + g_ephemeral_low = args->ephemeral_low; + g_ephemeral_high = args->ephemeral_high; ::StompWriteBarrierEphemeral(args->is_runtime_suspended); return; case WriteBarrierOp::Initialize: @@ -1255,6 +1263,8 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) assert(args->card_table != nullptr); assert(args->lowest_address != nullptr); assert(args->highest_address != nullptr); + assert(args->ephemeral_low != nullptr); + assert(args->ephemeral_high != nullptr); assert(args->is_runtime_suspended && "the runtime must be suspended here!"); assert(!args->requires_upper_bounds_check && "the ephemeral generation must be at the top of the heap!"); @@ -1263,6 +1273,31 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) g_lowest_address = args->lowest_address; VolatileStore(&g_highest_address, args->highest_address); ::StompWriteBarrierResize(true, false); + + // g_ephemeral_low/high aren't needed for the write barrier stomp, but they + // are needed in other places. + g_ephemeral_low = args->ephemeral_low; + g_ephemeral_high = args->ephemeral_high; + return; + case WriteBarrierOp::SwitchToWriteWatch: +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + assert(args->write_watch_table != nullptr); + assert(args->is_runtime_suspended && "the runtime must be suspended here!"); + g_sw_ww_table = args->write_watch_table; + g_sw_ww_enabled_for_gc_heap = true; + ::SwitchToWriteWatchBarrier(true); +#else + assert(!"should never be called without FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP"); +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + return; + case WriteBarrierOp::SwitchToNonWriteWatch: +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + assert(args->is_runtime_suspended && "the runtime must be suspended here!"); + g_sw_ww_enabled_for_gc_heap = false; + ::SwitchToNonWriteWatchBarrier(true); +#else + assert(!"should never be called without FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP"); +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP return; default: assert(!"unknown WriteBarrierOp enum"); diff --git a/src/vm/gcenv.h b/src/vm/gcenv.h index ad5baa262e..2e23b270b5 100644 --- a/src/vm/gcenv.h +++ b/src/vm/gcenv.h @@ -51,8 +51,6 @@ #include "gcenv.interlocked.h" #include "gcenv.interlocked.inl" -#include "../gc/softwarewritewatch.h" - namespace ETW { typedef enum _GC_ROOT_KIND { diff --git a/src/vm/gcheaputilities.cpp b/src/vm/gcheaputilities.cpp index ac24fa34ce..91f259d275 100644 --- a/src/vm/gcheaputilities.cpp +++ b/src/vm/gcheaputilities.cpp @@ -16,4 +16,11 @@ uint8_t* g_ephemeral_low = (uint8_t*)1; uint8_t* g_ephemeral_high = (uint8_t*)~0; // This is the global GC heap, maintained by the VM. -GPTR_IMPL(IGCHeap, g_pGCHeap);
\ No newline at end of file +GPTR_IMPL(IGCHeap, g_pGCHeap); + +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + +uint8_t* g_sw_ww_table = nullptr; +bool g_sw_ww_enabled_for_gc_heap = false; + +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
\ No newline at end of file diff --git a/src/vm/gcheaputilities.h b/src/vm/gcheaputilities.h index e5883fc919..e76a21173c 100644 --- a/src/vm/gcheaputilities.h +++ b/src/vm/gcheaputilities.h @@ -10,6 +10,31 @@ // The singular heap instance. GPTR_DECL(IGCHeap, g_pGCHeap); +#ifndef DACCESS_COMPILE +extern "C" { +#endif // !DACCESS_COMPILE +GPTR_DECL(uint8_t,g_lowest_address); +GPTR_DECL(uint8_t,g_highest_address); +GPTR_DECL(uint32_t,g_card_table); +#ifndef DACCESS_COMPILE +} +#endif // !DACCESS_COMPILE + +extern "C" uint8_t* g_ephemeral_low; +extern "C" uint8_t* g_ephemeral_high; + +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + +// Table containing the dirty state. This table is translated to exclude the lowest address it represents, see +// TranslateTableToExcludeHeapStartAddress. +extern "C" uint8_t *g_sw_ww_table; + +// Write watch may be disabled when it is not needed (between GCs for instance). This indicates whether it is enabled. +extern "C" bool g_sw_ww_enabled_for_gc_heap; + +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + + // GCHeapUtilities provides a number of static methods // that operate on the global heap instance. It can't be // instantiated. @@ -108,22 +133,74 @@ public: return IGCHeap::maxGeneration; } +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + + // Returns True if software write watch is currently enabled for the GC Heap, + // or False if it is not. + inline static bool SoftwareWriteWatchIsEnabled() + { + WRAPPER_NO_CONTRACT; + + return g_sw_ww_enabled_for_gc_heap; + } + + // In accordance with the SoftwareWriteWatch scheme, marks a given address as + // "dirty" (e.g. has been written to). + inline static void SoftwareWriteWatchSetDirty(void* address, size_t write_size) + { + LIMITED_METHOD_CONTRACT; + + // We presumably have just written something to this address, so it can't be null. + assert(address != nullptr); + + // The implementation is limited to writes of a pointer size or less. Writes larger + // than pointer size may cross page boundaries and would require us to potentially + // set more than one entry in the SWW table, which can't be done atomically under + // the current scheme. + assert(write_size <= sizeof(void*)); + + size_t table_byte_index = reinterpret_cast<size_t>(address) >> SOFTWARE_WRITE_WATCH_AddressToTableByteIndexShift; + + // The table byte index that we calculate for the address should be the same as the one + // calculated for a pointer to the end of the written region. If this were not the case, + // this write crossed a boundary and would dirty two pages. + uint8_t* end_of_write_ptr = reinterpret_cast<uint8_t*>(address) + (write_size - 1); + assert(table_byte_index == reinterpret_cast<size_t>(end_of_write_ptr) >> SOFTWARE_WRITE_WATCH_AddressToTableByteIndexShift); + uint8_t* table_address = &g_sw_ww_table[table_byte_index]; + if (*table_address == 0) + { + *table_address = 0xFF; + } + } + + // In accordance with the SoftwareWriteWatch scheme, marks a range of addresses + // as dirty, starting at the given address and with the given length. + inline static void SoftwareWriteWatchSetDirtyRegion(void* address, size_t length) + { + LIMITED_METHOD_CONTRACT; + + // We presumably have just memcopied something to this address, so it can't be null. + assert(address != nullptr); + + // The "base index" is the first index in the SWW table that covers the target + // region of memory. + size_t base_index = reinterpret_cast<size_t>(address) >> SOFTWARE_WRITE_WATCH_AddressToTableByteIndexShift; + + // The "end_index" is the last index in the SWW table that covers the target + // region of memory. + uint8_t* end_pointer = reinterpret_cast<uint8_t*>(address) + length - 1; + size_t end_index = reinterpret_cast<size_t>(end_pointer) >> SOFTWARE_WRITE_WATCH_AddressToTableByteIndexShift; + + // We'll mark the entire region of memory as dirty by memseting all entries in + // the SWW table between the start and end indexes. + memset(&g_sw_ww_table[base_index], ~0, end_index - base_index + 1); + } +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + + private: // This class should never be instantiated. GCHeapUtilities() = delete; }; -#ifndef DACCESS_COMPILE -extern "C" { -#endif // !DACCESS_COMPILE -GPTR_DECL(uint8_t,g_lowest_address); -GPTR_DECL(uint8_t,g_highest_address); -GPTR_DECL(uint32_t,g_card_table); -#ifndef DACCESS_COMPILE -} -#endif // !DACCESS_COMPILE - -extern "C" uint8_t* g_ephemeral_low; -extern "C" uint8_t* g_ephemeral_high; - #endif // _GCHEAPUTILITIES_H_
\ No newline at end of file diff --git a/src/vm/gchelpers.cpp b/src/vm/gchelpers.cpp index 20a3a29540..30f6dd0c81 100644 --- a/src/vm/gchelpers.cpp +++ b/src/vm/gchelpers.cpp @@ -35,7 +35,6 @@ #endif // FEATURE_COMINTEROP #include "rcwwalker.h" -#include "../gc/softwarewritewatch.h" //======================================================================== // @@ -1241,9 +1240,9 @@ extern "C" HCIMPL2_RAW(VOID, JIT_CheckedWriteBarrier, Object **dst, Object *ref) #endif #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - if (SoftwareWriteWatch::IsEnabledForGCHeap()) + if (GCHeapUtilities::SoftwareWriteWatchIsEnabled()) { - SoftwareWriteWatch::SetDirty(dst, sizeof(*dst)); + GCHeapUtilities::SoftwareWriteWatchSetDirty(dst, sizeof(*dst)); } #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP @@ -1298,9 +1297,9 @@ extern "C" HCIMPL2_RAW(VOID, JIT_WriteBarrier, Object **dst, Object *ref) #endif #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - if (SoftwareWriteWatch::IsEnabledForGCHeap()) + if (GCHeapUtilities::SoftwareWriteWatchIsEnabled()) { - SoftwareWriteWatch::SetDirty(dst, sizeof(*dst)); + GCHeapUtilities::SoftwareWriteWatchSetDirty(dst, sizeof(*dst)); } #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP @@ -1366,9 +1365,9 @@ void ErectWriteBarrier(OBJECTREF *dst, OBJECTREF ref) #endif #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - if (SoftwareWriteWatch::IsEnabledForGCHeap()) + if (GCHeapUtilities::SoftwareWriteWatchIsEnabled()) { - SoftwareWriteWatch::SetDirty(dst, sizeof(*dst)); + GCHeapUtilities::SoftwareWriteWatchSetDirty(dst, sizeof(*dst)); } #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP @@ -1399,10 +1398,11 @@ void ErectWriteBarrierForMT(MethodTable **dst, MethodTable *ref) if (ref->Collectible()) { #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - if (SoftwareWriteWatch::IsEnabledForGCHeap()) + if (GCHeapUtilities::SoftwareWriteWatchIsEnabled()) { - SoftwareWriteWatch::SetDirty(dst, sizeof(*dst)); + GCHeapUtilities::SoftwareWriteWatchSetDirty(dst, sizeof(*dst)); } + #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP BYTE *refObject = *(BYTE **)((MethodTable*)ref)->GetLoaderAllocatorObjectHandle(); @@ -1417,3 +1417,84 @@ void ErectWriteBarrierForMT(MethodTable **dst, MethodTable *ref) } } } + +//---------------------------------------------------------------------------- +// +// Write Barrier Support for bulk copy ("Clone") operations +// +// StartPoint is the target bulk copy start point +// len is the length of the bulk copy (in bytes) +// +// +// Performance Note: +// +// This is implemented somewhat "conservatively", that is we +// assume that all the contents of the bulk copy are object +// references. If they are not, and the value lies in the +// ephemeral range, we will set false positives in the card table. +// +// We could use the pointer maps and do this more accurately if necessary + +#if defined(_MSC_VER) && defined(_TARGET_X86_) +#pragma optimize("y", on) // Small critical routines, don't put in EBP frame +#endif //_MSC_VER && _TARGET_X86_ + +void +SetCardsAfterBulkCopy(Object **start, size_t len) +{ + // Check whether the writes were even into the heap. If not there's no card update required. + // Also if the size is smaller than a pointer, no write barrier is required. + if ((BYTE*)start < g_lowest_address || (BYTE*)start >= g_highest_address || len < sizeof(uintptr_t)) + { + return; + } + + + // Don't optimize the Generation 0 case if we are checking for write barrier violations + // since we need to update the shadow heap even in the generation 0 case. +#if defined (WRITE_BARRIER_CHECK) && !defined (SERVER_GC) + if (g_pConfig->GetHeapVerifyLevel() & EEConfig::HEAPVERIFY_BARRIERCHECK) + { + for(unsigned i=0; i < len / sizeof(Object*); i++) + { + updateGCShadow(&start[i], start[i]); + } + } +#endif //WRITE_BARRIER_CHECK && !SERVER_GC + +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + if (GCHeapUtilities::SoftwareWriteWatchIsEnabled()) + { + GCHeapUtilities::SoftwareWriteWatchSetDirtyRegion(start, len); + } +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + + size_t startAddress = (size_t)start; + size_t endAddress = startAddress + len; + size_t startingClump = startAddress >> card_byte_shift; + size_t endingClump = (endAddress + (1 << card_byte_shift) - 1) >> card_byte_shift; + + // calculate the number of clumps to mark (round_up(end) - start) + size_t clumpCount = endingClump - startingClump; + // VolatileLoadWithoutBarrier() is used here to prevent fetch of g_card_table from being reordered + // with g_lowest/highest_address check at the beginning of this function. + uint8_t* card = ((uint8_t*)VolatileLoadWithoutBarrier(&g_card_table)) + startingClump; + + // Fill the cards. To avoid cache line thrashing we check whether the cards have already been set before + // writing. + do + { + if (*card != 0xff) + { + *card = 0xff; + } + + card++; + clumpCount--; + } + while (clumpCount != 0); +} + +#if defined(_MSC_VER) && defined(_TARGET_X86_) +#pragma optimize("", on) // Go back to command line default optimizations +#endif //_MSC_VER && _TARGET_X86_
\ No newline at end of file diff --git a/src/vm/gchelpers.h b/src/vm/gchelpers.h index f5590beebe..449524aa9a 100644 --- a/src/vm/gchelpers.h +++ b/src/vm/gchelpers.h @@ -109,6 +109,8 @@ OBJECTREF AllocateObject(MethodTable *pMT extern void StompWriteBarrierEphemeral(bool isRuntimeSuspended); extern void StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck); +extern void SwitchToWriteWatchBarrier(bool isRuntimeSuspended); +extern void SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended); extern void ThrowOutOfMemoryDimensionsExceeded(); @@ -119,5 +121,5 @@ extern void ThrowOutOfMemoryDimensionsExceeded(); //======================================================================== void ErectWriteBarrier(OBJECTREF* dst, OBJECTREF ref); - +void SetCardsAfterBulkCopy(Object **start, size_t len); #endif // _GCHELPERS_H_ |