1 files changed, 339 insertions, 0 deletions
diff --git a/src/gc/softwarewritewatch.h b/src/gc/softwarewritewatch.h
new file mode 100644
index 0000000000..3c8491cecb
--- /dev/null
+++ b/src/gc/softwarewritewatch.h
@@ -0,0 +1,339 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#ifndef __SOFTWARE_WRITE_WATCH_H__
+#define __SOFTWARE_WRITE_WATCH_H__
+
+#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+#ifndef DACCESS_COMPILE
+
+extern void SwitchToWriteWatchBarrier(bool isRuntimeSuspended);
+extern void SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended);
+
+#define SOFTWARE_WRITE_WATCH_AddressToTableByteIndexShift 0xc
+
+extern "C"
+{
+    // Table containing the dirty state. This table is translated to exclude the lowest address it represents, see
+    // TranslateTableToExcludeHeapStartAddress.
+    extern uint8_t *g_sw_ww_table;
+
+    // Write watch may be disabled when it is not needed (between GCs for instance). This indicates whether it is enabled.
+    extern bool g_sw_ww_enabled_for_gc_heap;
+
+    extern uint8_t *g_lowest_address; // start address of the GC heap
+    extern uint8_t *g_highest_address; // end address of the GC heap
+}
+
+class SoftwareWriteWatch
+{
+private:
+    // The granularity of dirty state in the table is one page. Dirtiness is tracked per byte of the table so that
+    // synchronization is not required when changing the dirty state. Shifting-right an address by the following value yields
+    // the byte index of the address into the write watch table. For instance,
+    // GetTable()[address >> AddressToTableByteIndexShift] is the byte that represents the region of memory for 'address'.
+    static const uint8_t AddressToTableByteIndexShift = SOFTWARE_WRITE_WATCH_AddressToTableByteIndexShift;
+
+private:
+    static void VerifyCreated();
+    static void VerifyMemoryRegion(void *baseAddress, size_t regionByteSize);
+    static void VerifyMemoryRegion(void *baseAddress, size_t regionByteSize, void *heapStartAddress, void *heapEndAddress);
+
+public:
+    static uint8_t *GetTable();
+private:
+    static uint8_t *GetUntranslatedTable();
+    static uint8_t *GetUntranslatedTable(uint8_t *table, void *heapStartAddress);
+    static uint8_t *GetUntranslatedTableEnd();
+    static uint8_t *GetUntranslatedTableEnd(uint8_t *table, void *heapEndAddress);
+public:
+    static void InitializeUntranslatedTable(uint8_t *untranslatedTable, void *heapStartAddress);
+private:
+    static void SetUntranslatedTable(uint8_t *untranslatedTable, void *heapStartAddress);
+public:
+    static void SetResizedUntranslatedTable(uint8_t *untranslatedTable, void *heapStartAddress, void *heapEndAddress);
+    static bool IsEnabledForGCHeap();
+    static void EnableForGCHeap();
+    static void DisableForGCHeap();
+private:
+    static void *GetHeapStartAddress();
+    static void *GetHeapEndAddress();
+
+public:
+    static void StaticClose();
+
+private:
+    static size_t GetTableByteIndex(void *address);
+    static void *GetPageAddress(size_t tableByteIndex);
+public:
+    static size_t GetTableByteSize(void *heapStartAddress, void *heapEndAddress);
+    static size_t GetTableStartByteOffset(size_t byteSizeBeforeTable);
+private:
+    static uint8_t *TranslateTableToExcludeHeapStartAddress(uint8_t *table, void *heapStartAddress);
+    static void TranslateToTableRegion(void *baseAddress, size_t regionByteSize, uint8_t **tableBaseAddressRef, size_t *tableRegionByteSizeRef);
+
+public:
+    static void ClearDirty(void *baseAddress, size_t regionByteSize);
+    static void SetDirty(void *address, size_t writeByteSize);
+    static void SetDirtyRegion(void *baseAddress, size_t regionByteSize);
+private:
+    static bool GetDirtyFromBlock(uint8_t *block, uint8_t *firstPageAddressInBlock, size_t startByteIndex, size_t endByteIndex, void **dirtyPages, size_t *dirtyPageIndexRef, size_t dirtyPageCount, bool clearDirty);
+public:
+    static void GetDirty(void *baseAddress, size_t regionByteSize, void **dirtyPages, size_t *dirtyPageCountRef, bool clearDirty, bool isRuntimeSuspended);
+};
+
+inline void SoftwareWriteWatch::VerifyCreated()
+{
+    assert(GetTable() != nullptr);
+    assert(GetHeapStartAddress() != nullptr);
+    assert(GetHeapEndAddress() != nullptr);
+    assert(GetHeapStartAddress() < GetHeapEndAddress());
+}
+
+inline void SoftwareWriteWatch::VerifyMemoryRegion(void *baseAddress, size_t regionByteSize)
+{
+    VerifyMemoryRegion(baseAddress, regionByteSize, GetHeapStartAddress(), GetHeapEndAddress());
+}
+
+inline void SoftwareWriteWatch::VerifyMemoryRegion(
+    void *baseAddress,
+    size_t regionByteSize,
+    void *heapStartAddress,
+    void *heapEndAddress)
+{
+    VerifyCreated();
+    assert(baseAddress != nullptr);
+    assert(heapStartAddress != nullptr);
+    assert(heapStartAddress >= GetHeapStartAddress());
+    assert(heapEndAddress != nullptr);
+    assert(heapEndAddress <= GetHeapEndAddress());
+    assert(baseAddress >= heapStartAddress);
+    assert(baseAddress < heapEndAddress);
+    assert(regionByteSize != 0);
+    assert(regionByteSize <= reinterpret_cast<size_t>(heapEndAddress) - reinterpret_cast<size_t>(baseAddress));
+}
+
+inline uint8_t *SoftwareWriteWatch::GetTable()
+{
+    return g_sw_ww_table;
+}
+
+inline uint8_t *SoftwareWriteWatch::GetUntranslatedTable()
+{
+    VerifyCreated();
+    return GetUntranslatedTable(GetTable(), GetHeapStartAddress());
+}
+
+inline uint8_t *SoftwareWriteWatch::GetUntranslatedTable(uint8_t *table, void *heapStartAddress)
+{
+    assert(table != nullptr);
+    assert(heapStartAddress != nullptr);
+    assert(heapStartAddress >= GetHeapStartAddress());
+
+    uint8_t *untranslatedTable = table + GetTableByteIndex(heapStartAddress);
+    assert(ALIGN_DOWN(untranslatedTable, sizeof(size_t)) == untranslatedTable);
+    return untranslatedTable;
+}
+
+inline uint8_t *SoftwareWriteWatch::GetUntranslatedTableEnd()
+{
+    VerifyCreated();
+    return GetUntranslatedTableEnd(GetTable(), GetHeapEndAddress());
+}
+
+inline uint8_t *SoftwareWriteWatch::GetUntranslatedTableEnd(uint8_t *table, void *heapEndAddress)
+{
+    assert(table != nullptr);
+    assert(heapEndAddress != nullptr);
+    assert(heapEndAddress <= GetHeapEndAddress());
+
+    return ALIGN_UP(&table[GetTableByteIndex(reinterpret_cast<uint8_t *>(heapEndAddress) - 1) + 1], sizeof(size_t));
+}
+
+inline void SoftwareWriteWatch::InitializeUntranslatedTable(uint8_t *untranslatedTable, void *heapStartAddress)
+{
+    assert(GetTable() == nullptr);
+    SetUntranslatedTable(untranslatedTable, heapStartAddress);
+}
+
+inline void SoftwareWriteWatch::SetUntranslatedTable(uint8_t *untranslatedTable, void *heapStartAddress)
+{
+    assert(untranslatedTable != nullptr);
+    assert(ALIGN_DOWN(untranslatedTable, sizeof(size_t)) == untranslatedTable);
+    assert(heapStartAddress != nullptr);
+
+    g_sw_ww_table = TranslateTableToExcludeHeapStartAddress(untranslatedTable, heapStartAddress);
+}
+
+inline void SoftwareWriteWatch::SetResizedUntranslatedTable(
+    uint8_t *untranslatedTable,
+    void *heapStartAddress,
+    void *heapEndAddress)
+{
+    // The runtime needs to be suspended during this call, and background GC threads need to synchronize calls to ClearDirty()
+    // and GetDirty() such that they are not called concurrently with this function
+
+    VerifyCreated();
+    assert(untranslatedTable != nullptr);
+    assert(ALIGN_DOWN(untranslatedTable, sizeof(size_t)) == untranslatedTable);
+    assert(heapStartAddress != nullptr);
+    assert(heapEndAddress != nullptr);
+    assert(heapStartAddress <= GetHeapStartAddress());
+    assert(heapEndAddress >= GetHeapEndAddress());
+    assert(heapStartAddress < GetHeapStartAddress() || heapEndAddress > GetHeapEndAddress());
+
+    uint8_t *oldUntranslatedTable = GetUntranslatedTable();
+    void *oldTableHeapStartAddress = GetHeapStartAddress();
+    size_t oldTableByteSize = GetTableByteSize(oldTableHeapStartAddress, GetHeapEndAddress());
+    SetUntranslatedTable(untranslatedTable, heapStartAddress);
+
+    uint8_t *tableRegionStart = &GetTable()[GetTableByteIndex(oldTableHeapStartAddress)];
+    memcpy(tableRegionStart, oldUntranslatedTable, oldTableByteSize);
+}
+
+inline bool SoftwareWriteWatch::IsEnabledForGCHeap()
+{
+    return g_sw_ww_enabled_for_gc_heap;
+}
+
+inline void SoftwareWriteWatch::EnableForGCHeap()
+{
+    // The runtime needs to be suspended during this call. This is how it currently guarantees that GC heap writes from other
+    // threads between calls to EnableForGCHeap() and DisableForGCHeap() will be tracked.
+
+    VerifyCreated();
+    assert(!IsEnabledForGCHeap());
+
+    g_sw_ww_enabled_for_gc_heap = true;
+    SwitchToWriteWatchBarrier(true);
+}
+
+inline void SoftwareWriteWatch::DisableForGCHeap()
+{
+    // The runtime needs to be suspended during this call. This is how it currently guarantees that GC heap writes from other
+    // threads between calls to EnableForGCHeap() and DisableForGCHeap() will be tracked.
+
+    VerifyCreated();
+    assert(IsEnabledForGCHeap());
+
+    g_sw_ww_enabled_for_gc_heap = false;
+    SwitchToNonWriteWatchBarrier(true);
+}
+
+inline void *SoftwareWriteWatch::GetHeapStartAddress()
+{
+    return g_lowest_address;
+}
+
+inline void *SoftwareWriteWatch::GetHeapEndAddress()
+{
+    return g_highest_address;
+}
+
+inline size_t SoftwareWriteWatch::GetTableByteIndex(void *address)
+{
+    assert(address != nullptr);
+
+    size_t tableByteIndex = reinterpret_cast<size_t>(address) >> AddressToTableByteIndexShift;
+    assert(tableByteIndex != 0);
+    return tableByteIndex;
+}
+
+inline void *SoftwareWriteWatch::GetPageAddress(size_t tableByteIndex)
+{
+    assert(tableByteIndex != 0);
+
+    void *pageAddress = reinterpret_cast<void *>(tableByteIndex << AddressToTableByteIndexShift);
+    assert(pageAddress >= GetHeapStartAddress());
+    assert(pageAddress < GetHeapEndAddress());
+    assert(ALIGN_DOWN(pageAddress, OS_PAGE_SIZE) == pageAddress);
+    return pageAddress;
+}
+
+inline size_t SoftwareWriteWatch::GetTableByteSize(void *heapStartAddress, void *heapEndAddress)
+{
+    assert(heapStartAddress != nullptr);
+    assert(heapEndAddress != nullptr);
+    assert(heapStartAddress < heapEndAddress);
+
+    size_t tableByteSize =
+        GetTableByteIndex(reinterpret_cast<uint8_t *>(heapEndAddress) - 1) - GetTableByteIndex(heapStartAddress) + 1;
+    tableByteSize = ALIGN_UP(tableByteSize, sizeof(size_t));
+    return tableByteSize;
+}
+
+inline size_t SoftwareWriteWatch::GetTableStartByteOffset(size_t byteSizeBeforeTable)
+{
+    return ALIGN_UP(byteSizeBeforeTable, sizeof(size_t)); // start of the table needs to be aligned to size_t
+}
+
+inline uint8_t *SoftwareWriteWatch::TranslateTableToExcludeHeapStartAddress(uint8_t *table, void *heapStartAddress)
+{
+    assert(table != nullptr);
+    assert(heapStartAddress != nullptr);
+
+    // Exclude the table byte index corresponding to the heap start address from the table pointer, so that each lookup in the
+    // table by address does not have to calculate (address - heapStartAddress)
+    return table - GetTableByteIndex(heapStartAddress);
+}
+
+inline void SoftwareWriteWatch::TranslateToTableRegion(
+    void *baseAddress,
+    size_t regionByteSize,
+    uint8_t **tableBaseAddressRef,
+    size_t *tableRegionByteSizeRef)
+{
+    VerifyCreated();
+    VerifyMemoryRegion(baseAddress, regionByteSize);
+    assert(tableBaseAddressRef != nullptr);
+    assert(tableRegionByteSizeRef != nullptr);
+
+    size_t baseAddressTableByteIndex = GetTableByteIndex(baseAddress);
+    *tableBaseAddressRef = &GetTable()[baseAddressTableByteIndex];
+    *tableRegionByteSizeRef =
+        GetTableByteIndex(reinterpret_cast<uint8_t *>(baseAddress) + (regionByteSize - 1)) - baseAddressTableByteIndex + 1;
+}
+
+inline void SoftwareWriteWatch::ClearDirty(void *baseAddress, size_t regionByteSize)
+{
+    VerifyCreated();
+    VerifyMemoryRegion(baseAddress, regionByteSize);
+
+    uint8_t *tableBaseAddress;
+    size_t tableRegionByteSize;
+    TranslateToTableRegion(baseAddress, regionByteSize, &tableBaseAddress, &tableRegionByteSize);
+    memset(tableBaseAddress, 0, tableRegionByteSize);
+}
+
+inline void SoftwareWriteWatch::SetDirty(void *address, size_t writeByteSize)
+{
+    VerifyCreated();
+    VerifyMemoryRegion(address, writeByteSize);
+    assert(address != nullptr);
+    assert(writeByteSize <= sizeof(void *));
+
+    size_t tableByteIndex = GetTableByteIndex(address);
+    assert(GetTableByteIndex(reinterpret_cast<uint8_t *>(address) + (writeByteSize - 1)) == tableByteIndex);
+
+    uint8_t *tableByteAddress = &GetTable()[tableByteIndex];
+    if (*tableByteAddress == 0)
+    {
+        *tableByteAddress = 0xff;
+    }
+}
+
+inline void SoftwareWriteWatch::SetDirtyRegion(void *baseAddress, size_t regionByteSize)
+{
+    VerifyCreated();
+    VerifyMemoryRegion(baseAddress, regionByteSize);
+
+    uint8_t *tableBaseAddress;
+    size_t tableRegionByteSize;
+    TranslateToTableRegion(baseAddress, regionByteSize, &tableBaseAddress, &tableRegionByteSize);
+    memset(tableBaseAddress, ~0, tableRegionByteSize);
+}
+
+#endif // !DACCESS_COMPILE
+#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+#endif // !__SOFTWARE_WRITE_WATCH_H__