Merge pull request #9770 from adityamandaleeka/card_bundles_unix_2

Implement second-level card tables for non-Windows platforms

4 files changed, 341 insertions, 30 deletions

diff --git a/src/vm/amd64/jithelpers_fast.S b/src/vm/amd64/jithelpers_fast.S
index 8076655ad9..0ec63f7c5c 100644
--- a/src/vm/amd64/jithelpers_fast.S
+++ b/src/vm/amd64/jithelpers_fast.S
@@ -119,6 +119,22 @@ LEAF_ENTRY JIT_WriteBarrier, _TEXT
 
     UpdateCardTable:
         mov     byte ptr [rdi + rax], 0FFh
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        NOP_2_BYTE // padding for alignment of constant
+        shr     rdi, 0x0A
+
+        movabs  rax, 0xF0F0F0F0F0F0F0F0
+        cmp     byte ptr [rdi + rax], 0FFh
+
+        .byte 0x75, 0x02
+        // jne     UpdateCardBundle_WriteWatch_PostGrow64
+        REPRET
+
+    UpdateCardBundle_WriteWatch_PostGrow64:
+        mov     byte ptr [rdi + rax], 0FFh
+#endif
+
         ret
 
     .balign 16
@@ -163,12 +179,31 @@ LEAF_ENTRY JIT_WriteBarrier, _TEXT
         // Touch the card table entry, if not already dirty.
         shr     rdi, 0Bh
         cmp     byte ptr [rdi + rax], 0FFh
-        // jne     UpdateCardTable
         .byte 0x75, 0x02
+        // jne     UpdateCardTable
         REPRET
 
     UpdateCardTable:
         mov     byte ptr [rdi + rax], 0FFh
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        NOP_6_BYTE // padding for alignment of constant
+
+        movabs  rax, 0xF0F0F0F0F0F0F0F0
+
+        // Touch the card bundle, if not already dirty.
+        // rdi is already shifted by 0xB, so shift by 0xA more
+        shr     rdi, 0x0A
+        cmp     byte ptr [rdi + rax], 0FFh
+
+        .byte 0x75, 0x02 
+        // jne     UpdateCardBundle
+        REPRET
+
+    UpdateCardBundle:
+        mov     byte ptr [rdi + rax], 0FFh
+#endif
+
         ret
 
     .balign 16
@@ -303,16 +338,36 @@ LEAF_ENTRY JIT_ByRefWriteBarrier, _TEXT
         // Check if we need to update the card table
         // Calc pCardByte
         shr     rcx, 0x0B
+        
         PREPARE_EXTERNAL_VAR g_card_table, rax
-        add     rcx, [rax]
+        mov     rax, [rax]
 
         // Check if this card is dirty
-        cmp     byte ptr [rcx], 0FFh
+        cmp     byte ptr [rcx + rax], 0FFh
+
         jne     UpdateCardTable_ByRefWriteBarrier
         REPRET
 
     UpdateCardTable_ByRefWriteBarrier:
+        mov     byte ptr [rcx + rax], 0FFh
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        // Shift rcx by 0x0A more to get the card bundle byte (we shifted by 0x0B already)
+        shr     rcx, 0x0A
+        
+        PREPARE_EXTERNAL_VAR g_card_bundle_table, rax
+        add     rcx, [rax]
+
+        // Check if this bundle byte is dirty
+        cmp     byte ptr [rcx], 0FFh
+
+        jne     UpdateCardBundle_ByRefWriteBarrier
+        REPRET
+
+    UpdateCardBundle_ByRefWriteBarrier:
         mov     byte ptr [rcx], 0FFh
+#endif
+
         ret
 
     .balign 16
diff --git a/src/vm/amd64/jithelpers_fastwritebarriers.S b/src/vm/amd64/jithelpers_fastwritebarriers.S
index 6d61b26c26..23c1115165 100644
--- a/src/vm/amd64/jithelpers_fastwritebarriers.S
+++ b/src/vm/amd64/jithelpers_fastwritebarriers.S
@@ -23,7 +23,12 @@ PATCH_LABEL JIT_WriteBarrier_PreGrow64_Patch_Label_Lower
 
         // Check the lower ephemeral region bound.
         cmp     rsi, rax
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        .byte 0x72, 0x4B
+#else
         .byte 0x72, 0x23
+#endif
         // jb      Exit_PreGrow64
 
         nop // padding for alignment of constant
@@ -40,6 +45,26 @@ PATCH_LABEL JIT_WriteBarrier_PreGrow64_Patch_Label_CardTable
 
     UpdateCardTable_PreGrow64:
         mov     byte ptr [rdi + rax], 0FFh
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        NOP_6_BYTE // padding for alignment of constant
+
+PATCH_LABEL JIT_WriteBarrier_PreGrow64_Patch_Label_CardBundleTable
+        movabs  rax, 0xF0F0F0F0F0F0F0F0
+
+        // Touch the card bundle, if not already dirty.
+        // rdi is already shifted by 0xB, so shift by 0xA more
+        shr     rdi, 0x0A
+        cmp     byte ptr [rdi + rax], 0FFh
+
+        .byte 0x75, 0x02 
+        // jne     UpdateCardBundle_PreGrow64
+        REPRET
+
+    UpdateCardBundle_PreGrow64:
+        mov     byte ptr [rdi + rax], 0FFh
+#endif
+
         ret
 
     .balign 16
@@ -69,7 +94,12 @@ PATCH_LABEL JIT_WriteBarrier_PostGrow64_Patch_Label_Lower
 
         // Check the lower and upper ephemeral region bounds
         cmp     rsi, rax
-        .byte 0x72,0x33
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        .byte 0x72, 0x53
+#else
+        .byte 0x72, 0x33
+#endif
         // jb      Exit_PostGrow64
 
         nop // padding for alignment of constant
@@ -78,7 +108,12 @@ PATCH_LABEL JIT_WriteBarrier_PostGrow64_Patch_Label_Upper
         movabs  r8, 0xF0F0F0F0F0F0F0F0
 
         cmp     rsi, r8
-        .byte 0x73,0x23
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        .byte 0x73, 0x43
+#else
+        .byte 0x73, 0x23
+#endif
         // jae     Exit_PostGrow64
 
         nop // padding for alignment of constant
@@ -95,6 +130,26 @@ PATCH_LABEL JIT_WriteBarrier_PostGrow64_Patch_Label_CardTable
 
     UpdateCardTable_PostGrow64:
         mov     byte ptr [rdi + rax], 0FFh
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        NOP_6_BYTE // padding for alignment of constant
+
+PATCH_LABEL JIT_WriteBarrier_PostGrow64_Patch_Label_CardBundleTable
+        movabs  rax, 0xF0F0F0F0F0F0F0F0
+
+        // Touch the card bundle, if not already dirty.
+        // rdi is already shifted by 0xB, so shift by 0xA more
+        shr     rdi, 0x0A
+        cmp     byte ptr [rdi + rax], 0FFh
+
+        .byte 0x75, 0x02 
+        // jne     UpdateCardBundle_PostGrow64
+        REPRET
+
+    UpdateCardBundle_PostGrow64:
+        mov     byte ptr [rdi + rax], 0FFh
+#endif
+
         ret
 
     .balign 16
@@ -134,7 +189,27 @@ PATCH_LABEL JIT_WriteBarrier_SVR64_PatchLabel_CardTable
 
     UpdateCardTable_SVR64:
         mov     byte ptr [rdi + rax], 0FFh
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        NOP_6_BYTE // padding for alignment of constant
+
+PATCH_LABEL JIT_WriteBarrier_SVR64_PatchLabel_CardBundleTable
+        movabs  rax, 0xF0F0F0F0F0F0F0F0
+
+        // Shift the address by 0xA more since already shifted by 0xB
+        shr     rdi, 0x0A
+        cmp     byte ptr [rdi + rax], 0FFh
+
+        .byte 0x75, 0x02 
+        // jne     UpdateCardBundle_SVR64
+        REPRET
+
+    UpdateCardBundle_SVR64:
+        mov     byte ptr [rdi + rax], 0FFh
+#endif
+
         ret
+
 LEAF_END_MARKED JIT_WriteBarrier_SVR64, _TEXT
 
 #endif
@@ -174,7 +249,13 @@ PATCH_LABEL JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_Lower
     CheckCardTable_WriteWatch_PreGrow64:
         // Check the lower ephemeral region bound.
         cmp     rsi, r11
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        .byte 0x72, 0x40
+#else
         .byte 0x72, 0x20
+#endif
+
         // jb      Exit_WriteWatch_PreGrow64
 
         // Touch the card table entry, if not already dirty.
@@ -189,6 +270,23 @@ PATCH_LABEL JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_CardTable
 
     UpdateCardTable_WriteWatch_PreGrow64:
         mov     byte ptr [rdi + rax], 0FFh
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        NOP_2_BYTE // padding for alignment of constant
+PATCH_LABEL JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_CardBundleTable
+        movabs  rax, 0xF0F0F0F0F0F0F0F0
+
+        shr     rdi, 0x0A
+        cmp     byte ptr [rdi + rax], 0FFh
+
+        .byte 0x75, 0x02
+        // jne     UpdateCardBundle_WriteWatch_PreGrow64
+        REPRET
+
+    UpdateCardBundle_WriteWatch_PreGrow64:
+        mov     byte ptr [rdi + rax], 0FFh
+#endif
+
         ret
 
     .balign 16
@@ -231,7 +329,12 @@ PATCH_LABEL JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_Lower
         // Check the lower and upper ephemeral region bounds
     CheckCardTable_WriteWatch_PostGrow64:
         cmp     rsi, r11
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        .byte 0x72, 0x55
+#else
         .byte 0x72, 0x3d
+#endif
         // jb      Exit_WriteWatch_PostGrow64
 
         NOP_3_BYTE // padding for alignment of constant
@@ -240,7 +343,12 @@ PATCH_LABEL JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_Upper
         movabs  r10, 0xF0F0F0F0F0F0F0F0
 
         cmp     rsi, r10
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        .byte 0x73, 0x43
+#else
         .byte 0x73, 0x2b
+#endif
         // jae     Exit_WriteWatch_PostGrow64
 
         nop // padding for alignment of constant
@@ -257,8 +365,24 @@ PATCH_LABEL JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_CardTable
 
     UpdateCardTable_WriteWatch_PostGrow64:
         mov     byte ptr [rdi + rax], 0FFh
-        ret
 
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        NOP_2_BYTE // padding for alignment of constant
+        shr     rdi, 0x0A
+
+PATCH_LABEL JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_CardBundleTable
+        movabs  rax, 0xF0F0F0F0F0F0F0F0
+        cmp     byte ptr [rdi + rax], 0FFh
+
+        .byte 0x75, 0x02
+        // jne     UpdateCardBundle_WriteWatch_PostGrow64
+        REPRET
+
+    UpdateCardBundle_WriteWatch_PostGrow64:
+        mov     byte ptr [rdi + rax], 0FFh
+#endif
+
+        ret
     .balign 16
     Exit_WriteWatch_PostGrow64:
         REPRET
@@ -279,7 +403,7 @@ LEAF_ENTRY JIT_WriteBarrier_WriteWatch_SVR64, _TEXT
         //
         // SVR GC has multiple heaps, so it cannot provide one single 
         // ephemeral region to bounds check against, so we just skip the
-        // bounds checking all together and do our card table update 
+        // bounds checking altogether and do our card table update 
         // unconditionally.
         //
 
@@ -312,7 +436,25 @@ PATCH_LABEL JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_CardTable
 
     UpdateCardTable_WriteWatch_SVR64:
         mov     byte ptr [rdi + r11], 0FFh
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        NOP // padding for alignment of constant
+
+PATCH_LABEL JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_CardBundleTable
+        movabs  r11, 0xF0F0F0F0F0F0F0F0
+
+        shr     rdi, 0x0A
+        cmp     byte ptr [rdi + r11], 0FFh
+        .byte 0x75, 0x02
+        // jne     UpdateCardBundle_WriteWatch_SVR64
+        REPRET
+
+    UpdateCardBundle_WriteWatch_SVR64:
+        mov     byte ptr [rdi + r11], 0FFh
+#endif
+
         ret
+
 LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_SVR64, _TEXT
 
 #endif
diff --git a/src/vm/amd64/jithelpers_slow.S b/src/vm/amd64/jithelpers_slow.S
index 0a5da69393..f61b42afc7 100644
--- a/src/vm/amd64/jithelpers_slow.S
+++ b/src/vm/amd64/jithelpers_slow.S
@@ -94,16 +94,36 @@ LEAF_ENTRY JIT_WriteBarrier_Debug, _TEXT
         // Check if we need to update the card table
         // Calc pCardByte
         shr     rdi, 0x0B
+
         PREPARE_EXTERNAL_VAR g_card_table, r10
-        add     rdi, [r10]
+        mov     r10, [r10]
 
         // Check if this card is dirty
-        cmp     byte ptr [rdi], 0FFh
+        cmp     byte ptr [rdi + r10], 0FFh
+
         jne     UpdateCardTable_Debug
         REPRET
 
     UpdateCardTable_Debug:
+        mov     byte ptr [rdi + r10], 0FFh
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        // Shift rdi by 0x0A more to get the card bundle byte (we shifted by 0x0B already)
+        shr     rdi, 0x0A
+
+        PREPARE_EXTERNAL_VAR g_card_bundle_table, r10
+        add     rdi, [r10]
+
+        // Check if this bundle byte is dirty
+        cmp     byte ptr [rdi], 0FFh
+
+        jne     UpdateCardBundle_Debug
+        REPRET
+
+    UpdateCardBundle_Debug:
         mov     byte ptr [rdi], 0FFh
+#endif
+
         ret
 
     .balign 16
diff --git a/src/vm/amd64/jitinterfaceamd64.cpp b/src/vm/amd64/jitinterfaceamd64.cpp
index 53d8f74f1b..6a19e274e0 100644
--- a/src/vm/amd64/jitinterfaceamd64.cpp
+++ b/src/vm/amd64/jitinterfaceamd64.cpp
@@ -20,6 +20,7 @@
 extern uint8_t* g_ephemeral_low;
 extern uint8_t* g_ephemeral_high;
 extern uint32_t* g_card_table;
+extern uint32_t* g_card_bundle_table;
 
 // Patch Labels for the various write barriers
 EXTERN_C void JIT_WriteBarrier_End();
@@ -27,17 +28,26 @@ EXTERN_C void JIT_WriteBarrier_End();
 EXTERN_C void JIT_WriteBarrier_PreGrow64(Object **dst, Object *ref);
 EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_Lower();
 EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_CardTable();
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_CardBundleTable();
+#endif
 EXTERN_C void JIT_WriteBarrier_PreGrow64_End();
 
 EXTERN_C void JIT_WriteBarrier_PostGrow64(Object **dst, Object *ref);
 EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_Lower();
 EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_Upper();
 EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_CardTable();
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_CardBundleTable();
+#endif
 EXTERN_C void JIT_WriteBarrier_PostGrow64_End();
 
 #ifdef FEATURE_SVR_GC
 EXTERN_C void JIT_WriteBarrier_SVR64(Object **dst, Object *ref);
 EXTERN_C void JIT_WriteBarrier_SVR64_PatchLabel_CardTable();
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+EXTERN_C void JIT_WriteBarrier_SVR64_PatchLabel_CardBundleTable();
+#endif
 EXTERN_C void JIT_WriteBarrier_SVR64_End();
 #endif // FEATURE_SVR_GC
 
@@ -46,6 +56,9 @@ EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64(Object **dst, Object *ref);
 EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_WriteWatchTable();
 EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_Lower();
 EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_CardTable();
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_CardBundleTable();
+#endif
 EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_End();
 
 EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64(Object **dst, Object *ref);
@@ -53,20 +66,26 @@ EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_WriteWatchTable
 EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_Lower();
 EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_Upper();
 EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_CardTable();
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_CardBundleTable();
+#endif
 EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_End();
 
 #ifdef FEATURE_SVR_GC
 EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64(Object **dst, Object *ref);
 EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_WriteWatchTable();
 EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_CardTable();
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_CardBundleTable();
+#endif
 EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_End();
 #endif // FEATURE_SVR_GC
 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
 
 WriteBarrierManager g_WriteBarrierManager;
 
-// Use this somewhat hokey macro to concantonate the function start with the patch 
-// label, this allows the code below to look relatively nice, but relies on the 
+// Use this somewhat hokey macro to concatenate the function start with the patch
+// label. This allows the code below to look relatively nice, but relies on the
 // naming convention which we have established for these helpers.
 #define CALC_PATCH_LOCATION(func,label,offset)      CalculatePatchLocation((PVOID)func, (PVOID)func##_##label, offset)
 
@@ -76,8 +95,9 @@ WriteBarrierManager::WriteBarrierManager() :
     LIMITED_METHOD_CONTRACT;
 }
 
-#ifndef CODECOVERAGE        // Deactivate alignment validation for code coverage builds 
-                            // because the instrumentation tool will not preserve alignmant constraits and we will fail.
+#ifndef CODECOVERAGE        // Deactivate alignment validation for code coverage builds
+                            // because the instrumentation tool will not preserve alignment
+                            // constraints and we will fail.
 
 void WriteBarrierManager::Validate()
 {
@@ -96,21 +116,41 @@ void WriteBarrierManager::Validate()
 
     PBYTE pLowerBoundImmediate, pUpperBoundImmediate, pCardTableImmediate;
 
-    pLowerBoundImmediate  = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_Lower, 2);
-    pCardTableImmediate   = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardTable, 2);
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+    PBYTE pCardBundleTableImmediate;
+#endif
+
+    pLowerBoundImmediate      = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_Lower, 2);
+    pCardTableImmediate       = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardTable, 2);
+
     _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pLowerBoundImmediate) & 0x7) == 0);
     _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardTableImmediate) & 0x7) == 0);
 
-    pLowerBoundImmediate  = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Lower, 2);
-    pUpperBoundImmediate  = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Upper, 2);
-    pCardTableImmediate   = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardTable, 2);
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+    pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardBundleTable, 2);
+    _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardBundleTableImmediate) & 0x7) == 0);
+#endif
+
+    pLowerBoundImmediate      = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Lower, 2);
+    pUpperBoundImmediate      = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Upper, 2);
+    pCardTableImmediate       = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardTable, 2);
     _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pLowerBoundImmediate) & 0x7) == 0);
     _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pUpperBoundImmediate) & 0x7) == 0);
     _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardTableImmediate) & 0x7) == 0);
 
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+    pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardBundleTable, 2);
+    _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardBundleTableImmediate) & 0x7) == 0);
+#endif
+
 #ifdef FEATURE_SVR_GC
-    pCardTableImmediate   = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardTable, 2);
+    pCardTableImmediate        = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardTable, 2);
     _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardTableImmediate) & 0x7) == 0);
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+    pCardBundleTableImmediate  = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardBundleTable, 2);
+    _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardBundleTableImmediate) & 0x7) == 0);
+#endif // FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
 #endif // FEATURE_SVR_GC
 
 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
@@ -119,24 +159,41 @@ void WriteBarrierManager::Validate()
     pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_WriteWatchTable, 2);
     pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_Lower, 2);
     pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardTable, 2);
+
     _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pWriteWatchTableImmediate) & 0x7) == 0);
     _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pLowerBoundImmediate) & 0x7) == 0);
     _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardTableImmediate) & 0x7) == 0);
 
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+    pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardBundleTable, 2);
+    _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardBundleTableImmediate) & 0x7) == 0);
+#endif
+
     pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_WriteWatchTable, 2);
     pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Lower, 2);
     pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Upper, 2);
     pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardTable, 2);
+
     _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pWriteWatchTableImmediate) & 0x7) == 0);
     _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pLowerBoundImmediate) & 0x7) == 0);
     _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pUpperBoundImmediate) & 0x7) == 0);
     _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardTableImmediate) & 0x7) == 0);
 
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+    pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardBundleTable, 2);
+    _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardBundleTableImmediate) & 0x7) == 0);
+#endif
+
 #ifdef FEATURE_SVR_GC
     pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_WriteWatchTable, 2);
     pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardTable, 2);
     _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pWriteWatchTableImmediate) & 0x7) == 0);
     _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardTableImmediate) & 0x7) == 0);
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+    pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardBundleTable, 2);
+    _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardBundleTableImmediate) & 0x7) == 0);
+#endif // FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
 #endif // FEATURE_SVR_GC
 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
 }
@@ -242,36 +299,51 @@ void WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier,
     {
         case WRITE_BARRIER_PREGROW64:
         {
-            m_pLowerBoundImmediate  = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_Lower, 2);
-            m_pCardTableImmediate   = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardTable, 2);
+            m_pLowerBoundImmediate      = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_Lower, 2);
+            m_pCardTableImmediate       = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardTable, 2);
 
             // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0).
             _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate);
             _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate);
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+            m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardBundleTable, 2);
+            _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate);
+#endif
             break;
         }
 
         case WRITE_BARRIER_POSTGROW64:
         {
-            m_pLowerBoundImmediate  = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Lower, 2);
-            m_pUpperBoundImmediate  = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Upper, 2);
-            m_pCardTableImmediate   = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardTable, 2);
+            m_pLowerBoundImmediate      = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Lower, 2);
+            m_pUpperBoundImmediate      = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Upper, 2);
+            m_pCardTableImmediate       = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardTable, 2);
 
             // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0).
             _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate);
             _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate);
             _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate);
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+            m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardBundleTable, 2);
+            _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate);
+#endif
             break;
         }
 
 #ifdef FEATURE_SVR_GC
         case WRITE_BARRIER_SVR64:
         {
-            m_pCardTableImmediate   = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardTable, 2);
+            m_pCardTableImmediate       = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardTable, 2);
 
             // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0).
             _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate);
-                        break;
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+            m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardBundleTable, 2);
+            _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate);
+#endif
+            break;
         }
 #endif // FEATURE_SVR_GC
 
@@ -286,6 +358,11 @@ void WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier,
             _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate);
             _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate);
             _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate);
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+            m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardBundleTable, 2);
+            _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate);
+#endif
             break;
         }
 
@@ -301,6 +378,11 @@ void WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier,
             _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate);
             _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate);
             _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate);
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+            m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardBundleTable, 2);
+            _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate);
+#endif
             break;
         }
 
@@ -313,6 +395,11 @@ void WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier,
             // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0).
             _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate);
             _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate);
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+            m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardBundleTable, 2);
+            _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate);
+#endif
             break;
         }
 #endif // FEATURE_SVR_GC
@@ -504,15 +591,14 @@ void WriteBarrierManager::UpdateEphemeralBounds(bool isRuntimeSuspended)
 
 void WriteBarrierManager::UpdateWriteWatchAndCardTableLocations(bool isRuntimeSuspended, bool bReqUpperBoundsCheck)
 {
-    // If we are told that we require an upper bounds check (GC did some heap
-    // reshuffling), we need to switch to the WriteBarrier_PostGrow function for
-    // good.
+    // If we are told that we require an upper bounds check (GC did some heap reshuffling),
+    // we need to switch to the WriteBarrier_PostGrow function for good.
 
     WriteBarrierType newType;
     if (NeedDifferentWriteBarrier(bReqUpperBoundsCheck, &newType))
     {
         ChangeWriteBarrierTo(newType, isRuntimeSuspended);
-        return; 
+        return;
     }
 
 #ifdef _DEBUG
@@ -549,6 +635,14 @@ void WriteBarrierManager::UpdateWriteWatchAndCardTableLocations(bool isRuntimeSu
         fFlushCache = true;
     }
 
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+    if (*(UINT64*)m_pCardBundleTableImmediate != (size_t)g_card_bundle_table)
+    {
+        *(UINT64*)m_pCardBundleTableImmediate = (size_t)g_card_bundle_table;
+        fFlushCache = true;
+    }
+#endif
+
     if (fFlushCache)
     {
         FlushInstructionCache(GetCurrentProcess(), (LPVOID)JIT_WriteBarrier, GetCurrentWriteBarrierSize());