Implement card bundles for non-Windows platforms.

1 files changed, 149 insertions, 4 deletions

diff --git a/src/vm/amd64/jithelpers_fastwritebarriers.S b/src/vm/amd64/jithelpers_fastwritebarriers.S
index 6d61b26c26..eff33cd890 100644
--- a/src/vm/amd64/jithelpers_fastwritebarriers.S
+++ b/src/vm/amd64/jithelpers_fastwritebarriers.S
@@ -23,7 +23,12 @@ PATCH_LABEL JIT_WriteBarrier_PreGrow64_Patch_Label_Lower
 
         // Check the lower ephemeral region bound.
         cmp     rsi, rax
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        .byte 0x72, 0x4B
+#else
         .byte 0x72, 0x23
+#endif
         // jb      Exit_PreGrow64
 
         nop // padding for alignment of constant
@@ -40,6 +45,27 @@ PATCH_LABEL JIT_WriteBarrier_PreGrow64_Patch_Label_CardTable
 
     UpdateCardTable_PreGrow64:
         mov     byte ptr [rdi + rax], 0FFh
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        NOP_3_BYTE // padding for alignment of constant
+        NOP_3_BYTE // padding for alignment of constant
+
+PATCH_LABEL JIT_WriteBarrier_PreGrow64_Patch_Label_CardBundleTable
+        movabs  rax, 0xF0F0F0F0F0F0F0F0
+
+        // Touch the card bundle, if not already dirty.
+        // rdi is already shifted by 0xB, so shift by 0xA more
+        shr     rdi, 0x0A
+        cmp     byte ptr [rdi + rax], 0FFh
+
+        .byte 0x75, 0x02 
+        // jne     UpdateCardBundle_PreGrow64
+        REPRET
+
+    UpdateCardBundle_PreGrow64:
+        mov     byte ptr [rdi + rax], 0FFh
+#endif
+
         ret
 
     .balign 16
@@ -69,7 +95,12 @@ PATCH_LABEL JIT_WriteBarrier_PostGrow64_Patch_Label_Lower
 
         // Check the lower and upper ephemeral region bounds
         cmp     rsi, rax
-        .byte 0x72,0x33
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        .byte 0x72, 0x53
+#else
+        .byte 0x72, 0x33
+#endif
         // jb      Exit_PostGrow64
 
         nop // padding for alignment of constant
@@ -78,7 +109,12 @@ PATCH_LABEL JIT_WriteBarrier_PostGrow64_Patch_Label_Upper
         movabs  r8, 0xF0F0F0F0F0F0F0F0
 
         cmp     rsi, r8
-        .byte 0x73,0x23
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        .byte 0x73, 0x43
+#else
+        .byte 0x73, 0x23
+#endif
         // jae     Exit_PostGrow64
 
         nop // padding for alignment of constant
@@ -95,6 +131,27 @@ PATCH_LABEL JIT_WriteBarrier_PostGrow64_Patch_Label_CardTable
 
     UpdateCardTable_PostGrow64:
         mov     byte ptr [rdi + rax], 0FFh
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        NOP_3_BYTE // padding for alignment of constant
+        NOP_3_BYTE // padding for alignment of constant
+
+PATCH_LABEL JIT_WriteBarrier_PostGrow64_Patch_Label_CardBundleTable
+        movabs  rax, 0xF0F0F0F0F0F0F0F0
+
+        // Touch the card bundle, if not already dirty.
+        // rdi is already shifted by 0xB, so shift by 0xA more
+        shr     rdi, 0x0A
+        cmp     byte ptr [rdi + rax], 0FFh
+
+        .byte 0x75, 0x02 
+        // jne     UpdateCardBundle_PostGrow64
+        REPRET
+
+    UpdateCardBundle_PostGrow64:
+        mov     byte ptr [rdi + rax], 0FFh
+#endif
+
         ret
 
     .balign 16
@@ -134,7 +191,28 @@ PATCH_LABEL JIT_WriteBarrier_SVR64_PatchLabel_CardTable
 
     UpdateCardTable_SVR64:
         mov     byte ptr [rdi + rax], 0FFh
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        NOP_3_BYTE // padding for alignment of constant
+        NOP_3_BYTE // padding for alignment of constant
+
+PATCH_LABEL JIT_WriteBarrier_SVR64_PatchLabel_CardBundleTable
+        movabs  rax, 0xF0F0F0F0F0F0F0F0
+
+        // Shift the address by 0xA more since already shifted by 0xB
+        shr     rdi, 0x0A
+        cmp     byte ptr [rdi + rax], 0FFh
+
+        .byte 0x75, 0x02 
+        // jne     UpdateCardBundle_SVR64
+        REPRET
+
+    UpdateCardBundle_SVR64:
+        mov     byte ptr [rdi + rax], 0FFh
+#endif
+
         ret
+
 LEAF_END_MARKED JIT_WriteBarrier_SVR64, _TEXT
 
 #endif
@@ -174,7 +252,13 @@ PATCH_LABEL JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_Lower
     CheckCardTable_WriteWatch_PreGrow64:
         // Check the lower ephemeral region bound.
         cmp     rsi, r11
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        .byte 0x72, 0x40
+#else
         .byte 0x72, 0x20
+#endif
+
         // jb      Exit_WriteWatch_PreGrow64
 
         // Touch the card table entry, if not already dirty.
@@ -189,6 +273,23 @@ PATCH_LABEL JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_CardTable
 
     UpdateCardTable_WriteWatch_PreGrow64:
         mov     byte ptr [rdi + rax], 0FFh
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        NOP_2_BYTE // padding for alignment of constant
+PATCH_LABEL JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_CardBundleTable
+        movabs  rax, 0xF0F0F0F0F0F0F0F0
+
+        shr     rdi, 0x0A
+        cmp     byte ptr [rdi + rax], 0FFh
+
+        .byte 0x75, 0x02
+        // jne     UpdateCardBundle_WriteWatch_PreGrow64
+        REPRET
+
+    UpdateCardBundle_WriteWatch_PreGrow64:
+        mov     byte ptr [rdi + rax], 0FFh
+#endif
+
         ret
 
     .balign 16
@@ -231,7 +332,12 @@ PATCH_LABEL JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_Lower
         // Check the lower and upper ephemeral region bounds
     CheckCardTable_WriteWatch_PostGrow64:
         cmp     rsi, r11
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        .byte 0x72, 0x55
+#else
         .byte 0x72, 0x3d
+#endif
         // jb      Exit_WriteWatch_PostGrow64
 
         NOP_3_BYTE // padding for alignment of constant
@@ -240,7 +346,12 @@ PATCH_LABEL JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_Upper
         movabs  r10, 0xF0F0F0F0F0F0F0F0
 
         cmp     rsi, r10
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        .byte 0x73, 0x43
+#else
         .byte 0x73, 0x2b
+#endif
         // jae     Exit_WriteWatch_PostGrow64
 
         nop // padding for alignment of constant
@@ -257,8 +368,24 @@ PATCH_LABEL JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_CardTable
 
     UpdateCardTable_WriteWatch_PostGrow64:
         mov     byte ptr [rdi + rax], 0FFh
-        ret
 
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        NOP_2_BYTE // padding for alignment of constant
+        shr     rdi, 0x0A
+
+PATCH_LABEL JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_CardBundleTable
+        movabs  rax, 0xF0F0F0F0F0F0F0F0
+        cmp     byte ptr [rdi + rax], 0FFh
+
+        .byte 0x75, 0x02
+        // jne     UpdateCardBundle_WriteWatch_PostGrow64
+        REPRET
+
+    UpdateCardBundle_WriteWatch_PostGrow64:
+        mov     byte ptr [rdi + rax], 0FFh
+#endif
+
+        ret
     .balign 16
     Exit_WriteWatch_PostGrow64:
         REPRET
@@ -279,7 +406,7 @@ LEAF_ENTRY JIT_WriteBarrier_WriteWatch_SVR64, _TEXT
         //
         // SVR GC has multiple heaps, so it cannot provide one single 
         // ephemeral region to bounds check against, so we just skip the
-        // bounds checking all together and do our card table update 
+        // bounds checking altogether and do our card table update 
         // unconditionally.
         //
 
@@ -312,7 +439,25 @@ PATCH_LABEL JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_CardTable
 
     UpdateCardTable_WriteWatch_SVR64:
         mov     byte ptr [rdi + r11], 0FFh
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        NOP // padding for alignment of constant
+
+PATCH_LABEL JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_CardBundleTable
+        movabs  r11, 0xF0F0F0F0F0F0F0F0
+
+        shr     rdi, 0x0A
+        cmp     byte ptr [rdi + r11], 0FFh
+        .byte 0x75, 0x02
+        // jne     UpdateCardBundle_WriteWatch_SVR64
+        REPRET
+
+    UpdateCardBundle_WriteWatch_SVR64:
+        mov     byte ptr [rdi + r11], 0FFh
+#endif
+
         ret
+
 LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_SVR64, _TEXT
 
 #endif