Merge pull request #22003 from adityamandaleeka/writebarrier_updates_arm64

Write barrier optimizations for ARM64 Windows

2 files changed, 149 insertions, 51 deletions

diff --git a/src/vm/arm64/asmhelpers.asm b/src/vm/arm64/asmhelpers.asm
index 07e78cbd56..37efbaeccd 100644
--- a/src/vm/arm64/asmhelpers.asm
+++ b/src/vm/arm64/asmhelpers.asm
@@ -276,6 +276,108 @@ ThePreStubPatchLabel
 
     MEND
 
+; ------------------------------------------------------------------
+; Start of the writeable code region
+    LEAF_ENTRY JIT_PatchedCodeStart
+        ret      lr
+    LEAF_END
+
+;-----------------------------------------------------------------------------
+; void JIT_UpdateWriteBarrierState(bool skipEphemeralCheck)
+;
+; Update shadow copies of the various state info required for barrier
+;
+; State info is contained in a literal pool at the end of the function
+; Placed in text section so that it is close enough to use ldr literal and still
+; be relocatable. Eliminates need for PREPARE_EXTERNAL_VAR in hot code.
+;
+; Align and group state info together so it fits in a single cache line
+; and each entry can be written atomically
+;
+    WRITE_BARRIER_ENTRY JIT_UpdateWriteBarrierState
+        PROLOG_SAVE_REG_PAIR   fp, lr, #-16!
+
+        ; x0-x7 will contain intended new state
+        ; x8 will preserve skipEphemeralCheck
+        ; x12 will be used for pointers
+
+        mov      x8, x0
+
+        adrp     x12, g_card_table
+        ldr      x0, [x12, g_card_table]
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        adrp     x12, g_card_bundle_table
+        ldr      x1, [x12, g_card_bundle_table]
+#endif
+
+#ifdef WRITE_BARRIER_CHECK
+        adrp     x12, $g_GCShadow
+        ldr      x2, [x12, $g_GCShadow]
+#endif
+
+#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+        adrp     x12, g_sw_ww_table
+        ldr      x3, [x12, g_sw_ww_table]
+#endif
+
+        adrp     x12, g_ephemeral_low
+        ldr      x4, [x12, g_ephemeral_low]
+
+        adrp     x12, g_ephemeral_high
+        ldr      x5, [x12, g_ephemeral_high]
+
+        ; Check skipEphemeralCheck
+        cbz      x8, EphemeralCheckEnabled
+        movz     x4, #0
+        movn     x5, #0
+
+EphemeralCheckEnabled
+        adrp     x12, g_lowest_address
+        ldr      x6, [x12, g_lowest_address]
+
+        adrp     x12, g_highest_address
+        ldr      x7, [x12, g_highest_address]
+
+        ; Update wbs state
+        adr      x12, wbs_begin
+        stp      x0, x1, [x12], 16
+        stp      x2, x3, [x12], 16
+        stp      x4, x5, [x12], 16
+        stp      x6, x7, [x12], 16
+
+        EPILOG_RESTORE_REG_PAIR fp, lr, #16!
+        EPILOG_RETURN
+
+        ; Begin patchable literal pool
+        ALIGN 64  ; Align to power of two at least as big as patchable literal pool so that it fits optimally in cache line
+
+wbs_begin
+wbs_card_table
+        DCQ 0
+wbs_card_bundle_table
+        DCQ 0
+wbs_GCShadow
+        DCQ 0
+wbs_sw_ww_table
+        DCQ 0
+wbs_ephemeral_low
+        DCQ 0
+wbs_ephemeral_high
+        DCQ 0
+wbs_lowest_address
+        DCQ 0
+wbs_highest_address
+        DCQ 0
+
+    WRITE_BARRIER_END JIT_UpdateWriteBarrierState
+
+; ------------------------------------------------------------------
+; End of the writeable code region
+    LEAF_ENTRY JIT_PatchedCodeLast
+        ret      lr
+    LEAF_END
+
 ; void JIT_ByRefWriteBarrier
 ; On entry:
 ;   x13  : the source address (points to object reference to write)
@@ -307,15 +409,12 @@ ThePreStubPatchLabel
 ;   x15  : trashed
 ;
     WRITE_BARRIER_ENTRY JIT_CheckedWriteBarrier
-        adrp     x12,  g_lowest_address
-        ldr      x12,  [x12, g_lowest_address]
+        ldr      x12,  wbs_lowest_address
         cmp      x14,  x12
-        blt      NotInHeap
 
-        adrp      x12, g_highest_address 
-        ldr      x12, [x12, g_highest_address] 
-        cmp      x14, x12
-        blt      JIT_WriteBarrier
+        ldr      x12,  wbs_highest_address
+        ccmphs   x14,  x12, #0x2
+        blo      JIT_WriteBarrier
 
 NotInHeap
         str      x15, [x14], 8
@@ -336,25 +435,26 @@ NotInHeap
         stlr     x15, [x14]
 
 #ifdef WRITE_BARRIER_CHECK
-        ; Update GC Shadow Heap  
+        ; Update GC Shadow Heap 
+
+        ; Do not perform the work if g_GCShadow is 0
+        ldr      x12, wbs_GCShadow
+        cbz      x12, ShadowUpdateDisabled 
 
-        ; need temporary registers. Save them before using. 
-        stp      x12, x13, [sp, #-16]!
+        ; need temporary register. Save before using.
+        str      x13, [sp, #-16]!
 
         ; Compute address of shadow heap location:
         ;   pShadow = $g_GCShadow + (x14 - g_lowest_address)
-        adrp     x12, g_lowest_address
-        ldr      x12, [x12, g_lowest_address]
-        sub      x12, x14, x12
-        adrp     x13, $g_GCShadow
-        ldr      x13, [x13, $g_GCShadow]
+        ldr      x13, wbs_lowest_address
+        sub      x13, x14, x13
         add      x12, x13, x12
 
         ; if (pShadow >= $g_GCShadowEnd) goto end
         adrp     x13, $g_GCShadowEnd
         ldr      x13, [x13, $g_GCShadowEnd]
         cmp      x12, x13
-        bhs      shadowupdateend
+        bhs      ShadowUpdateEnd
 
         ; *pShadow = x15
         str      x15, [x12]
@@ -366,57 +466,58 @@ NotInHeap
         ; if ([x14] == x15) goto end
         ldr      x13, [x14]
         cmp      x13, x15
-        beq shadowupdateend
+        beq ShadowUpdateEnd
 
-        ; *pShadow = INVALIDGCVALUE (0xcccccccd)        
-        mov      x13, #0
-        movk     x13, #0xcccd
+        ; *pShadow = INVALIDGCVALUE (0xcccccccd)
+        movz     x13, #0xcccd
         movk     x13, #0xcccc, LSL #16
         str      x13, [x12]
 
-shadowupdateend
-        ldp      x12, x13, [sp],#16        
+ShadowUpdateEnd
+        ldr      x13, [sp], #16
+ShadowUpdateDisabled
+#endif
+
+#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+#error Need to implement for ARM64
 #endif
 
+CheckCardTable
         ; Branch to Exit if the reference is not in the Gen0 heap
         ;
-        adrp     x12,  g_ephemeral_low
-        ldr      x12,  [x12, g_ephemeral_low]
+        adr      x12,  wbs_ephemeral_low
+        ldp      x12,  x16, [x12]
+        cbz      x12,  SkipEphemeralCheck
+
         cmp      x15,  x12
         blo      Exit
 
-        adrp     x12, g_ephemeral_high 
-        ldr      x12, [x12, g_ephemeral_high]
-        cmp      x15,  x12
+        cmp      x15,  x16
         bhi      Exit
 
-        ; Check if we need to update the card table        
-        adrp     x12, g_card_table
-        ldr      x12, [x12, g_card_table]
-        add      x15,  x12, x14 lsr #11
-        ldrb     w12, [x15]
-        cmp      x12, 0xFF
+SkipEphemeralCheck
+        ; Check if we need to update the card table
+        ldr      x12, wbs_card_table
+
+        ; x15 := offset within card table
+        lsr      x15, x14, #11
+
+        ldrb     w16, [x12, x15]
+        cmp      w16, 0xFF
         beq      Exit
 
 UpdateCardTable
-        mov      x12, 0xFF 
-        strb     w12, [x15]
-Exit
-        add      x14, x14, 8
-        ret      lr          
-    WRITE_BARRIER_END JIT_WriteBarrier
+        mov      x16, 0xFF
+        strb     w16, [x12, x15]
 
-; ------------------------------------------------------------------
-; Start of the writeable code region
-    LEAF_ENTRY JIT_PatchedCodeStart
-        ret      lr
-    LEAF_END
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+#error Need to implement for ARM64
+#endif
 
-; ------------------------------------------------------------------
-; End of the writeable code region
-    LEAF_ENTRY JIT_PatchedCodeLast
+Exit
+        add      x14, x14, 8
         ret      lr
-    LEAF_END
+    WRITE_BARRIER_END JIT_WriteBarrier
 
 ;------------------------------------------------
 ; VirtualMethodFixupStub
diff --git a/src/vm/arm64/stubs.cpp b/src/vm/arm64/stubs.cpp
index 846ce3685b..83b2a21ec3 100644
--- a/src/vm/arm64/stubs.cpp
+++ b/src/vm/arm64/stubs.cpp
@@ -1111,9 +1111,6 @@ void InitJITHelpers1()
 
     JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap());
 }
-#ifndef FEATURE_PAL // TODO-ARM64-WINDOWS #13592
-EXTERN_C void JIT_UpdateWriteBarrierState(bool) {}
-#endif
 
 #else
 EXTERN_C void JIT_UpdateWriteBarrierState(bool) {}