StompWriteBarrier initialization path refactoring (#14105)

* refactored arm, arm64, amd64 and x86 to signal about icache flush and ee restarts

* refactored gc init stage to stomp write barrier (hence flush icache) only once

* review fixes, care taken of icache invalidation during StompResize

* fixed heap boundaries initialization bug introduced after refactoring gc.cpp

* stylistic review fixe

* global variable rename

* global variable rename once more

1 files changed, 42 insertions, 17 deletions

diff --git a/src/vm/gcenv.ee.cpp b/src/vm/gcenv.ee.cpp
index b135173c0f..29c75e2577 100644
--- a/src/vm/gcenv.ee.cpp
+++ b/src/vm/gcenv.ee.cpp
@@ -843,6 +843,8 @@ void GCToEEInterface::DiagWalkBGCSurvivors(void* gcContext)
 
 void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
 {
+    int stompWBCompleteActions = SWB_PASS;
+
     assert(args != nullptr);
     switch (args->operation)
     {
@@ -868,7 +870,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
         }
 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
 
-        ::StompWriteBarrierResize(args->is_runtime_suspended, args->requires_upper_bounds_check);
+        stompWBCompleteActions |= ::StompWriteBarrierResize(args->is_runtime_suspended, args->requires_upper_bounds_check);
 
         // We need to make sure that other threads executing checked write barriers
         // will see the g_card_table update before g_lowest/highest_address updates.
@@ -884,30 +886,45 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
         // "cross-modifying code": We need all _executing_ threads to invalidate
         // their instruction cache, which FlushProcessWriteBuffers achieves by sending
         // an IPI (inter-process interrupt).
-        FlushProcessWriteBuffers();
+
+        if (stompWBCompleteActions & SWB_ICACHE_FLUSH)
+        {
+            // flushing icache on current processor (thread)
+            ::FlushWriteBarrierInstructionCache();
+            // asking other processors (threads) to invalidate their icache
+            FlushProcessWriteBuffers();
+        }
 
         g_lowest_address = args->lowest_address;
         VolatileStore(&g_highest_address, args->highest_address);
 
 #if defined(_ARM64_)
         // Need to reupdate for changes to g_highest_address g_lowest_address
-        ::StompWriteBarrierResize(args->is_runtime_suspended, args->requires_upper_bounds_check);
+        bool is_runtime_suspended = (stompWBCompleteActions & SWB_EE_RESTART) || args->is_runtime_suspended;
+        stompWBCompleteActions |= ::StompWriteBarrierResize(is_runtime_suspended, args->requires_upper_bounds_check);
 
-        if(!args->is_runtime_suspended)
+        is_runtime_suspended = (stompWBCompleteActions & SWB_EE_RESTART) || args->is_runtime_suspended;
+        if(!is_runtime_suspended)
         {
             // If runtime is not suspended, force updated state to be visible to all threads
             MemoryBarrier();
         }
 #endif
-        return;
+        if (stompWBCompleteActions & SWB_EE_RESTART)
+        {
+            assert(!args->is_runtime_suspended &&
+                "if runtime was suspended in patching routines then it was in running state at begining");
+            ThreadSuspend::RestartEE(FALSE, TRUE);
+        }
+        return; // unlike other branches we have already done cleanup so bailing out here
     case WriteBarrierOp::StompEphemeral:
         // StompEphemeral requires a new ephemeral low and a new ephemeral high
         assert(args->ephemeral_low != nullptr);
         assert(args->ephemeral_high != nullptr);
         g_ephemeral_low = args->ephemeral_low;
         g_ephemeral_high = args->ephemeral_high;
-        ::StompWriteBarrierEphemeral(args->is_runtime_suspended);
-        return;
+        stompWBCompleteActions |= ::StompWriteBarrierEphemeral(args->is_runtime_suspended);
+        break;
     case WriteBarrierOp::Initialize:
         // This operation should only be invoked once, upon initialization.
         assert(g_card_table == nullptr);
@@ -927,12 +944,10 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
         assert(g_card_bundle_table == nullptr);
         g_card_bundle_table = args->card_bundle_table;
 #endif
-
-        FlushProcessWriteBuffers();
         
         g_lowest_address = args->lowest_address;
-        VolatileStore(&g_highest_address, args->highest_address);
-        ::StompWriteBarrierResize(true, false);
+        g_highest_address = args->highest_address;
+        stompWBCompleteActions |= ::StompWriteBarrierResize(true, false);
 
         // StompWriteBarrierResize does not necessarily bash g_ephemeral_low
         // usages, so we must do so here. This is particularly true on x86,
@@ -940,32 +955,42 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
         // called with the parameters (true, false), as it is above.
         g_ephemeral_low = args->ephemeral_low;
         g_ephemeral_high = args->ephemeral_high;
-        ::StompWriteBarrierEphemeral(true);
-        return;
+        stompWBCompleteActions |= ::StompWriteBarrierEphemeral(true);
+        break;
     case WriteBarrierOp::SwitchToWriteWatch:
 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
         assert(args->write_watch_table != nullptr);
         assert(args->is_runtime_suspended && "the runtime must be suspended here!");
         g_sw_ww_table = args->write_watch_table;
         g_sw_ww_enabled_for_gc_heap = true;
-        ::SwitchToWriteWatchBarrier(true);
+        stompWBCompleteActions |= ::SwitchToWriteWatchBarrier(true);
 #else
         assert(!"should never be called without FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP");
 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
-        return;
+        break;
     case WriteBarrierOp::SwitchToNonWriteWatch:
 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
         assert(args->is_runtime_suspended && "the runtime must be suspended here!");
         g_sw_ww_table = 0;
         g_sw_ww_enabled_for_gc_heap = false;
-        ::SwitchToNonWriteWatchBarrier(true);
+        stompWBCompleteActions |= ::SwitchToNonWriteWatchBarrier(true);
 #else
         assert(!"should never be called without FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP");
 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
-        return;
+        break;
     default:
         assert(!"unknown WriteBarrierOp enum");
     }
+    if (stompWBCompleteActions & SWB_ICACHE_FLUSH) 
+    {
+        ::FlushWriteBarrierInstructionCache();
+    }
+    if (stompWBCompleteActions & SWB_EE_RESTART) 
+    {
+        assert(!args->is_runtime_suspended && 
+            "if runtime was suspended in patching routines then it was in running state at begining");
+        ThreadSuspend::RestartEE(FALSE, TRUE);
+    }
 }
 
 void GCToEEInterface::EnableFinalization(bool foundFinalizers)