This is to separate the diagnostics code out from gc.cpp (except

GC's own logging), gcee.cpp and objecthandle.cpp.

The rule is GC will call the diagnostics functions at various points to communicate
info and these diagnostics functions might call back into the GC if they require
initimate knowledge of the GC in order to get certain info. This way gc.cpp does
not need to know about details of the diagnostics components, eg, the profiling context;
and the diagnostics components do not need to know details about the GC. So got rid of
ProfilingScanContext in gcinterface.h and passed scanning functions to GC and handle table.

Got rid of where it goes through things per heap as this is not something that diagnostics
should care about, including going through stack roots per heap (which is only done in
GC for scalability purposes but profiling is doing this on a single thread). This also makes it faster.

Got rid of the checks for gc_low/high in ProfScanRootsHelper as this is knowledge
profiling shouldn't have. And it was also incorrectly not including all interior
pointer stack roots.

4 files changed, 388 insertions, 10 deletions

diff --git a/src/vm/eventtrace.cpp b/src/vm/eventtrace.cpp
index fbb292b777..cec79214a4 100644
--- a/src/vm/eventtrace.cpp
+++ b/src/vm/eventtrace.cpp
@@ -443,7 +443,7 @@ VOID ETW::GCLog::GCSettingsEvent()
             Info.GCSettings.LargeObjectSegmentSize = GCHeapUtilities::GetGCHeap()->GetValidSegmentSize (TRUE);
             FireEtwGCSettings_V1(Info.GCSettings.SegmentSize, Info.GCSettings.LargeObjectSegmentSize, Info.GCSettings.ServerGC, GetClrInstanceId());
         }  
-        GCHeapUtilities::GetGCHeap()->TraceGCSegments();
+        GCHeapUtilities::GetGCHeap()->DiagTraceGCSegments();
     }
 };
 
@@ -902,7 +902,7 @@ VOID ETW::GCLog::FireGcStartAndGenerationRanges(ETW_GC_INFO * pGcInfo)
 
         // Fire an event per range per generation
         IGCHeap *hp = GCHeapUtilities::GetGCHeap();
-        hp->DescrGenerationsToProfiler(FireSingleGenerationRangeEvent, NULL /* context */);
+        hp->DiagDescrGenerations(FireSingleGenerationRangeEvent, NULL /* context */);
     }
 }
 
@@ -929,7 +929,7 @@ VOID ETW::GCLog::FireGcEndAndGenerationRanges(ULONG Count, ULONG Depth)
     {
         // Fire an event per range per generation
         IGCHeap *hp = GCHeapUtilities::GetGCHeap();
-        hp->DescrGenerationsToProfiler(FireSingleGenerationRangeEvent, NULL /* context */);
+        hp->DiagDescrGenerations(FireSingleGenerationRangeEvent, NULL /* context */);
 
         // GCEnd
         FireEtwGCEnd_V1(Count, Depth, GetClrInstanceId());
@@ -938,7 +938,7 @@ VOID ETW::GCLog::FireGcEndAndGenerationRanges(ULONG Count, ULONG Depth)
  
 //---------------------------------------------------------------------------------------
 //
-// Callback made by GC when we call GCHeapUtilities::DescrGenerationsToProfiler().  This is
+// Callback made by GC when we call GCHeapUtilities::DiagDescrGenerations().  This is
 // called once per range per generation, and results in a single ETW event per range per
 // generation.
 //
diff --git a/src/vm/gcenv.ee.cpp b/src/vm/gcenv.ee.cpp
index 5afae88ca8..604ac297aa 100644
--- a/src/vm/gcenv.ee.cpp
+++ b/src/vm/gcenv.ee.cpp
@@ -845,3 +845,372 @@ Thread* GCToEEInterface::CreateBackgroundThread(GCBackgroundThreadFunction threa
     threadStubArgs.thread->DecExternalCount(FALSE);
     return NULL;
 }
+
+//
+// Diagnostics code
+//
+
+#if defined(GC_PROFILING) || defined(FEATURE_EVENT_TRACE)
+inline BOOL ShouldTrackMovementForProfilerOrEtw()
+{
+#ifdef GC_PROFILING
+    if (CORProfilerTrackGC())
+        return true;
+#endif
+
+#ifdef FEATURE_EVENT_TRACE
+    if (ETW::GCLog::ShouldTrackMovementForEtw())
+        return true;
+#endif
+
+    return false;
+}
+#endif // defined(GC_PROFILING) || defined(FEATURE_EVENT_TRACE)
+
+void ProfScanRootsHelper(Object** ppObject, ScanContext *pSC, uint32_t dwFlags)
+{
+#if defined(GC_PROFILING) || defined(FEATURE_EVENT_TRACE)
+    Object *pObj = *ppObject;
+    if (dwFlags & GC_CALL_INTERIOR)
+    {
+        pObj = GCHeapUtilities::GetGCHeap()->GetContainingObject(pObj);
+    }
+    ScanRootsHelper(pObj, ppObject, pSC, dwFlags);
+#endif // defined(GC_PROFILING) || defined(FEATURE_EVENT_TRACE)
+}
+
+// TODO - at some point we would like to completely decouple profiling
+// from ETW tracing using a pattern similar to this, where the
+// ProfilingScanContext has flags about whether or not certain things
+// should be tracked, and each one of these ProfilerShouldXYZ functions
+// will check these flags and determine what to do based upon that.
+// GCProfileWalkHeapWorker can, in turn, call those methods without fear
+// of things being ifdef'd out.
+
+// Returns TRUE if GC profiling is enabled and the profiler
+// should scan dependent handles, FALSE otherwise.
+BOOL ProfilerShouldTrackConditionalWeakTableElements() 
+{
+#if defined(GC_PROFILING)
+    return CORProfilerTrackConditionalWeakTableElements();
+#else
+    return FALSE;
+#endif // defined (GC_PROFILING)
+}
+
+// If GC profiling is enabled, informs the profiler that we are done
+// tracing dependent handles.
+void ProfilerEndConditionalWeakTableElementReferences(void* heapId)
+{
+#if defined (GC_PROFILING)
+    g_profControlBlock.pProfInterface->EndConditionalWeakTableElementReferences(heapId);
+#else
+    UNREFERENCED_PARAMETER(heapId);
+#endif // defined (GC_PROFILING)
+}
+
+// If GC profiling is enabled, informs the profiler that we are done
+// tracing root references.
+void ProfilerEndRootReferences2(void* heapId) 
+{
+#if defined (GC_PROFILING)
+    g_profControlBlock.pProfInterface->EndRootReferences2(heapId);
+#else
+    UNREFERENCED_PARAMETER(heapId);
+#endif // defined (GC_PROFILING)
+}
+
+void GcScanRootsForProfilerAndETW(promote_func* fn, int condemned, int max_gen, ScanContext* sc)
+{
+    Thread* pThread = NULL;
+    while ((pThread = ThreadStore::GetThreadList(pThread)) != NULL)
+    {
+        sc->thread_under_crawl = pThread;
+#ifdef FEATURE_EVENT_TRACE
+        sc->dwEtwRootKind = kEtwGCRootKindStack;
+#endif // FEATURE_EVENT_TRACE
+        ScanStackRoots(pThread, fn, sc);
+#ifdef FEATURE_EVENT_TRACE
+        sc->dwEtwRootKind = kEtwGCRootKindOther;
+#endif // FEATURE_EVENT_TRACE
+    }
+}
+
+void ScanHandleForProfilerAndETW(Object** pRef, Object* pSec, uint32_t flags, ScanContext* context, BOOL isDependent)
+{
+    ProfilingScanContext* pSC = (ProfilingScanContext*)context;
+
+#ifdef GC_PROFILING
+    // Give the profiler the objectref.
+    if (pSC->fProfilerPinned)
+    {
+        if (!isDependent)
+        {
+            BEGIN_PIN_PROFILER(CORProfilerTrackGC());
+            g_profControlBlock.pProfInterface->RootReference2(
+                (uint8_t *)*pRef,
+                kEtwGCRootKindHandle,
+                (EtwGCRootFlags)flags,
+                pRef, 
+                &pSC->pHeapId);
+            END_PIN_PROFILER();
+        }
+        else
+        {
+            BEGIN_PIN_PROFILER(CORProfilerTrackConditionalWeakTableElements());
+            g_profControlBlock.pProfInterface->ConditionalWeakTableElementReference(
+                (uint8_t*)*pRef,
+                (uint8_t*)pSec,
+                pRef,
+                &pSC->pHeapId);
+            END_PIN_PROFILER();
+        }
+    }
+#endif // GC_PROFILING
+
+#if defined(FEATURE_EVENT_TRACE)
+    // Notify ETW of the handle
+    if (ETW::GCLog::ShouldWalkHeapRootsForEtw())
+    {
+        ETW::GCLog::RootReference(
+            pRef,
+            *pRef,          // object being rooted
+            pSec,           // pSecondaryNodeForDependentHandle
+            isDependent,
+            pSC,
+            0,              // dwGCFlags,
+            flags);     // ETW handle flags
+    }
+#endif // defined(FEATURE_EVENT_TRACE) 
+}
+
+// This is called only if we've determined that either:
+//     a) The Profiling API wants to do a walk of the heap, and it has pinned the
+//     profiler in place (so it cannot be detached), and it's thus safe to call into the
+//     profiler, OR
+//     b) ETW infrastructure wants to do a walk of the heap either to log roots,
+//     objects, or both.
+// This can also be called to do a single walk for BOTH a) and b) simultaneously.  Since
+// ETW can ask for roots, but not objects
+#if defined(GC_PROFILING) || defined(FEATURE_EVENT_TRACE)
+void GCProfileWalkHeapWorker(BOOL fProfilerPinned, BOOL fShouldWalkHeapRootsForEtw, BOOL fShouldWalkHeapObjectsForEtw)
+{
+    {
+        ProfilingScanContext SC(fProfilerPinned);
+
+        // **** Scan roots:  Only scan roots if profiling API wants them or ETW wants them.
+        if (fProfilerPinned || fShouldWalkHeapRootsForEtw)
+        {
+            GcScanRootsForProfilerAndETW(&ProfScanRootsHelper, max_generation, max_generation, &SC);
+            SC.dwEtwRootKind = kEtwGCRootKindFinalizer;
+            GCHeapUtilities::GetGCHeap()->DiagScanFinalizeQueue(&ProfScanRootsHelper, &SC);
+
+            // Handles are kept independent of wks/svr/concurrent builds
+            SC.dwEtwRootKind = kEtwGCRootKindHandle;
+            GCHeapUtilities::GetGCHeap()->DiagScanHandles(&ScanHandleForProfilerAndETW, max_generation, &SC);
+
+            // indicate that regular handle scanning is over, so we can flush the buffered roots
+            // to the profiler.  (This is for profapi only.  ETW will flush after the
+            // entire heap was is complete, via ETW::GCLog::EndHeapDump.)
+            if (fProfilerPinned)
+            {
+                ProfilerEndRootReferences2(&SC.pHeapId);
+            }
+        }
+
+        // **** Scan dependent handles: only if the profiler supports it or ETW wants roots
+        if ((fProfilerPinned && ProfilerShouldTrackConditionalWeakTableElements()) ||
+            fShouldWalkHeapRootsForEtw)
+        {
+            // GcScanDependentHandlesForProfiler double-checks
+            // CORProfilerTrackConditionalWeakTableElements() before calling into the profiler
+
+            ProfilingScanContext* pSC = ≻
+
+            // we'll re-use pHeapId (which was either unused (0) or freed by EndRootReferences2
+            // (-1)), so reset it to NULL
+            _ASSERTE((*((size_t *)(&pSC->pHeapId)) == (size_t)(-1)) ||
+                    (*((size_t *)(&pSC->pHeapId)) == (size_t)(0)));
+            pSC->pHeapId = NULL;
+
+            GCHeapUtilities::GetGCHeap()->DiagScanDependentHandles(&ScanHandleForProfilerAndETW, max_generation, &SC);
+
+            // indicate that dependent handle scanning is over, so we can flush the buffered roots
+            // to the profiler.  (This is for profapi only.  ETW will flush after the
+            // entire heap was is complete, via ETW::GCLog::EndHeapDump.)
+            if (fProfilerPinned && ProfilerShouldTrackConditionalWeakTableElements())
+            {
+                ProfilerEndConditionalWeakTableElementReferences(&SC.pHeapId);
+            }
+        }
+
+        ProfilerWalkHeapContext profilerWalkHeapContext(fProfilerPinned, SC.pvEtwContext);
+
+        // **** Walk objects on heap: only if profiling API wants them or ETW wants them.
+        if (fProfilerPinned || fShouldWalkHeapObjectsForEtw)
+        {
+            GCHeapUtilities::GetGCHeap()->DiagWalkHeap(&HeapWalkHelper, &profilerWalkHeapContext, max_generation, TRUE /* walk the large object heap */);
+        }
+
+#ifdef FEATURE_EVENT_TRACE
+        // **** Done! Indicate to ETW helpers that the heap walk is done, so any buffers
+        // should be flushed into the ETW stream
+        if (fShouldWalkHeapObjectsForEtw || fShouldWalkHeapRootsForEtw)
+        {
+            ETW::GCLog::EndHeapDump(&profilerWalkHeapContext);
+        }
+#endif // FEATURE_EVENT_TRACE
+    }
+}
+#endif // defined(GC_PROFILING) || defined(FEATURE_EVENT_TRACE)
+
+void GCProfileWalkHeap()
+{
+    BOOL fWalkedHeapForProfiler = FALSE;
+
+#ifdef FEATURE_EVENT_TRACE
+    if (ETW::GCLog::ShouldWalkStaticsAndCOMForEtw())
+        ETW::GCLog::WalkStaticsAndCOMForETW();
+    
+    BOOL fShouldWalkHeapRootsForEtw = ETW::GCLog::ShouldWalkHeapRootsForEtw();
+    BOOL fShouldWalkHeapObjectsForEtw = ETW::GCLog::ShouldWalkHeapObjectsForEtw();
+#else // !FEATURE_EVENT_TRACE
+    BOOL fShouldWalkHeapRootsForEtw = FALSE;
+    BOOL fShouldWalkHeapObjectsForEtw = FALSE;
+#endif // FEATURE_EVENT_TRACE
+
+#if defined (GC_PROFILING)
+    {
+        BEGIN_PIN_PROFILER(CORProfilerTrackGC());
+        GCProfileWalkHeapWorker(TRUE /* fProfilerPinned */, fShouldWalkHeapRootsForEtw, fShouldWalkHeapObjectsForEtw);
+        fWalkedHeapForProfiler = TRUE;
+        END_PIN_PROFILER();
+    }
+#endif // defined (GC_PROFILING)
+
+#if defined (GC_PROFILING) || defined(FEATURE_EVENT_TRACE)
+    // we need to walk the heap if one of GC_PROFILING or FEATURE_EVENT_TRACE
+    // is defined, since both of them make use of the walk heap worker.
+    if (!fWalkedHeapForProfiler &&
+        (fShouldWalkHeapRootsForEtw || fShouldWalkHeapObjectsForEtw))
+    {
+        GCProfileWalkHeapWorker(FALSE /* fProfilerPinned */, fShouldWalkHeapRootsForEtw, fShouldWalkHeapObjectsForEtw);
+    }
+#endif // defined(GC_PROFILING) || defined(FEATURE_EVENT_TRACE)
+}
+
+void WalkFReachableObjects(BOOL isCritical, void* objectID)
+{
+	g_profControlBlock.pProfInterface->FinalizeableObjectQueued(isCritical, (ObjectID)objectID);
+}
+
+static fq_walk_fn g_FQWalkFn = &WalkFReachableObjects;
+
+void GCToEEInterface::DiagGCStart(int gen, bool isInduced)
+{
+#ifdef GC_PROFILING
+    DiagUpdateGenerationBounds();
+    GarbageCollectionStartedCallback(gen, isInduced);
+    {
+        BEGIN_PIN_PROFILER(CORProfilerTrackGC());
+        size_t context = 0;
+
+        // When we're walking objects allocated by class, then we don't want to walk the large
+        // object heap because then it would count things that may have been around for a while.
+        GCHeapUtilities::GetGCHeap()->DiagWalkHeap(&AllocByClassHelper, (void *)&context, 0, FALSE);
+
+        // Notify that we've reached the end of the Gen 0 scan
+        g_profControlBlock.pProfInterface->EndAllocByClass(&context);
+        END_PIN_PROFILER();
+    }
+
+#endif // GC_PROFILING
+}
+
+void GCToEEInterface::DiagUpdateGenerationBounds()
+{
+#ifdef GC_PROFILING
+    if (CORProfilerTrackGC())
+        UpdateGenerationBounds();
+#endif // GC_PROFILING
+}
+
+void GCToEEInterface::DiagGCEnd(size_t index, int gen, int reason, bool fConcurrent)
+{
+#ifdef GC_PROFILING
+    if (!fConcurrent)
+    {
+        GCProfileWalkHeap();
+        DiagUpdateGenerationBounds();
+        GarbageCollectionFinishedCallback();
+    }
+#endif // GC_PROFILING
+}
+
+void GCToEEInterface::DiagWalkFReachableObjects(void* gcContext)
+{
+#ifdef GC_PROFILING
+    if (CORProfilerTrackGC())
+    {
+        BEGIN_PIN_PROFILER(CORProfilerPresent());
+        GCHeapUtilities::GetGCHeap()->DiagWalkFinalizeQueue(gcContext, g_FQWalkFn);
+        END_PIN_PROFILER();
+    }
+#endif //GC_PROFILING
+}
+
+// Note on last parameter: when calling this for bgc, only ETW
+// should be sending these events so that existing profapi profilers
+// don't get confused.
+void WalkMovedReferences(uint8_t* begin, uint8_t* end, 
+                         ptrdiff_t reloc,
+                         size_t context, 
+                         BOOL fCompacting,
+                         BOOL fBGC)
+{
+    ETW::GCLog::MovedReference(begin, end,
+                               (fCompacting ? reloc : 0),
+                               context,
+                               fCompacting,
+                               !fBGC);
+}
+
+void GCToEEInterface::DiagWalkSurvivors(void* gcContext)
+{
+#if defined(GC_PROFILING) || defined(FEATURE_EVENT_TRACE)
+    if (ShouldTrackMovementForProfilerOrEtw())
+    {
+        size_t context = 0;
+        ETW::GCLog::BeginMovedReferences(&context);
+        GCHeapUtilities::GetGCHeap()->DiagWalkSurvivorsWithType(gcContext, &WalkMovedReferences, context, walk_for_gc);
+        ETW::GCLog::EndMovedReferences(context);
+    }
+#endif //GC_PROFILING || FEATURE_EVENT_TRACE
+}
+
+void GCToEEInterface::DiagWalkLOHSurvivors(void* gcContext)
+{
+#if defined(GC_PROFILING) || defined(FEATURE_EVENT_TRACE)
+    if (ShouldTrackMovementForProfilerOrEtw())
+    {
+        size_t context = 0;
+        ETW::GCLog::BeginMovedReferences(&context);
+        GCHeapUtilities::GetGCHeap()->DiagWalkSurvivorsWithType(gcContext, &WalkMovedReferences, context, walk_for_loh);
+        ETW::GCLog::EndMovedReferences(context);
+    }
+#endif //GC_PROFILING || FEATURE_EVENT_TRACE
+}
+
+void GCToEEInterface::DiagWalkBGCSurvivors(void* gcContext)
+{
+#if defined(GC_PROFILING) || defined(FEATURE_EVENT_TRACE)
+    if (ShouldTrackMovementForProfilerOrEtw())
+    {
+        size_t context = 0;
+        ETW::GCLog::BeginMovedReferences(&context);
+        GCHeapUtilities::GetGCHeap()->DiagWalkSurvivorsWithType(gcContext, &WalkMovedReferences, context, walk_for_bgc);
+        ETW::GCLog::EndMovedReferences(context);
+    }
+#endif //GC_PROFILING || FEATURE_EVENT_TRACE
+}
+
diff --git a/src/vm/gcenv.ee.h b/src/vm/gcenv.ee.h
index 26cdca3c93..7511a7c6a9 100644
--- a/src/vm/gcenv.ee.h
+++ b/src/vm/gcenv.ee.h
@@ -32,6 +32,15 @@ public:
     bool CatchAtSafePoint(Thread * pThread);
     void GcEnumAllocContexts(enum_alloc_context_func* fn, void* param);
     Thread* CreateBackgroundThread(GCBackgroundThreadFunction threadStart, void* arg);
+
+    // Diagnostics methods.
+    void DiagGCStart(int gen, bool isInduced);
+    void DiagUpdateGenerationBounds();
+    void DiagGCEnd(size_t index, int gen, int reason, bool fConcurrent);
+    void DiagWalkFReachableObjects(void* gcContext);
+    void DiagWalkSurvivors(void* gcContext);
+    void DiagWalkLOHSurvivors(void* gcContext);
+    void DiagWalkBGCSurvivors(void* gcContext);
 };
 
 #endif // FEATURE_STANDALONE_GC
diff --git a/src/vm/proftoeeinterfaceimpl.cpp b/src/vm/proftoeeinterfaceimpl.cpp
index dbfe2e0fba..1aee26dde3 100644
--- a/src/vm/proftoeeinterfaceimpl.cpp
+++ b/src/vm/proftoeeinterfaceimpl.cpp
@@ -758,7 +758,7 @@ struct GenerationTable
 
 //---------------------------------------------------------------------------------------
 //
-// This is a callback used by the GC when we call GCHeapUtilities::DescrGenerationsToProfiler
+// This is a callback used by the GC when we call GCHeapUtilities::DiagDescrGenerations
 // (from UpdateGenerationBounds() below).  The GC gives us generation information through
 // this callback, which we use to update the GenerationDesc in the corresponding
 // GenerationTable
@@ -879,7 +879,7 @@ void __stdcall UpdateGenerationBounds()
         // fill in the values by calling back into the gc, which will report
         // the ranges by calling GenWalkFunc for each one
         IGCHeap *hp = GCHeapUtilities::GetGCHeap();
-        hp->DescrGenerationsToProfiler(GenWalkFunc, newGenerationTable);
+        hp->DiagDescrGenerations(GenWalkFunc, newGenerationTable);
 
         // remember the old table and plug in the new one
         GenerationTable *oldGenerationTable = s_currentGenerationTable;
@@ -1022,7 +1022,7 @@ ClassID SafeGetClassIDFromObject(Object * pObj)
 
 //---------------------------------------------------------------------------------------
 //
-// Callback of type walk_fn used by GCHeapUtilities::WalkObject.  Keeps a count of each
+// Callback of type walk_fn used by GCHeapUtilities::DiagWalkObject.  Keeps a count of each
 // object reference found.
 //
 // Arguments:
@@ -1044,7 +1044,7 @@ BOOL CountContainedObjectRef(Object * pBO, void * context)
 
 //---------------------------------------------------------------------------------------
 //
-// Callback of type walk_fn used by GCHeapUtilities::WalkObject.  Stores each object reference
+// Callback of type walk_fn used by GCHeapUtilities::DiagWalkObject.  Stores each object reference
 // encountered into an array.
 //
 // Arguments:
@@ -1117,7 +1117,7 @@ BOOL HeapWalkHelper(Object * pBO, void * pvContext)
     if (pMT->ContainsPointersOrCollectible())
     {
         // First round through calculates the number of object refs for this class
-        GCHeapUtilities::GetGCHeap()->WalkObject(pBO, &CountContainedObjectRef, (void *)&cNumRefs);
+        GCHeapUtilities::GetGCHeap()->DiagWalkObject(pBO, &CountContainedObjectRef, (void *)&cNumRefs);
 
         if (cNumRefs > 0)
         {
@@ -1142,7 +1142,7 @@ BOOL HeapWalkHelper(Object * pBO, void * pvContext)
 
             // Second round saves off all of the ref values
             OBJECTREF * pCurObjRef = arrObjRef;
-            GCHeapUtilities::GetGCHeap()->WalkObject(pBO, &SaveContainedObjectRef, (void *)&pCurObjRef);
+            GCHeapUtilities::GetGCHeap()->DiagWalkObject(pBO, &SaveContainedObjectRef, (void *)&pCurObjRef);
         }
     }