Fix GC reporting for slow tail calls with by-ref-like args (#16264)

Fix GC reporting for slow tail calls with by-ref-like args

Fixes https://github.com/dotnet/coreclr/issues/16229:
- Offsets recorded in the GC layout are intended to be offsets from the top of the stack, fixed by calculating the proper offset

1 files changed, 27 insertions, 3 deletions

diff --git a/src/vm/i386/stublinkerx86.cpp b/src/vm/i386/stublinkerx86.cpp
index 7674464cfd..28a012e178 100644
--- a/src/vm/i386/stublinkerx86.cpp
+++ b/src/vm/i386/stublinkerx86.cpp
@@ -5930,11 +5930,35 @@ static void AppendGCLayout(ULONGARRAY &gcLayout, size_t baseOffset, BOOL fIsType
         {
             FindByRefPointerOffsetsInByRefLikeObject(
                 pMT,
-                baseOffset,
+                0 /* baseOffset */,
                 [&](size_t pointerOffset)
                 {
-                    _ASSERTE(gcLayout.Count() == 0 || pointerOffset > (gcLayout[gcLayout.Count() - 1] & ~(ULONG)1));
-                    *gcLayout.AppendThrowing() = (ULONG)(pointerOffset | 1); // "| 1" to mark it as an interior pointer
+                    // 'gcLayout' requires stack offsets relative to the top of the stack to be recorded, such that subtracting
+                    // the offset from the stack top yields the address of the field, given that subtracting 'baseOffset' from
+                    // the stack top yields the address of the first field in this struct. See TailCallFrame::GcScanRoots() for
+                    // how these offsets are used to calculate stack addresses for fields.
+                    _ASSERTE(pointerOffset < baseOffset);
+                    size_t stackOffsetFromTop = baseOffset - pointerOffset;
+                    _ASSERTE(FitsInU4(stackOffsetFromTop));
+
+                    // Offsets in 'gcLayout' are expected to be in increasing order
+                    int gcLayoutInsertIndex = gcLayout.Count();
+                    _ASSERTE(gcLayoutInsertIndex >= 0);
+                    for (; gcLayoutInsertIndex != 0; --gcLayoutInsertIndex)
+                    {
+                        ULONG prevStackOffsetFromTop = gcLayout[gcLayoutInsertIndex - 1] & ~(ULONG)1;
+                        if (stackOffsetFromTop > prevStackOffsetFromTop)
+                        {
+                            break;
+                        }
+                        if (stackOffsetFromTop == prevStackOffsetFromTop)
+                        {
+                            return;
+                        }
+                    }
+
+                    _ASSERTE(gcLayout.Count() == 0 || stackOffsetFromTop > (gcLayout[gcLayout.Count() - 1] & ~(ULONG)1));
+                    *gcLayout.InsertThrowing(gcLayoutInsertIndex) = (ULONG)(stackOffsetFromTop | 1); // "| 1" to mark it as an interior pointer
                 });
         }