Initial gcdump support (#94673)

* Initial gcdump support minus GCBulkRootStaticVar

* Update src/coreclr/nativeaot/Runtime/eventtrace_bulktype.cpp

Co-authored-by: Jan Kotas <[email protected]>

* Apply suggestions from code review

Co-authored-by: Elinor Fung <[email protected]>

* remove rgTypeParameters

* Add comments to the parameterized count

---------

Co-authored-by: Jan Kotas <[email protected]>
Co-authored-by: Elinor Fung <[email protected]>

Author: 3 people

src/coreclr/nativeaot/Runtime/eventpipe/gen-eventing-event-inc.lst

@@ -15,6 +15,7 @@ BGCOverflow_V1
 BGCPlanEnd
 BGCRevisit
 BGCSweepEnd
+BulkType
 Contention
 ContentionLockCreated
 ContentionStart_V2

src/coreclr/nativeaot/Runtime/eventtrace_bulktype.cpp

@@ -22,7 +22,6 @@
 
 BulkTypeValue::BulkTypeValue()
     : cTypeParameters(0)
-    , rgTypeParameters()
     , ullSingleTypeParameter(0)
 {
     LIMITED_METHOD_CONTRACT;
@@ -47,7 +46,6 @@ void BulkTypeValue::Clear()
     ZeroMemory(&fixedSizedData, sizeof(fixedSizedData));
     cTypeParameters = 0;
     ullSingleTypeParameter = 0;
-    rgTypeParameters.Release();
 }
 
 //---------------------------------------------------------------------------------------
@@ -59,7 +57,6 @@ void BulkTypeValue::Clear()
 // Fire an ETW event for all the types we batched so far, and then reset our state
 // so we can start batching new types at the beginning of the array.
 //
-
 void BulkTypeEventLogger::FireBulkTypeEvent()
 {
     LIMITED_METHOD_CONTRACT;
@@ -69,64 +66,51 @@ void BulkTypeEventLogger::FireBulkTypeEvent()
         // No types were batched up, so nothing to send
         return;
     }
-
-    // Normally, we'd use the MC-generated FireEtwBulkType for all this gunk, but
-    // it's insufficient as the bulk type event is too complex (arrays of structs of
-    // varying size). So we directly log the event via EventDataDescCreate and
-    // EventWrite
-
-    // We use one descriptor for the count + one for the ClrInstanceID + 4
-    // per batched type (to include fixed-size data + name + param count + param
-    // array).  But the system limit of 128 descriptors per event kicks in way
-    // before the 64K event size limit, and we already limit our batch size
-    // (m_nBulkTypeValueCount) to stay within the 128 descriptor limit.
-    EVENT_DATA_DESCRIPTOR EventData[128];
     UINT16 nClrInstanceID = GetClrInstanceId();
 
-    UINT iDesc = 0;
-
-    _ASSERTE(iDesc < _countof(EventData));
-    EventDataDescCreate(&EventData[iDesc++], &m_nBulkTypeValueCount, sizeof(m_nBulkTypeValueCount));
+    if(m_pBulkTypeEventBuffer == NULL)
+    {
+        // The buffer could not be allocated when this object was created, so bail.
+        return;
+    }
 
-    _ASSERTE(iDesc < _countof(EventData));
-    EventDataDescCreate(&EventData[iDesc++], &nClrInstanceID, sizeof(nClrInstanceID));
+    UINT iSize = 0;
 
     for (int iTypeData = 0; iTypeData < m_nBulkTypeValueCount; iTypeData++)
     {
+        BulkTypeValue& target = m_rgBulkTypeValues[iTypeData];
+
         // Do fixed-size data as one bulk copy
-        _ASSERTE(iDesc < _countof(EventData));
-        EventDataDescCreate(
-            &EventData[iDesc++],
-            &(m_rgBulkTypeValues[iTypeData].fixedSizedData),
-            sizeof(m_rgBulkTypeValues[iTypeData].fixedSizedData));
+        memcpy(
+                m_pBulkTypeEventBuffer + iSize,
+                &(target.fixedSizedData),
+                sizeof(target.fixedSizedData));
+        iSize += sizeof(target.fixedSizedData);
 
         // Do var-sized data individually per field
-
-        // Type name (nonexistent and thus empty on nativeaot)
-        _ASSERTE(iDesc < _countof(EventData));
-        EventDataDescCreate(&EventData[iDesc++], L"", sizeof(WCHAR));
+        // No name in event, so just the null terminator
+        m_pBulkTypeEventBuffer[iSize++] = 0;
+        m_pBulkTypeEventBuffer[iSize++] = 0;
 
         // Type parameter count
-        _ASSERTE(iDesc < _countof(EventData));
-        EventDataDescCreate(
-            &EventData[iDesc++],
-            &(m_rgBulkTypeValues[iTypeData].cTypeParameters),
-            sizeof(m_rgBulkTypeValues[iTypeData].cTypeParameters));
+        ULONG cTypeParams = target.cTypeParameters;
+        ULONG *ptrInt = (ULONG*)(m_pBulkTypeEventBuffer + iSize);
+        *ptrInt = cTypeParams;
+        iSize += sizeof(ULONG);
 
         // Type parameter array
-        if (m_rgBulkTypeValues[iTypeData].cTypeParameters > 0)
+        if (cTypeParams == 1)
+        {
+            memcpy(m_pBulkTypeEventBuffer + iSize, &target.ullSingleTypeParameter, sizeof(ULONGLONG) * cTypeParams);
+            iSize += sizeof(ULONGLONG) * cTypeParams;
+        }
+        else if (cTypeParams > 1)
         {
-            _ASSERTE(iDesc < _countof(EventData));
-            EventDataDescCreate(
-                &EventData[iDesc++],
-                ((m_rgBulkTypeValues[iTypeData].cTypeParameters == 1) ?
-                    &(m_rgBulkTypeValues[iTypeData].ullSingleTypeParameter) :
-                    (ULONGLONG*)(m_rgBulkTypeValues[iTypeData].rgTypeParameters)),
-                sizeof(ULONGLONG) * m_rgBulkTypeValues[iTypeData].cTypeParameters);
+            ASSERT_UNCONDITIONALLY("unexpected value of cTypeParams greater than 1");
         }
     }
 
-    EventWrite(Microsoft_Windows_DotNETRuntimeHandle, &BulkType, iDesc, EventData);
+    FireEtwBulkType(m_nBulkTypeValueCount, GetClrInstanceId(), iSize, m_pBulkTypeEventBuffer);
 
     // Reset state
     m_nBulkTypeValueCount = 0;
@@ -251,7 +235,6 @@ SHash<LoggedTypesTraits>* s_loggedTypesHash = NULL;
 //      Index into internal array where the info got batched.  Or -1 if there was a
 //      failure.
 //
-
 int BulkTypeEventLogger::LogSingleType(MethodTable * pEEType)
 {
 #ifdef MULTIPLE_HEAPS
@@ -295,8 +278,13 @@ int BulkTypeEventLogger::LogSingleType(MethodTable * pEEType)
     // Determine this MethodTable's module.
     RuntimeInstance * pRuntimeInstance = GetRuntimeInstance();
 
-    ULONGLONG osModuleHandle = (ULONGLONG) pEEType->GetTypeManagerPtr()->AsTypeManager()->GetOsModuleHandle();
-
+    // EEType for GC statics are not fully populated and they do not have a valid TypeManager. We will identify them by checking for `ElementType_Unknown`.
+    // We will not be able to get the osModuleHandle for these
+    ULONGLONG osModuleHandle = 0;
+    if (pEEType->GetElementType() != ElementType_Unknown)
+    {
+        osModuleHandle = (ULONGLONG) pEEType->GetTypeManagerPtr()->AsTypeManager()->GetOsModuleHandle();
+    }
     pVal->fixedSizedData.ModuleID = osModuleHandle;
 
     if (pEEType->IsParameterizedType())
@@ -371,15 +359,6 @@ int BulkTypeEventLogger::LogSingleType(MethodTable * pEEType)
 
 void BulkTypeEventLogger::LogTypeAndParameters(uint64_t thAsAddr)
 {
-    // BulkTypeEventLogger currently fires ETW events only
-    if (!ETW_TRACING_CATEGORY_ENABLED(
-        MICROSOFT_WINDOWS_DOTNETRUNTIME_PROVIDER_Context,
-        TRACE_LEVEL_INFORMATION,
-        CLR_TYPE_KEYWORD))
-    {
-        return;
-    }
-
     MethodTable * pEEType = (MethodTable *) thAsAddr;
 
     // Batch up this type.  This grabs useful info about the type, including any
@@ -398,25 +377,15 @@ void BulkTypeEventLogger::LogTypeAndParameters(uint64_t thAsAddr)
     // We're about to recursively call ourselves for the type parameters, so make a
     // local copy of their type handles first (else, as we log them we could flush
     // and clear out m_rgBulkTypeValues, thus trashing pVal)
-    NewArrayHolder<ULONGLONG> rgTypeParameters;
     DWORD cTypeParams = pVal->cTypeParameters;
     if (cTypeParams == 1)
     {
         LogTypeAndParameters(pVal->ullSingleTypeParameter);
     }
     else if (cTypeParams > 1)
     {
-        rgTypeParameters = new (nothrow) ULONGLONG[cTypeParams];
-        for (DWORD i=0; i < cTypeParams; i++)
-        {
-            rgTypeParameters[i] = pVal->rgTypeParameters[i];
-        }
-
-        // Recursively log any referenced parameter types
-        for (DWORD i=0; i < cTypeParams; i++)
-        {
-            LogTypeAndParameters(rgTypeParameters[i]);
-        }
+        
+        ASSERT_UNCONDITIONALLY("unexpected value of cTypeParams greater than 1");
     }
 }
 

src/coreclr/nativeaot/Runtime/eventtrace_gcheap.cpp

@@ -35,35 +35,29 @@
 
 BOOL ETW::GCLog::ShouldWalkHeapObjectsForEtw()
 {
-    // @TODO: until the below issue is fixed correctly
-    // https://github.com/dotnet/runtime/issues/88491
-    return FALSE;
+    return RUNTIME_PROVIDER_CATEGORY_ENABLED(
+            TRACE_LEVEL_INFORMATION,
+            CLR_GCHEAPDUMP_KEYWORD);    
 }
 
 BOOL ETW::GCLog::ShouldWalkHeapRootsForEtw()
 {
-    // @TODO: until the below issue is fixed correctly
-    // https://github.com/dotnet/runtime/issues/88491
-    return FALSE;
+    return RUNTIME_PROVIDER_CATEGORY_ENABLED(
+            TRACE_LEVEL_INFORMATION,
+            CLR_GCHEAPDUMP_KEYWORD);    
 }
 
 BOOL ETW::GCLog::ShouldTrackMovementForEtw()
 {
-    LIMITED_METHOD_CONTRACT;
     return RUNTIME_PROVIDER_CATEGORY_ENABLED(
         TRACE_LEVEL_INFORMATION,
         CLR_GCHEAPSURVIVALANDMOVEMENT_KEYWORD);
 }
 
 BOOL ETW::GCLog::ShouldWalkStaticsAndCOMForEtw()
 {
-    // @TODO:
-    return FALSE;
-}
-
-void ETW::GCLog::WalkStaticsAndCOMForETW()
-{
-    // @TODO:
+    // @TODO
+    return false;
 }
 
 // Batches the list of moved/surviving references for the GCBulkMovedObjectRanges /
@@ -482,6 +476,14 @@ HRESULT ETW::GCLog::ForceGCForDiagnostics()
     return hr;
 }
 
+//---------------------------------------------------------------------------------------
+// WalkStaticsAndCOMForETW walks both CCW/RCW objects and static variables.
+//---------------------------------------------------------------------------------------
+
+void ETW::GCLog::WalkStaticsAndCOMForETW()
+{
+}
+
 // Holds state that batches of roots, nodes, edges, and types as the GC walks the heap
 // at the end of a collection.
 class EtwGcHeapDumpContext

src/coreclr/nativeaot/Runtime/eventtracepriv.h

@@ -114,18 +114,9 @@ class BulkTypeValue
     // Below are the remainder of each struct in the bulk type event (i.e., the
     // variable-sized data). The var-sized fields are copied into the event individually
     // (not directly), so they don't need to have the same layout as in the ETW manifest
-
-    // This is really a denorm of the size already stored in rgTypeParameters, but we
-    // need a persistent place to stash this away so EventDataDescCreate & EventWrite
-    // have a reliable place to copy it from.  This is filled in at the last minute,
-    // when sending the event.
     ULONG cTypeParameters;
 
-    // If > 1 type parameter, this is an array of their MethodTable*'s
-    NewArrayHolder<ULONGLONG> rgTypeParameters;
-
-    // If exactly one type parameter, this is its MethodTable*.  (If != 1 type parameter,
-    // this is 0.)
+    // We expect only one type parameter. See the explanation at BulkTypeEventLogger::LogSingleType on generic parameters for additional details.
     ULONGLONG ullSingleTypeParameter;
 };
 
@@ -138,6 +129,9 @@ class BulkTypeEventLogger
 {
 private:
 
+    // The maximum event size, and the size of the buffer that we allocate to hold the event contents.
+    static const size_t kSizeOfEventBuffer = 65536;
+
     // Estimate of how many bytes we can squeeze in the event data for the value struct
     // array.  (Intentionally overestimate the size of the non-array parts to keep it safe.)
     static const int kMaxBytesTypeValues = (cbMaxEtwEvent - 0x30);
@@ -178,14 +172,25 @@ class BulkTypeEventLogger
     // List of types we've batched.
     BulkTypeValue m_rgBulkTypeValues[kMaxCountTypeValues];
 
+    BYTE *m_pBulkTypeEventBuffer;
+
     int LogSingleType(MethodTable * pEEType);
 
 public:
     BulkTypeEventLogger() :
         m_nBulkTypeValueCount(0),
         m_nBulkTypeValueByteCount(0)
+        , m_pBulkTypeEventBuffer(NULL)
     {
         LIMITED_METHOD_CONTRACT;
+
+        m_pBulkTypeEventBuffer = new (nothrow) BYTE[kSizeOfEventBuffer];
+    }
+
+    ~BulkTypeEventLogger()
+    {
+        delete[] m_pBulkTypeEventBuffer;
+        m_pBulkTypeEventBuffer = NULL;
     }
 
     void LogTypeAndParameters(ULONGLONG thAsAddr);

src/coreclr/vm/eventtrace_bulktype.cpp

@@ -20,7 +20,7 @@
 #include "eventtracepriv.h"
 
 //---------------------------------------------------------------------------------------
-// BulkStaticsLogger: Batches up and logs static variable roots
+// BulkComLogger: Batches up and logs RCW and CCW
 //---------------------------------------------------------------------------------------
 
 BulkComLogger::BulkComLogger(BulkTypeEventLogger *typeLogger)

src/tests/tracing/eventpipe/gcdump/gcdump.cs

@@ -5,6 +5,7 @@
 using System.Diagnostics.Tracing;
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using System.Collections.Generic;
@@ -43,7 +44,8 @@ public static int TestEntryPoint()
                 new EventPipeProvider("Microsoft-Windows-DotNETRuntime", eventLevel: EventLevel.Verbose, keywords: (long)ClrTraceEventParser.Keywords.GCHeapSnapshot)
             };
 
-            return IpcTraceTest.RunAndValidateEventCounts(_expectedEventCounts, _eventGeneratingAction, providers, 1024, _DoesRundownContainMethodEvents);
+            bool enableRundown = TestLibrary.Utilities.IsNativeAot? false: true;
+            return IpcTraceTest.RunAndValidateEventCounts(_expectedEventCounts, _eventGeneratingAction, providers, 1024, _DoesRundownContainMethodEvents, enableRundownProvider: enableRundown);
         }
 
         private static Dictionary<string, ExpectedEventCount> _expectedEventCounts = new Dictionary<string, ExpectedEventCount>()
@@ -101,18 +103,18 @@ public static int TestEntryPoint()
                 // and high enough to catch issues. There should be between hundreds and thousands
                 // for each, but the number is variable and the point of the test is to verify
                 // that we get any events at all.
+                
                 if (_seenGCStart
                      && _seenGCStop
                      && _bulkTypeCount > 50
                      && _bulkNodeCount > 50
-                     && _bulkEdgeCount > 50
-                     && _bulkRootEdgeCount > 50
-                     && _bulkRootStaticVarCount > 50)
+                     && _bulkEdgeCount > 50)
                 {
-                    return 100;
+                    // Native AOT hasn't yet implemented statics. Hence _bulkRootStaticVarCount is zero and _bulkRootEdgeCount can be low
+                    if ((TestLibrary.Utilities.IsNativeAot && _bulkRootEdgeCount > 20) || (_bulkRootStaticVarCount > 50 && _bulkRootEdgeCount > 50))
+                        return 100;
                 }
 
-
                 Console.WriteLine($"Test failed due to missing GC heap events.");
                 Console.WriteLine($"_seenGCStart =            {_seenGCStart}");
                 Console.WriteLine($"_seenGCStop =             {_seenGCStop}");

src/tests/tracing/eventpipe/gcdump/gcdump.csproj

@@ -13,5 +13,6 @@
     <Compile Include="$(MSBuildProjectName).cs" />
     <ProjectReference Include="../common/eventpipe_common.csproj" />
     <ProjectReference Include="../common/Microsoft.Diagnostics.NETCore.Client/Microsoft.Diagnostics.NETCore.Client.csproj" />
+     <ProjectReference Include="$(TestLibraryProjectPath)" />
   </ItemGroup>
 </Project>