Add support for dumping and using precise debug info (#61735)

* In the JIT, add support for dumping the precise debug info out through
  an environment variable `DOTNET_JitDumpPreciseDebugInfoFile` in a
  simple JSON format. This is a stopgap until we expose the extra
  information through ETW events.

* In dotnet-pgo, add an argument --precise-debug-info-file which can
  point to the file produced by the JIT. When used, dotnet-pgo will get
  native<->IL mappings from this file instead of through ETW events.

* In dotnet-pgo, add support for attributing samples to inlinees when
  that information is present. This changes the attribution process a
  bit: previously, we would group all LBR data/samples and then
  construct the profile from all the data. We now do it in a more
  streaming way where there is a SampleCorrelator that can handle
  individual LBR records and individual samples.

* In dotnet-pgo, add an argument --dump-worst-overlap-graphs-to which
  can be used in the compare-mibc command to dump out a .dot file
  containing the flow graph of the methods with the worst overlap
  measures, and showing the relative weight count on each basic block
  and edge for the two profiles being compared. This is particular
  useful to find out where we are producing incorrect debug mappings, by
  comparing spgo.mibc and instrumented.mibc files.

Author: jakobbotsch

src/coreclr/jit/codegen.h

@@ -556,11 +556,16 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
     void genIPmappingListDisp();
 #endif // DEBUG
 
-    IPmappingDsc* genCreateIPMapping(IPmappingDscKind kind, const DebugInfo& di, bool isLabel);
     void genIPmappingAdd(IPmappingDscKind kind, const DebugInfo& di, bool isLabel);
     void genIPmappingAddToFront(IPmappingDscKind kind, const DebugInfo& di, bool isLabel);
     void genIPmappingGen();
 
+#ifdef DEBUG
+    void genDumpPreciseDebugInfo();
+    void genDumpPreciseDebugInfoInlineTree(FILE* file, InlineContext* context, bool* first);
+    void genAddPreciseIPMappingHere(const DebugInfo& di);
+#endif
+
     void genEnsureCodeEmitted(const DebugInfo& di);
 
     //-------------------------------------------------------------------------

src/coreclr/jit/codegencommon.cpp

@@ -127,6 +127,9 @@ CodeGen::CodeGen(Compiler* theCompiler) : CodeGenInterface(theCompiler)
     compiler->genIPmappingLast         = nullptr;
     compiler->genCallSite2DebugInfoMap = nullptr;
 
+    compiler->genPreciseIPMappingsHead = nullptr;
+    compiler->genPreciseIPMappingsTail = nullptr;
+
     /* Assume that we not fully interruptible */
 
     SetInterruptible(false);
@@ -2446,6 +2449,8 @@ void CodeGen::genEmitUnwindDebugGCandEH()
 
     genIPmappingGen();
 
+    INDEBUG(genDumpPreciseDebugInfo());
+
     /* Finalize the Local Var info in terms of generated code */
 
     genSetScopeInfo();
@@ -10801,6 +10806,98 @@ void CodeGen::genIPmappingGen()
     compiler->eeSetLIdone();
 }
 
+#ifdef DEBUG
+void CodeGen::genDumpPreciseDebugInfoInlineTree(FILE* file, InlineContext* context, bool* first)
+{
+    if (context->GetSibling() != nullptr)
+    {
+        genDumpPreciseDebugInfoInlineTree(file, context->GetSibling(), first);
+    }
+
+    if (context->IsSuccess())
+    {
+        if (!*first)
+        {
+            fprintf(file, ",");
+        }
+
+        *first = false;
+
+        fprintf(file, "{\"Ordinal\":%u,", context->GetOrdinal());
+        fprintf(file, "\"MethodID\":%lld,", (INT64)context->GetCallee());
+        const char* className;
+        const char* methodName = compiler->eeGetMethodName(context->GetCallee(), &className);
+        fprintf(file, "\"MethodName\":\"%s\",", methodName);
+        fprintf(file, "\"Inlinees\":[");
+        if (context->GetChild() != nullptr)
+        {
+            bool childFirst = true;
+            genDumpPreciseDebugInfoInlineTree(file, context->GetChild(), &childFirst);
+        }
+        fprintf(file, "]}");
+    }
+}
+
+void CodeGen::genDumpPreciseDebugInfo()
+{
+    if (JitConfig.JitDumpPreciseDebugInfoFile() == nullptr)
+        return;
+
+    static CritSecObject s_critSect;
+    CritSecHolder        holder(s_critSect);
+
+    FILE* file = _wfopen(JitConfig.JitDumpPreciseDebugInfoFile(), W("a"));
+    if (file == nullptr)
+        return;
+
+    // MethodID in ETW events are the method handles.
+    fprintf(file, "{\"MethodID\":%lld,", (INT64)compiler->info.compMethodHnd);
+    // Print inline tree.
+    fprintf(file, "\"InlineTree\":");
+
+    bool first = true;
+    genDumpPreciseDebugInfoInlineTree(file, compiler->compInlineContext, &first);
+    fprintf(file, ",\"Mappings\":[");
+    first = true;
+    for (PreciseIPMapping* mapping = compiler->genPreciseIPMappingsHead; mapping != nullptr; mapping = mapping->next)
+    {
+        if (!first)
+        {
+            fprintf(file, ",");
+        }
+
+        first = false;
+
+        fprintf(file, "{\"NativeOffset\":%u,\"InlineContext\":%u,\"ILOffset\":%u}",
+                mapping->nativeLoc.CodeOffset(GetEmitter()), mapping->debugInfo.GetInlineContext()->GetOrdinal(),
+                mapping->debugInfo.GetLocation().GetOffset());
+    }
+
+    fprintf(file, "]}\n");
+
+    fclose(file);
+}
+
+void CodeGen::genAddPreciseIPMappingHere(const DebugInfo& di)
+{
+    PreciseIPMapping* mapping = new (compiler, CMK_DebugInfo) PreciseIPMapping;
+    mapping->next             = nullptr;
+    mapping->nativeLoc.CaptureLocation(GetEmitter());
+    mapping->debugInfo = di;
+
+    if (compiler->genPreciseIPMappingsTail != nullptr)
+    {
+        compiler->genPreciseIPMappingsTail->next = mapping;
+    }
+    else
+    {
+        compiler->genPreciseIPMappingsHead = mapping;
+    }
+
+    compiler->genPreciseIPMappingsTail = mapping;
+}
+#endif
+
 /*============================================================================
  *
  *   These are empty stubs to help the late dis-assembler to compile

src/coreclr/jit/codegenlinear.cpp

@@ -441,7 +441,13 @@ void CodeGen::genCodeForBBlist()
                     genIPmappingAdd(IPmappingDscKind::Normal, currentDI, firstMapping);
                     firstMapping = false;
                 }
+
 #ifdef DEBUG
+                if ((JitConfig.JitDumpPreciseDebugInfoFile() != nullptr) && ilOffset->gtStmtDI.IsValid())
+                {
+                    genAddPreciseIPMappingHere(ilOffset->gtStmtDI);
+                }
+
                 assert(ilOffset->gtStmtLastILoffs <= compiler->info.compILCodeSize ||
                        ilOffset->gtStmtLastILoffs == BAD_IL_OFFSET);
 

src/coreclr/jit/compiler.h

@@ -2534,6 +2534,13 @@ struct IPmappingDsc
     bool             ipmdIsLabel;   // Can this code be a branch label?
 };
 
+struct PreciseIPMapping
+{
+    PreciseIPMapping* next;
+    emitLocation      nativeLoc;
+    DebugInfo         debugInfo;
+};
+
 /*
 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
@@ -8269,6 +8276,9 @@ class Compiler
     IPmappingDsc* genIPmappingList;
     IPmappingDsc* genIPmappingLast;
 
+    PreciseIPMapping* genPreciseIPMappingsHead;
+    PreciseIPMapping* genPreciseIPMappingsTail;
+
     // Managed RetVal - A side hash table meant to record the mapping from a
     // GT_CALL node to its debug info.  This info is used to emit sequence points
     // that can be used by debugger to determine the native offset at which the

src/coreclr/jit/inline.h

@@ -726,6 +726,18 @@ class InlineContext
         return m_Parent;
     }
 
+    // Get the sibling context.
+    InlineContext* GetSibling() const
+    {
+        return m_Sibling;
+    }
+
+    // Get the first child context.
+    InlineContext* GetChild() const
+    {
+        return m_Child;
+    }
+
     // Get the code pointer for this context.
     const BYTE* GetCode() const
     {
@@ -806,7 +818,6 @@ class InlineContext
 private:
     InlineContext(InlineStrategy* strategy);
 
-private:
     InlineStrategy*   m_InlineStrategy;    // overall strategy
     InlineContext*    m_Parent;            // logical caller (parent)
     InlineContext*    m_Child;             // first child

src/coreclr/jit/jitconfigvalues.h

@@ -232,6 +232,8 @@ CONFIG_INTEGER(JitDumpFgConstrained, W("JitDumpFgConstrained"), 1) // 0 == don't
 CONFIG_INTEGER(JitDumpFgBlockID, W("JitDumpFgBlockID"), 0) // 0 == display block with bbNum; 1 == display with both
                                                            // bbNum and bbID
 
+CONFIG_STRING(JitDumpPreciseDebugInfoFile, W("JitDumpPreciseDebugInfoFile"))
+
 CONFIG_STRING(JitLateDisasmTo, W("JITLateDisasmTo"))
 CONFIG_STRING(JitRange, W("JitRange"))
 CONFIG_STRING(JitStressModeNames, W("JitStressModeNames")) // Internal Jit stress mode: stress using the given set of

src/coreclr/tools/Common/TypeSystem/IL/FlowGraph.cs

@@ -88,47 +88,6 @@ public IEnumerable<BasicBlock> LookupRange(int ilOffsetStart, int ilOffsetEnd)
                 yield return BasicBlocks[i];
         }
 
-        internal string Dump(Func<BasicBlock, string> getNodeAnnot, Func<(BasicBlock, BasicBlock), string> getEdgeAnnot)
-        {
-            var sb = new StringBuilder();
-            sb.AppendLine("digraph G {");
-            sb.AppendLine("  forcelabels=true;");
-            sb.AppendLine();
-            Dictionary<long, int> bbToIndex = new Dictionary<long, int>();
-            for (int i = 0; i < BasicBlocks.Count; i++)
-                bbToIndex.Add(BasicBlocks[i].Start, i);
-
-            foreach (BasicBlock bb in BasicBlocks)
-            {
-                string label = $"[{bb.Start:x}..{bb.Start + bb.Size:x})\\n{getNodeAnnot(bb)}";
-                sb.AppendLine($"  BB{bbToIndex[bb.Start]} [label=\"{label}\"];");
-            }
-
-            sb.AppendLine();
-
-            foreach (BasicBlock bb in BasicBlocks)
-            {
-                foreach (BasicBlock tar in bb.Targets)
-                {
-                    string label = getEdgeAnnot((bb, tar));
-                    string postfix = string.IsNullOrEmpty(label) ? "" : $" [label=\"{label}\"]";
-                    sb.AppendLine($"  BB{bbToIndex[bb.Start]} -> BB{bbToIndex[tar.Start]}{postfix};");
-                }
-            }
-
-            // Write ranks with BFS.
-            List<BasicBlock> curRank = new List<BasicBlock> { BasicBlocks.Single(bb => bb.Start == 0) };
-            HashSet<BasicBlock> seen = new HashSet<BasicBlock>(curRank);
-            while (curRank.Count > 0)
-            {
-                sb.AppendLine($"  {{rank = same; {string.Concat(curRank.Select(bb => $"BB{bbToIndex[bb.Start]}; "))}}}");
-                curRank = curRank.SelectMany(bb => bb.Targets).Where(seen.Add).ToList();
-            }
-
-            sb.AppendLine("}");
-            return sb.ToString();
-        }
-
         public static FlowGraph Create(MethodIL il)
         {
             HashSet<int> bbStarts = GetBasicBlockStarts(il);

src/coreclr/tools/dotnet-pgo/CommandLineOptions.cs

@@ -18,6 +18,7 @@ internal class CommandLineOptions
 
         public FileInfo TraceFile;
         public FileInfo OutputFileName;
+        public FileInfo PreciseDebugInfoFile;
         public int? Pid;
         public string ProcessName;
         public PgoFileType? FileType;
@@ -29,8 +30,7 @@ internal class CommandLineOptions
         public bool ValidateOutputFile;
         public bool GenerateCallGraph;
         public bool Spgo;
-        public bool SpgoIncludeBlockCounts;
-        public bool SpgoIncludeEdgeCounts;
+        public bool IncludeFullGraphs;
         public int SpgoMinSamples = 50;
         public bool VerboseWarnings;
         public jittraceoptions JitTraceOptions;
@@ -45,6 +45,8 @@ internal class CommandLineOptions
         public bool DumpMibc = false;
         public FileInfo InputFileToDump;
         public List<FileInfo> CompareMibc;
+        public DirectoryInfo DumpWorstOverlapGraphsTo;
+        public int DumpWorstOverlapGraphs = -1;
         public bool InheritTimestamp;
 
         public string[] HelpArgs = Array.Empty<string>();
@@ -196,13 +198,15 @@ void HelpOption()
                 CommonOptions();
                 CompressedOption();
 
+                string preciseDebugInfoFile = null;
+                syntax.DefineOption(name: "precise-debug-info-file", ref preciseDebugInfoFile, "Name of file of newline separated JSON objects containing precise debug info");
+                if (preciseDebugInfoFile != null)
+                    PreciseDebugInfoFile = new FileInfo(preciseDebugInfoFile);
+
                 syntax.DefineOption(name: "spgo", value: ref Spgo, help: "Base profile on samples in the input. Uses last branch records if available and otherwise raw IP samples.", requireValue: false);
-                syntax.DefineOption(name: "spgo-with-block-counts", value: ref SpgoIncludeBlockCounts, help: "Include block counts in the written .mibc file. If neither this nor spgo-with-edge-counts are specified, then defaults to true.", requireValue: false);
-                syntax.DefineOption(name: "spgo-with-edge-counts", value: ref SpgoIncludeEdgeCounts, help: "Include edge counts in the written .mibc file.", requireValue: false);
                 syntax.DefineOption(name: "spgo-min-samples", value: ref SpgoMinSamples, help: $"The minimum number of total samples a function must have before generating profile data for it with SPGO. Default: {SpgoMinSamples}", requireValue: false);
 
-                if (!SpgoIncludeBlockCounts && !SpgoIncludeEdgeCounts)
-                    SpgoIncludeBlockCounts = true;
+                syntax.DefineOption(name: "include-full-graphs", value: ref IncludeFullGraphs, help: "Include all blocks and edges in the written .mibc file, regardless of profile counts", requireValue: false);
 
                 HelpOption();
             }
@@ -305,6 +309,12 @@ void HelpOption()
                 CompareMibc = DefineFileOptionList(name: "i|input", help: "The input .mibc files to be compared. Specify as --input file1.mibc --input file2.mibc");
                 if (CompareMibc.Count != 2)
                     Help = true;
+
+                syntax.DefineOption(name: "dump-worst-overlap-graphs", value: ref DumpWorstOverlapGraphs, help: "Number of graphs to dump to .dot format in dump-worst-overlap-graphs-to directory");
+                string dumpWorstOverlapGraphsTo = null;
+                syntax.DefineOption(name: "dump-worst-overlap-graphs-to", value: ref dumpWorstOverlapGraphsTo, help: "Number of graphs to dump to .dot format in dump-worst-overlap-graphs-to directory");
+                if (dumpWorstOverlapGraphsTo != null)
+                    DumpWorstOverlapGraphsTo = new DirectoryInfo(dumpWorstOverlapGraphsTo);
             }
 
             if (syntax.ActiveCommand == null)