diff --git a/BenchmarkDotNet.sln b/BenchmarkDotNet.sln index 1df6c0aabd..ad1cb3e28e 100644 --- a/BenchmarkDotNet.sln +++ b/BenchmarkDotNet.sln @@ -33,10 +33,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BenchmarkDotNet.Integration EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BenchmarkDotNet.IntegrationTests.ConfigPerAssembly", "tests\BenchmarkDotNet.IntegrationTests.ConfigPerAssembly\BenchmarkDotNet.IntegrationTests.ConfigPerAssembly.csproj", "{043F1DA4-CD51-45FD-805E-6571D67AA661}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BenchmarkDotNet.Disassembler.x64", "src\BenchmarkDotNet.Disassembler.x64\BenchmarkDotNet.Disassembler.x64.csproj", "{E5A0833C-B633-4D62-B645-A927CEBFEEBB}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BenchmarkDotNet.Disassembler.x86", "src\BenchmarkDotNet.Disassembler.x86\BenchmarkDotNet.Disassembler.x86.csproj", "{D189AAB3-46B4-4437-8E9C-72F021AB2B6E}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BenchmarkDotNet.IntegrationTests.ManualRunning", "tests\BenchmarkDotNet.IntegrationTests.ManualRunning\BenchmarkDotNet.IntegrationTests.ManualRunning.csproj", "{9816D316-95C4-42E6-9E7B-A256C7E5D4BF}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BenchmarkDotNet.IntegrationTests.Static", "tests\BenchmarkDotNet.IntegrationTests.Static\BenchmarkDotNet.IntegrationTests.Static.csproj", "{B4405781-40D3-42B8-B168-00E711FABA15}" @@ -59,6 +55,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.P EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.Plotting.Tests", "tests\BenchmarkDotNet.Exporters.Plotting.Tests\BenchmarkDotNet.Exporters.Plotting.Tests.csproj", "{199AC83E-30BD-40CD-87CE-0C838AC0320D}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Disassembler", "src\BenchmarkDotNet.Disassembler\BenchmarkDotNet.Disassembler.csproj", "{86DB5A20-73FA-DB96-1545-08FE5878637C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -113,14 +111,6 @@ Global {043F1DA4-CD51-45FD-805E-6571D67AA661}.Debug|Any CPU.Build.0 = Debug|Any CPU {043F1DA4-CD51-45FD-805E-6571D67AA661}.Release|Any CPU.ActiveCfg = Release|Any CPU {043F1DA4-CD51-45FD-805E-6571D67AA661}.Release|Any CPU.Build.0 = Release|Any CPU - {E5A0833C-B633-4D62-B645-A927CEBFEEBB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {E5A0833C-B633-4D62-B645-A927CEBFEEBB}.Debug|Any CPU.Build.0 = Debug|Any CPU - {E5A0833C-B633-4D62-B645-A927CEBFEEBB}.Release|Any CPU.ActiveCfg = Release|Any CPU - {E5A0833C-B633-4D62-B645-A927CEBFEEBB}.Release|Any CPU.Build.0 = Release|Any CPU - {D189AAB3-46B4-4437-8E9C-72F021AB2B6E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {D189AAB3-46B4-4437-8E9C-72F021AB2B6E}.Debug|Any CPU.Build.0 = Debug|Any CPU - {D189AAB3-46B4-4437-8E9C-72F021AB2B6E}.Release|Any CPU.ActiveCfg = Release|Any CPU - {D189AAB3-46B4-4437-8E9C-72F021AB2B6E}.Release|Any CPU.Build.0 = Release|Any CPU {9816D316-95C4-42E6-9E7B-A256C7E5D4BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {9816D316-95C4-42E6-9E7B-A256C7E5D4BF}.Debug|Any CPU.Build.0 = Debug|Any CPU {9816D316-95C4-42E6-9E7B-A256C7E5D4BF}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -161,6 +151,10 @@ Global {199AC83E-30BD-40CD-87CE-0C838AC0320D}.Debug|Any CPU.Build.0 = Debug|Any CPU {199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.ActiveCfg = Release|Any CPU {199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.Build.0 = Release|Any CPU + {86DB5A20-73FA-DB96-1545-08FE5878637C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {86DB5A20-73FA-DB96-1545-08FE5878637C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {86DB5A20-73FA-DB96-1545-08FE5878637C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {86DB5A20-73FA-DB96-1545-08FE5878637C}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -178,8 +172,6 @@ Global {45FE17A7-0E04-48C0-8CDC-493CDA449F7A} = {14195214-591A-45B7-851A-19D3BA2413F9} {6A3CBB07-E337-488E-BDAC-ED96AF8ED608} = {14195214-591A-45B7-851A-19D3BA2413F9} {043F1DA4-CD51-45FD-805E-6571D67AA661} = {14195214-591A-45B7-851A-19D3BA2413F9} - {E5A0833C-B633-4D62-B645-A927CEBFEEBB} = {D6597E3A-6892-4A68-8E14-042FC941FDA2} - {D189AAB3-46B4-4437-8E9C-72F021AB2B6E} = {D6597E3A-6892-4A68-8E14-042FC941FDA2} {9816D316-95C4-42E6-9E7B-A256C7E5D4BF} = {14195214-591A-45B7-851A-19D3BA2413F9} {B4405781-40D3-42B8-B168-00E711FABA15} = {14195214-591A-45B7-851A-19D3BA2413F9} {D9F5065B-6190-431B-850C-117E3D64AB33} = {D6597E3A-6892-4A68-8E14-042FC941FDA2} @@ -190,6 +182,7 @@ Global {2E2283A3-6DA6-4482-8518-99D6D9F689AB} = {D6597E3A-6892-4A68-8E14-042FC941FDA2} {B92ECCEF-7C27-4012-9E19-679F3C40A6A6} = {D6597E3A-6892-4A68-8E14-042FC941FDA2} {199AC83E-30BD-40CD-87CE-0C838AC0320D} = {14195214-591A-45B7-851A-19D3BA2413F9} + {86DB5A20-73FA-DB96-1545-08FE5878637C} = {D6597E3A-6892-4A68-8E14-042FC941FDA2} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {4D9AF12B-1F7F-45A7-9E8C-E4E46ADCBD1F} diff --git a/src/BenchmarkDotNet.Disassembler.x64/BenchmarkDotNet.Disassembler.x64.csproj b/src/BenchmarkDotNet.Disassembler.x64/BenchmarkDotNet.Disassembler.x64.csproj deleted file mode 100644 index 2f15efcc13..0000000000 --- a/src/BenchmarkDotNet.Disassembler.x64/BenchmarkDotNet.Disassembler.x64.csproj +++ /dev/null @@ -1,21 +0,0 @@ - - - - net462 - Exe - BenchmarkDotNet.Disassembler.x64 - BenchmarkDotNet.Disassembler.x64 - win7-x64 - x64 - True - $(DefineConstants);CLRMDV1 - - - ..\BenchmarkDotNet\Disassemblers - BenchmarkDotNet.Disassembler - - - - - - diff --git a/src/BenchmarkDotNet.Disassembler.x64/ClrMdV1Disassembler.cs b/src/BenchmarkDotNet.Disassembler.x64/ClrMdV1Disassembler.cs deleted file mode 100644 index 734aa470e4..0000000000 --- a/src/BenchmarkDotNet.Disassembler.x64/ClrMdV1Disassembler.cs +++ /dev/null @@ -1,308 +0,0 @@ -using Iced.Intel; -using Microsoft.Diagnostics.Runtime; -using Microsoft.Diagnostics.Runtime.Interop; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.RegularExpressions; - -namespace BenchmarkDotNet.Disassemblers -{ - // This Disassembler uses ClrMd v1x. Please keep it in sync with ClrMdV2Disassembler (if possible). - internal static class ClrMdV1Disassembler - { - internal static DisassemblyResult AttachAndDisassemble(Settings settings) - { - using (var dataTarget = DataTarget.AttachToProcess( - settings.ProcessId, - (uint)TimeSpan.FromSeconds(5).TotalMilliseconds, - AttachFlag.Passive)) - { - var runtime = dataTarget.ClrVersions.Single().CreateRuntime(); - - // Per https://github.com/microsoft/clrmd/issues/303 - dataTarget.DataReader.Flush(); - - ConfigureSymbols(dataTarget); - - var state = new State(runtime, settings.TargetFrameworkMoniker); - - if (settings.Filters.Length > 0) - { - FilterAndEnqueue(state, settings); - } - else - { - var typeWithBenchmark = state.Runtime.Heap.GetTypeByName(settings.TypeName); - - state.Todo.Enqueue( - new MethodInfo( - // the Disassembler Entry Method is always parameterless, so check by name is enough - typeWithBenchmark.Methods.Single(method => method.IsPublic && method.Name == settings.MethodName), - 0)); - } - - var disassembledMethods = Disassemble(settings, state); - - // we don't want to export the disassembler entry point method which is just an artificial method added to get generic types working - var filteredMethods = disassembledMethods.Length == 1 - ? disassembledMethods // if there is only one method we want to return it (most probably benchmark got inlined) - : disassembledMethods.Where(method => !method.Name.Contains(DisassemblerConstants.DisassemblerEntryMethodName)).ToArray(); - - return new DisassemblyResult - { - Methods = filteredMethods, - SerializedAddressToNameMapping = state.AddressToNameMapping.Select(x => new DisassemblyResult.MutablePair { Key = x.Key, Value = x.Value }).ToArray(), - PointerSize = (uint)IntPtr.Size - }; - } - } - - private static void ConfigureSymbols(DataTarget dataTarget) - { - // code copied from https://github.com/Microsoft/clrmd/issues/34#issuecomment-161926535 - var symbols = dataTarget.DebuggerInterface as IDebugSymbols; - symbols?.SetSymbolPath("http://msdl.microsoft.com/download/symbols"); - var control = dataTarget.DebuggerInterface as IDebugControl; - control?.Execute(DEBUG_OUTCTL.NOT_LOGGED, ".reload", DEBUG_EXECUTE.NOT_LOGGED); - } - - private static void FilterAndEnqueue(State state, Settings settings) - { - Regex[] filters = settings.Filters - .Select(pattern => new Regex(WildcardToRegex(pattern), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)).ToArray(); - - foreach (ClrModule module in state.Runtime.Modules) - foreach (ClrType type in module.EnumerateTypes()) - foreach (ClrMethod method in type.Methods.Where(method => CanBeDisassembled(method) && method.GetFullSignature() != null)) - foreach (Regex filter in filters) - { - if (filter.IsMatch(method.GetFullSignature())) - { - state.Todo.Enqueue(new MethodInfo(method, - depth: settings.MaxDepth)); // don't allow for recursive disassembling - break; - } - } - } - - // copied from GlobFilter type (this type must not reference BDN) - private static string WildcardToRegex(string pattern) => $"^{Regex.Escape(pattern).Replace(@"\*", ".*").Replace(@"\?", ".")}$"; - - private static DisassembledMethod[] Disassemble(Settings settings, State state) - { - var result = new List(); - - using var sourceCodeProvider = new SourceCodeProvider(); - while (state.Todo.Count != 0) - { - var methodInfo = state.Todo.Dequeue(); - - if (!state.HandledMethods.Add(methodInfo.Method)) // add it now to avoid StackOverflow for recursive methods - continue; // already handled - - if (settings.MaxDepth >= methodInfo.Depth) - result.Add(DisassembleMethod(methodInfo, state, settings, sourceCodeProvider)); - } - - return result.ToArray(); - } - - private static bool CanBeDisassembled(ClrMethod method) - => !((method.ILOffsetMap is null || method.ILOffsetMap.Length == 0) && (method.HotColdInfo is null || method.HotColdInfo.HotStart == 0 || method.HotColdInfo.HotSize == 0)); - - private static DisassembledMethod DisassembleMethod(MethodInfo methodInfo, State state, Settings settings, SourceCodeProvider sourceCodeProvider) - { - var method = methodInfo.Method; - - if (!CanBeDisassembled(method)) - { - if (method.IsPInvoke) - return CreateEmpty(method, "PInvoke method"); - if (method.IL is null || method.IL.Length == 0) - return CreateEmpty(method, "Extern method"); - if (method.CompilationType == MethodCompilationType.None) - return CreateEmpty(method, "Method was not JITted yet."); - - return CreateEmpty(method, $"No valid {nameof(method.ILOffsetMap)} and {nameof(method.HotColdInfo)}"); - } - - var codes = new List(); - if (settings.PrintSource && !(method.ILOffsetMap is null)) - { - // we use HashSet to prevent from duplicates - var uniqueSourceCodeLines = new HashSet(new SharpComparer()); - // for getting C# code we always use the original ILOffsetMap - foreach (var map in method.ILOffsetMap.Where(map => map.StartAddress < map.EndAddress && map.ILOffset >= 0).OrderBy(map => map.StartAddress)) - foreach (var sharp in sourceCodeProvider.GetSource(method, map)) - uniqueSourceCodeLines.Add(sharp); - - codes.AddRange(uniqueSourceCodeLines); - } - - // for getting ASM we try to use data from HotColdInfo if available (better for decoding) - foreach (var map in GetCompleteNativeMap(method)) - codes.AddRange(Decode(map.StartAddress, (uint)(map.EndAddress - map.StartAddress), state, methodInfo.Depth, method)); - - Map[] maps = settings.PrintSource - ? codes.GroupBy(code => code.InstructionPointer).OrderBy(group => group.Key).Select(group => new Map() { SourceCodes = group.ToArray() }).ToArray() - : new[] { new Map() { SourceCodes = codes.ToArray() } }; - - return new DisassembledMethod - { - Maps = maps, - Name = method.GetFullSignature(), - NativeCode = method.NativeCode - }; - } - - private static IEnumerable Decode(ulong startAddress, uint size, State state, int depth, ClrMethod currentMethod) - { - byte[] code = new byte[size]; - if (!state.Runtime.DataTarget.ReadProcessMemory(startAddress, code, code.Length, out int bytesRead) || bytesRead == 0) - yield break; - - var reader = new ByteArrayCodeReader(code, 0, bytesRead); - var decoder = Decoder.Create(state.Runtime.PointerSize * 8, reader); - decoder.IP = startAddress; - - while (reader.CanReadByte) - { - decoder.Decode(out var instruction); - - TryTranslateAddressToName(instruction, state, depth, currentMethod, out ulong referencedAddress); - - yield return new IntelAsm - { - InstructionPointer = instruction.IP, - InstructionLength = instruction.Length, - Instruction = instruction, - ReferencedAddress = (referencedAddress > ushort.MaxValue) ? referencedAddress : null, - }; - } - } - - private static void TryTranslateAddressToName(Instruction instruction, State state, int depth, ClrMethod currentMethod, out ulong address) - { - var runtime = state.Runtime; - - if (!TryGetReferencedAddress(instruction, (uint)runtime.PointerSize, out address)) - return; - - if (state.AddressToNameMapping.ContainsKey(address)) - return; - - var jitHelperFunctionName = runtime.GetJitHelperFunctionName(address); - if (!string.IsNullOrEmpty(jitHelperFunctionName)) - { - state.AddressToNameMapping.Add(address, jitHelperFunctionName); - return; - } - - var methodTableName = runtime.GetMethodTableName(address); - if (!string.IsNullOrEmpty(methodTableName)) - { - state.AddressToNameMapping.Add(address, $"MT_{methodTableName}"); - return; - } - - var methodDescriptor = runtime.GetMethodByHandle(address); - if (!(methodDescriptor is null)) - { - state.AddressToNameMapping.Add(address, $"MD_{methodDescriptor.GetFullSignature()}"); - return; - } - - var method = runtime.GetMethodByAddress(address); - if (method is null && (address & ((uint)runtime.PointerSize - 1)) == 0) - { - if (runtime.ReadPointer(address, out ulong newAddress) && newAddress > ushort.MaxValue) - method = runtime.GetMethodByAddress(newAddress); - } - - if (method is null) - return; - - if (method.NativeCode == currentMethod.NativeCode && method.GetFullSignature() == currentMethod.GetFullSignature()) - return; // in case of a call which is just a jump within the method or a recursive call - - if (!state.HandledMethods.Contains(method)) - state.Todo.Enqueue(new MethodInfo(method, depth + 1)); - - var methodName = method.GetFullSignature(); - if (!methodName.Any(c => c == '.')) // the method name does not contain namespace and type name - methodName = $"{method.Type.Name}.{method.GetFullSignature()}"; - state.AddressToNameMapping.Add(address, methodName); - } - - internal static bool TryGetReferencedAddress(Instruction instruction, uint pointerSize, out ulong referencedAddress) - { - for (int i = 0; i < instruction.OpCount; i++) - { - switch (instruction.GetOpKind(i)) - { - case OpKind.NearBranch16: - case OpKind.NearBranch32: - case OpKind.NearBranch64: - referencedAddress = instruction.NearBranchTarget; - return referencedAddress > ushort.MaxValue; - case OpKind.Immediate16: - case OpKind.Immediate8to16: - case OpKind.Immediate8to32: - case OpKind.Immediate8to64: - case OpKind.Immediate32to64: - case OpKind.Immediate32 when pointerSize == 4: - case OpKind.Immediate64: - referencedAddress = instruction.GetImmediate(i); - return referencedAddress > ushort.MaxValue; - case OpKind.Memory when instruction.IsIPRelativeMemoryOperand: - referencedAddress = instruction.IPRelativeMemoryAddress; - return referencedAddress > ushort.MaxValue; - case OpKind.Memory: - referencedAddress = instruction.MemoryDisplacement64; - return referencedAddress > ushort.MaxValue; - } - } - - referencedAddress = default; - return false; - } - - private static ILToNativeMap[] GetCompleteNativeMap(ClrMethod method) - { - // it's better to use one single map rather than few small ones - // it's simply easier to get next instruction when decoding ;) - var hotColdInfo = method.HotColdInfo; - if (!(hotColdInfo is null) && hotColdInfo.HotSize > 0 && hotColdInfo.HotStart > 0) - { - return hotColdInfo.ColdSize <= 0 - ? new[] { new ILToNativeMap() { StartAddress = hotColdInfo.HotStart, EndAddress = hotColdInfo.HotStart + hotColdInfo.HotSize, ILOffset = -1 } } - : new[] - { - new ILToNativeMap() { StartAddress = hotColdInfo.HotStart, EndAddress = hotColdInfo.HotStart + hotColdInfo.HotSize, ILOffset = -1 }, - new ILToNativeMap() { StartAddress = hotColdInfo.ColdStart, EndAddress = hotColdInfo.ColdStart + hotColdInfo.ColdSize, ILOffset = -1 } - }; - } - - return method.ILOffsetMap - .Where(map => map.StartAddress < map.EndAddress) // some maps have 0 length? - .OrderBy(map => map.StartAddress) // we need to print in the machine code order, not IL! #536 - .ToArray(); - } - - private static DisassembledMethod CreateEmpty(ClrMethod method, string reason) - => DisassembledMethod.Empty(method.GetFullSignature(), method.NativeCode, reason); - - private class SharpComparer : IEqualityComparer - { - public bool Equals(Sharp x, Sharp y) - { - // sometimes some C# code lines are duplicated because the same line is the best match for multiple ILToNativeMaps - // we don't want to confuse the users, so this must also be removed - return x.FilePath == y.FilePath && x.LineNumber == y.LineNumber; - } - - public int GetHashCode(Sharp obj) => obj.FilePath.GetHashCode() ^ obj.LineNumber; - } - } -} diff --git a/src/BenchmarkDotNet.Disassembler.x64/DataContracts.cs b/src/BenchmarkDotNet.Disassembler.x64/DataContracts.cs deleted file mode 100644 index f088430970..0000000000 --- a/src/BenchmarkDotNet.Disassembler.x64/DataContracts.cs +++ /dev/null @@ -1,224 +0,0 @@ -using Iced.Intel; -using Microsoft.Diagnostics.Runtime; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Xml.Serialization; - -#pragma warning disable CS3001 // Argument type 'ulong' is not CLS-compliant -#pragma warning disable CS3003 // Type is not CLS-compliant -#pragma warning disable CS1591 // XML comments for public types... -namespace BenchmarkDotNet.Disassemblers -{ - public abstract class SourceCode - { - public ulong InstructionPointer { get; set; } - } - - public class Sharp : SourceCode - { - public string Text { get; set; } - public string FilePath { get; set; } - public int LineNumber { get; set; } - } - - public abstract class Asm : SourceCode - { - public int InstructionLength { get; set; } - public ulong? ReferencedAddress { get; set; } - public bool IsReferencedAddressIndirect { get; set; } - } - - public class IntelAsm : Asm - { - public Instruction Instruction { get; set; } - - public override string ToString() => Instruction.ToString(); - } - - public class Arm64Asm : Asm - { -#if !CLRMDV1 // don't include it in ClrMD V1 disassembler that supports only x86 and x64 - public Gee.External.Capstone.Arm64.Arm64Instruction Instruction { get; set; } - - public override string ToString() => Instruction.ToString(); -#endif - } - - public class MonoCode : SourceCode - { - public string Text { get; set; } - } - - public class Map - { - [XmlArray("Instructions")] - [XmlArrayItem(nameof(SourceCode), typeof(SourceCode))] - [XmlArrayItem(nameof(Sharp), typeof(Sharp))] - [XmlArrayItem(nameof(IntelAsm), typeof(IntelAsm))] - public SourceCode[] SourceCodes { get; set; } - } - - public class DisassembledMethod - { - public string Name { get; set; } - - public ulong NativeCode { get; set; } - - public string Problem { get; set; } - - public Map[] Maps { get; set; } - - public string CommandLine { get; set; } - - public static DisassembledMethod Empty(string fullSignature, ulong nativeCode, string problem) - => new DisassembledMethod - { - Name = fullSignature, - NativeCode = nativeCode, - Maps = Array.Empty(), - Problem = problem - }; - } - - public class DisassemblyResult - { - public DisassembledMethod[] Methods { get; set; } - public string[] Errors { get; set; } - public MutablePair[] SerializedAddressToNameMapping { get; set; } - public uint PointerSize { get; set; } - - [XmlIgnore] // XmlSerializer does not support dictionaries ;) - public Dictionary AddressToNameMapping - => _addressToNameMapping ?? (_addressToNameMapping = SerializedAddressToNameMapping.ToDictionary(x => x.Key, x => x.Value)); - - [XmlIgnore] - private Dictionary _addressToNameMapping; - - public DisassemblyResult() - { - Methods = Array.Empty(); - Errors = Array.Empty(); - } - - // KeyValuePair is not serializable, because it has read-only properties - // so we need to define our own... - [Serializable] - [XmlType(TypeName = "Workaround")] - public struct MutablePair - { - public ulong Key { get; set; } - public string Value { get; set; } - } - } - - public static class DisassemblerConstants - { - public const string DisassemblerEntryMethodName = "__ForDisassemblyDiagnoser__"; - } - - internal class Settings - { - internal Settings(int processId, string typeName, string methodName, bool printSource, int maxDepth, string resultsPath, string syntax, string tfm, string[] filters) - { - ProcessId = processId; - TypeName = typeName; - MethodName = methodName; - PrintSource = printSource; - MaxDepth = methodName == DisassemblerConstants.DisassemblerEntryMethodName && maxDepth != int.MaxValue ? maxDepth + 1 : maxDepth; - ResultsPath = resultsPath; - Syntax = syntax; - TargetFrameworkMoniker = tfm; - Filters = filters; - } - - internal int ProcessId { get; } - internal string TypeName { get; } - internal string MethodName { get; } - internal bool PrintSource { get; } - internal int MaxDepth { get; } - internal string[] Filters; - internal string Syntax { get; } - internal string TargetFrameworkMoniker { get; } - internal string ResultsPath { get; } - - internal static Settings FromArgs(string[] args) - => new Settings( - processId: int.Parse(args[0]), - typeName: args[1], - methodName: args[2], - printSource: bool.Parse(args[3]), - maxDepth: int.Parse(args[4]), - resultsPath: args[5], - syntax: args[6], - tfm: args[7], - filters: args.Skip(8).ToArray() - ); - } - - internal class State - { - internal State(ClrRuntime runtime, string targetFrameworkMoniker) - { - Runtime = runtime; - Todo = new Queue(); - HandledMethods = new HashSet(new ClrMethodComparer()); - AddressToNameMapping = new Dictionary(); - RuntimeVersion = ParseVersion(targetFrameworkMoniker); - } - - internal ClrRuntime Runtime { get; } - internal string TargetFrameworkMoniker { get; } - internal Queue Todo { get; } - internal HashSet HandledMethods { get; } - internal Dictionary AddressToNameMapping { get; } - internal Version RuntimeVersion { get; } - - internal static Version ParseVersion(string targetFrameworkMoniker) - { - int firstDigit = -1, lastDigit = -1; - for (int i = 0; i < targetFrameworkMoniker.Length; i++) - { - if (char.IsDigit(targetFrameworkMoniker[i])) - { - if (firstDigit == -1) - firstDigit = i; - - lastDigit = i; - } - else if (targetFrameworkMoniker[i] == '-') - { - break; // it can be platform specific like net7.0-windows8 - } - } - - string versionToParse = targetFrameworkMoniker.Substring(firstDigit, lastDigit - firstDigit + 1); - if (!versionToParse.Contains(".")) // Full .NET Framework (net48 etc) - versionToParse = string.Join(".", versionToParse.ToCharArray()); - - return Version.Parse(versionToParse); - } - - private sealed class ClrMethodComparer : IEqualityComparer - { - public bool Equals(ClrMethod x, ClrMethod y) => x.NativeCode == y.NativeCode; - - public int GetHashCode(ClrMethod obj) => (int)obj.NativeCode; - } - } - - internal readonly struct MethodInfo // I am not using ValueTuple here (would be perfect) to keep the number of dependencies as low as possible - { - internal ClrMethod Method { get; } - internal int Depth { get; } - - internal MethodInfo(ClrMethod method, int depth) - { - Method = method; - Depth = depth; - } - } -} -#pragma warning restore CS1591 // XML comments for public types... -#pragma warning restore CS3003 // Type is not CLS-compliant -#pragma warning restore CS3001 // Argument type 'ulong' is not CLS-compliant \ No newline at end of file diff --git a/src/BenchmarkDotNet.Disassembler.x64/SourceCodeProvider.cs b/src/BenchmarkDotNet.Disassembler.x64/SourceCodeProvider.cs deleted file mode 100644 index f827369775..0000000000 --- a/src/BenchmarkDotNet.Disassembler.x64/SourceCodeProvider.cs +++ /dev/null @@ -1,222 +0,0 @@ -using Microsoft.Diagnostics.Runtime; -using Microsoft.Diagnostics.Runtime.Utilities; -using Microsoft.Diagnostics.Runtime.Utilities.Pdb; -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; - -namespace BenchmarkDotNet.Disassemblers -{ - // This is taken from the Samples\FileAndLineNumbers projects from microsoft/clrmd, - // and replaces the previously-available SourceLocation functionality. - - internal class SourceLocation - { - public string FilePath; - public int LineNumber; - public int LineNumberEnd; - public int ColStart; - public int ColEnd; - } - - internal class SourceCodeProvider : IDisposable - { - private readonly Dictionary sourceFileCache = new Dictionary(); - private readonly Dictionary pdbReaders = new Dictionary(); - - public void Dispose() - { - foreach (var reader in pdbReaders.Values) - { - reader?.Dispose(); - } - } - - internal IEnumerable GetSource(ClrMethod method, ILToNativeMap map) - { - var sourceLocation = GetSourceLocation(method, map.ILOffset); - if (sourceLocation == null) - yield break; - - for (int line = sourceLocation.LineNumber; line <= sourceLocation.LineNumberEnd; ++line) - { - var sourceLine = ReadSourceLine(sourceLocation.FilePath, line); - if (sourceLine == null) - continue; - - var text = sourceLine + Environment.NewLine - + GetSmartPointer(sourceLine, - start: line == sourceLocation.LineNumber ? sourceLocation.ColStart - 1 : default(int?), - end: line == sourceLocation.LineNumberEnd ? sourceLocation.ColEnd - 1 : default(int?)); - - yield return new Sharp - { - Text = text, - InstructionPointer = map.StartAddress, - FilePath = sourceLocation.FilePath, - LineNumber = line - }; - } - } - - private string ReadSourceLine(string file, int line) - { - if (!sourceFileCache.TryGetValue(file, out string[] contents)) - { - // sometimes the symbols report some disk location from MS CI machine like "E:\A\_work\308\s\src\mscorlib\shared\System\Random.cs" for .NET Core 2.0 - if (!File.Exists(file)) - return null; - - contents = File.ReadAllLines(file); - sourceFileCache.Add(file, contents); - } - - return line - 1 < contents.Length - ? contents[line - 1] - : null; // "nop" can have no corresponding c# code ;) - } - - private static string GetSmartPointer(string sourceLine, int? start, int? end) - { - Debug.Assert(start is null || start < sourceLine.Length); - Debug.Assert(end is null || end <= sourceLine.Length); - - var prefix = new char[end ?? sourceLine.Length]; - var index = 0; - - // write offset using whitespaces - while (index < (start ?? prefix.Length)) - { - prefix[index] = - sourceLine.Length > index && - sourceLine[index] == '\t' - ? '\t' - : ' '; - index++; - } - - // write smart pointer - while (index < prefix.Length) - { - prefix[index] = '^'; - index++; - } - - return new string(prefix); - } - - internal SourceLocation GetSourceLocation(ClrMethod method, int ilOffset) - { - PdbReader reader = GetReaderForMethod(method); - if (reader == null) - return null; - - PdbFunction function = reader.GetFunctionFromToken(method.MetadataToken); - return FindNearestLine(function, ilOffset); - } - - internal SourceLocation GetSourceLocation(ClrStackFrame frame) - { - PdbReader reader = GetReaderForMethod(frame.Method); - if (reader == null) - return null; - - PdbFunction function = reader.GetFunctionFromToken(frame.Method.MetadataToken); - int ilOffset = FindIlOffset(frame); - - return FindNearestLine(function, ilOffset); - } - - private static SourceLocation FindNearestLine(PdbFunction function, int ilOffset) - { - if (function == null || function.SequencePoints == null) - return null; - - int distance = int.MaxValue; - SourceLocation? nearest = null; - - foreach (PdbSequencePointCollection sequenceCollection in function.SequencePoints) - { - foreach (PdbSequencePoint point in sequenceCollection.Lines) - { - int dist = (int)Math.Abs(point.Offset - ilOffset); - if (dist < distance) - { - if (nearest == null) - nearest = new SourceLocation(); - - nearest.FilePath = sequenceCollection.File.Name; - nearest.LineNumber = (int)point.LineBegin; - nearest.LineNumberEnd = (int)point.LineEnd; - nearest.ColStart = (int)point.ColBegin; - nearest.ColEnd = (int)point.ColEnd; - - distance = dist; - } - } - } - - return nearest; - } - - private static int FindIlOffset(ClrStackFrame frame) - { - ulong ip = frame.InstructionPointer; - int last = -1; - foreach (ILToNativeMap item in frame.Method.ILOffsetMap) - { - if (item.StartAddress > ip) - return last; - - if (ip <= item.EndAddress) - return item.ILOffset; - - last = item.ILOffset; - } - - return last; - } - - private PdbReader GetReaderForMethod(ClrMethod method) - { - ClrModule module = method?.Type?.Module; - PdbInfo info = module?.Pdb; - - PdbReader? reader = null; - if (info != null) - { - if (!pdbReaders.TryGetValue(info, out reader)) - { - SymbolLocator locator = GetSymbolLocator(module); - string pdbPath = locator.FindPdb(info); - if (pdbPath != null) - { - try - { - reader = new PdbReader(pdbPath); - } - catch (IOException) - { - // This will typically happen when trying to load information - // from public symbols, or symbol files generated by some weird - // compiler. We can ignore this, but there's no need to load - // this PDB anymore, so we will put null in the dictionary and - // be done with it. - reader = null; - } - } - - pdbReaders[info] = reader; - } - } - - return reader; - } - - private static SymbolLocator GetSymbolLocator(ClrModule module) - { - return module.Runtime.DataTarget.SymbolLocator; - } - } -} diff --git a/src/BenchmarkDotNet.Disassembler.x86/BenchmarkDotNet.Disassembler.x86.csproj b/src/BenchmarkDotNet.Disassembler.x86/BenchmarkDotNet.Disassembler.x86.csproj deleted file mode 100644 index 5410f6d77b..0000000000 --- a/src/BenchmarkDotNet.Disassembler.x86/BenchmarkDotNet.Disassembler.x86.csproj +++ /dev/null @@ -1,27 +0,0 @@ - - - - net462 - Exe - BenchmarkDotNet.Disassembler.x86 - BenchmarkDotNet.Disassembler.x86 - win7-x86 - x86 - True - $(DefineConstants);CLRMDV1 - - - ..\BenchmarkDotNet\Disassemblers - BenchmarkDotNet.Disassembler - - - - - - - - - - - - diff --git a/src/BenchmarkDotNet.Disassembler/BenchmarkDotNet.Disassembler.csproj b/src/BenchmarkDotNet.Disassembler/BenchmarkDotNet.Disassembler.csproj new file mode 100644 index 0000000000..6f48d2f030 --- /dev/null +++ b/src/BenchmarkDotNet.Disassembler/BenchmarkDotNet.Disassembler.csproj @@ -0,0 +1,18 @@ + + + + net462;net8.0 + Exe + BenchmarkDotNet.Disassembler + BenchmarkDotNet.Disassembler + x64 + + false + + + BenchmarkDotNet.Disassembler + + + + + diff --git a/src/BenchmarkDotNet.Disassembler.x64/Program.cs b/src/BenchmarkDotNet.Disassembler/Program.cs similarity index 89% rename from src/BenchmarkDotNet.Disassembler.x64/Program.cs rename to src/BenchmarkDotNet.Disassembler/Program.cs index 701c4764bc..1b7d89d7f6 100644 --- a/src/BenchmarkDotNet.Disassembler.x64/Program.cs +++ b/src/BenchmarkDotNet.Disassembler/Program.cs @@ -1,4 +1,5 @@ -using System; +using BenchmarkDotNet.Diagnosers; +using System; using System.Diagnostics; using System.IO; using System.Xml; @@ -13,18 +14,16 @@ internal static class Program // 2. disassemble the code // 3. save it to xml file // 4. detach & shut down - // - // requirements: must not have any dependencies to BenchmarkDotNet itself, KISS public static void Main(string[] args) { - var options = Settings.FromArgs(args); + var options = ClrMdArgs.FromArgs(args); if (Process.GetProcessById(options.ProcessId).HasExited) // possible when benchmark has already finished throw new Exception($"The process {options.ProcessId} has already exited"); // if we don't throw here the Clrmd will fail with some mysterious HRESULT: 0xd000010a ;) try { - var methodsToExport = ClrMdV1Disassembler.AttachAndDisassemble(options); + var methodsToExport = DisassemblyDiagnoser.GetClrMdDisassembler().AttachAndDisassemble(options); SaveToFile(methodsToExport, options.ResultsPath); } diff --git a/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs b/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs index d49c3d24e1..06b5e5b53a 100644 --- a/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs +++ b/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs @@ -58,7 +58,8 @@ public static BenchmarkRunInfo[] GetBenchmarksFromAssemblyPath(string assemblyPa benchmarkRunInfo = new BenchmarkRunInfo( benchmarkRunInfo.BenchmarksCases.Where(c => c.GetToolchain().IsInProcess).ToArray(), benchmarkRunInfo.Type, - benchmarkRunInfo.Config); + benchmarkRunInfo.Config, + benchmarkRunInfo.CompositeInProcessDiagnoser); } return benchmarkRunInfo; diff --git a/src/BenchmarkDotNet.TestAdapter/BenchmarkExecutor.cs b/src/BenchmarkDotNet.TestAdapter/BenchmarkExecutor.cs index aad574ed76..c132bb6b72 100644 --- a/src/BenchmarkDotNet.TestAdapter/BenchmarkExecutor.cs +++ b/src/BenchmarkDotNet.TestAdapter/BenchmarkExecutor.cs @@ -48,7 +48,7 @@ public void RunBenchmarks(string assemblyPath, TestExecutionRecorderWrapper reco if (filteredCases.Count > 0) { - filteredBenchmarks.Add(new BenchmarkRunInfo(filteredCases.ToArray(), benchmark.Type, benchmark.Config)); + filteredBenchmarks.Add(new BenchmarkRunInfo(filteredCases.ToArray(), benchmark.Type, benchmark.Config, benchmark.CompositeInProcessDiagnoser)); } } @@ -71,7 +71,8 @@ public void RunBenchmarks(string assemblyPath, TestExecutionRecorderWrapper reco b.Config.AddEventProcessor(eventProcessor) .AddLogger(logger) .RemoveLoggersOfType() // Console logs are also outputted by VSTestLogger. - .CreateImmutableConfig())) + .CreateImmutableConfig(), + b.CompositeInProcessDiagnoser)) .ToArray(); // Run all the benchmarks, and ensure that any tests that don't have a result yet are sent. diff --git a/src/BenchmarkDotNet/Attributes/DisassemblyDiagnoserAttribute.cs b/src/BenchmarkDotNet/Attributes/DisassemblyDiagnoserAttribute.cs index 9985332d76..6650f80dd4 100644 --- a/src/BenchmarkDotNet/Attributes/DisassemblyDiagnoserAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/DisassemblyDiagnoserAttribute.cs @@ -16,6 +16,10 @@ public class DisassemblyDiagnoserAttribute : Attribute, IConfigSource /// Exports all benchmarks to a single HTML report. Makes it easy to compare different runtimes or methods (each becomes a column in HTML table). /// Exports a diff of the assembly code to the Github markdown format. False by default. /// Glob patterns applied to full method signatures by the the disassembler. + /// + /// If , disassembly will be ran in the host process; otherwise it will be ran in the benchmark process. + /// Requires host and benchmark processes to target the same platform (must have the same bit-ness) for CoreCLR. + /// public DisassemblyDiagnoserAttribute( int maxDepth = 1, DisassemblySyntax syntax = DisassemblySyntax.Masm, @@ -25,6 +29,7 @@ public DisassemblyDiagnoserAttribute( bool exportHtml = false, bool exportCombinedDisassemblyReport = false, bool exportDiff = false, + bool runInHost = false, params string[] filters) { Config = ManualConfig.CreateEmpty().AddDiagnoser( @@ -38,7 +43,8 @@ public DisassemblyDiagnoserAttribute( exportGithubMarkdown: exportGithubMarkdown, exportHtml: exportHtml, exportCombinedDisassemblyReport: exportCombinedDisassemblyReport, - exportDiff: exportDiff))); + exportDiff: exportDiff, + runInHost: runInHost))); } // CLS-Compliant Code requires a constructor without an array in the argument list diff --git a/src/BenchmarkDotNet/BenchmarkDotNet.csproj b/src/BenchmarkDotNet/BenchmarkDotNet.csproj index ee31783012..144ea9e47a 100644 --- a/src/BenchmarkDotNet/BenchmarkDotNet.csproj +++ b/src/BenchmarkDotNet/BenchmarkDotNet.csproj @@ -36,18 +36,6 @@ - - - false - - - false - - - - - - @@ -55,7 +43,4 @@ - - - diff --git a/src/BenchmarkDotNet/Code/CodeGenerator.cs b/src/BenchmarkDotNet/Code/CodeGenerator.cs index 9a6228dd88..2800603c3d 100644 --- a/src/BenchmarkDotNet/Code/CodeGenerator.cs +++ b/src/BenchmarkDotNet/Code/CodeGenerator.cs @@ -66,6 +66,7 @@ internal static string Generate(BuildPartition buildPartition) .Replace("$MeasureExtraStats$", buildInfo.Config.HasExtraStatsDiagnoser() ? "true" : "false") .Replace("$DisassemblerEntryMethodName$", DisassemblerConstants.DisassemblerEntryMethodName) .Replace("$WorkloadMethodCall$", provider.GetWorkloadMethodCall(passArguments)) + .Replace("$InProcessDiagnosers$", string.Join($",\n", buildInfo.CompositeInProcessDiagnoser.GetHandlersSourceCode(benchmark))) .RemoveRedundantIfDefines(compilationId); benchmarkTypeCode = Unroll(benchmarkTypeCode, benchmark.Job.ResolveValue(RunMode.UnrollFactorCharacteristic, EnvironmentResolver.Instance)); @@ -277,7 +278,7 @@ private static string GetNativeAotSwitch(BuildPartition buildPartition) @switch.AppendLine("switch (id) {"); foreach (var buildInfo in buildPartition.Benchmarks) - @switch.AppendLine($"case {buildInfo.Id.Value}: BenchmarkDotNet.Autogenerated.Runnable_{buildInfo.Id.Value}.Run(host, benchmarkName); break;"); + @switch.AppendLine($"case {buildInfo.Id.Value}: BenchmarkDotNet.Autogenerated.Runnable_{buildInfo.Id.Value}.Run(host, benchmarkName, diagnoserRunMode); break;"); @switch.AppendLine("default: throw new System.NotSupportedException(\"invalid benchmark id\");"); @switch.AppendLine("}"); diff --git a/src/BenchmarkDotNet/Diagnosers/CompositeDiagnoser.cs b/src/BenchmarkDotNet/Diagnosers/CompositeDiagnoser.cs index 66675f1cb9..32cdb2447e 100644 --- a/src/BenchmarkDotNet/Diagnosers/CompositeDiagnoser.cs +++ b/src/BenchmarkDotNet/Diagnosers/CompositeDiagnoser.cs @@ -1,20 +1,24 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; +using System.ComponentModel; +using System.Diagnostics; using System.Linq; using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; using BenchmarkDotNet.Validators; +using JetBrains.Annotations; namespace BenchmarkDotNet.Diagnosers { - public class CompositeDiagnoser : IDiagnoser + public sealed class CompositeDiagnoser : IDiagnoser { - private readonly ImmutableHashSet diagnosers; + internal readonly ImmutableHashSet diagnosers; public CompositeDiagnoser(ImmutableHashSet diagnosers) => this.diagnosers = diagnosers; @@ -53,4 +57,68 @@ public void DisplayResults(ILogger logger) public IEnumerable Validate(ValidationParameters validationParameters) => diagnosers.SelectMany(diagnoser => diagnoser.Validate(validationParameters)); } + + public sealed class CompositeInProcessDiagnoser(IReadOnlyList inProcessDiagnosers) + { + public const string HeaderKey = "// InProcessDiagnoser"; + public const string ResultsKey = $"{HeaderKey}Results"; + + public IEnumerable GetHandlersSourceCode(BenchmarkCase benchmarkCase) + => inProcessDiagnosers + .Select((d, i) => d.GetHandlerSourceCode(benchmarkCase, i)) + .Where(s => !string.IsNullOrEmpty(s)); + + public IReadOnlyList GetInProcessHandlers(BenchmarkCase benchmarkCase) + => [.. inProcessDiagnosers + .Select((d, i) => d.GetHandler(benchmarkCase, i)) + .WhereNotNull()]; + + public void DeserializeResults(int index, BenchmarkCase benchmarkCase, string results) + => inProcessDiagnosers[index].DeserializeResults(benchmarkCase, results); + } + + [UsedImplicitly] + [EditorBrowsable(EditorBrowsableState.Never)] + public sealed class CompositeInProcessDiagnoserHandler(IReadOnlyList handlers, IHost host, RunMode runMode, InProcessDiagnoserActionArgs parameters) + { + public void Handle(BenchmarkSignal signal) + { + if (runMode == RunMode.None) + { + return; + } + + foreach (var handler in handlers) + { + if (handler.RunMode == runMode) + { + handler.Handle(signal, parameters); + } + } + + if (signal != BenchmarkSignal.AfterEngine) + { + return; + } + + foreach (var handler in handlers) + { + if (handler.RunMode != runMode) + { + continue; + } + + var results = handler.SerializeResults(); + // Send header with the diagnoser index for routing, and line count of payload (user handler may include newlines in their serialized results). + // Ideally we would simply use results.Length, write it directly to host, then the host reads the exact count of chars. + // But WasmExecutor does not use Broker, and reads all output, so we need to instead use line count and prepend every line with CompositeInProcessDiagnoser.ResultsKey. + var resultsLines = results.Split(["\r\n", "\r", "\n"], StringSplitOptions.None); + host.WriteLine($"{CompositeInProcessDiagnoser.HeaderKey} {handler.Index} {resultsLines.Length}"); + foreach (var line in resultsLines) + { + host.WriteLine($"{CompositeInProcessDiagnoser.ResultsKey} {line}"); + } + } + } + } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Diagnosers/IDiagnoser.cs b/src/BenchmarkDotNet/Diagnosers/IDiagnoser.cs index 355dd852e0..93b2477e39 100644 --- a/src/BenchmarkDotNet/Diagnosers/IDiagnoser.cs +++ b/src/BenchmarkDotNet/Diagnosers/IDiagnoser.cs @@ -33,4 +33,54 @@ public interface IConfigurableDiagnoser : IDiagnoser { [PublicAPI] IConfigurableDiagnoser Configure(TConfig config); } + + /// + /// Represents a diagnoser that will be handled in the same process as the benchmarks. + /// + public interface IInProcessDiagnoser : IDiagnoser + { + /// + /// Gets the C# source code used to instantiate the handler in the benchmark process. + /// + /// + /// The source code must be a single expression. + /// + string GetHandlerSourceCode(BenchmarkCase benchmarkCase, int index); + + /// + /// Gets the handler for the same process. + /// + IInProcessDiagnoserHandler GetHandler(BenchmarkCase benchmarkCase, int index); + + /// + /// Deserializes the results of the handler. + /// + void DeserializeResults(BenchmarkCase benchmarkCase, string results); + } + + /// + /// Represents a handler for an . + /// + public interface IInProcessDiagnoserHandler + { + /// + /// The index of the diagnoser. + /// + int Index { get; } + + /// + /// The of the diagnoser for the benchmark. + /// + RunMode RunMode { get; } + + /// + /// Handles the signal from the benchmark. + /// + void Handle(BenchmarkSignal signal, InProcessDiagnoserActionArgs parameters); + + /// + /// Serializes the results to be sent back to the host . + /// + string SerializeResults(); + } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Diagnosers/InProcessDiagnoserActionParameters.cs b/src/BenchmarkDotNet/Diagnosers/InProcessDiagnoserActionParameters.cs new file mode 100644 index 0000000000..3041588f65 --- /dev/null +++ b/src/BenchmarkDotNet/Diagnosers/InProcessDiagnoserActionParameters.cs @@ -0,0 +1,6 @@ +namespace BenchmarkDotNet.Diagnosers; + +public class InProcessDiagnoserActionArgs(object benchmarkInstance) +{ + public object BenchmarkInstance { get; } = benchmarkInstance; +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Diagnosers/RunMode.cs b/src/BenchmarkDotNet/Diagnosers/RunMode.cs index a591f4caaf..ca7448bc27 100644 --- a/src/BenchmarkDotNet/Diagnosers/RunMode.cs +++ b/src/BenchmarkDotNet/Diagnosers/RunMode.cs @@ -1,5 +1,8 @@ -namespace BenchmarkDotNet.Diagnosers +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Diagnosers { + [UsedImplicitly] public enum RunMode : byte { /// diff --git a/src/BenchmarkDotNet/Disassemblers/Arm64Disassembler.cs b/src/BenchmarkDotNet/Disassemblers/Arm64Disassembler.cs index 7c10765b8e..2ead0a6299 100644 --- a/src/BenchmarkDotNet/Disassemblers/Arm64Disassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/Arm64Disassembler.cs @@ -139,7 +139,7 @@ public void Feed(Arm64Instruction instruction) public Arm64RegisterId RegisterId { get { return _registerId; } } } - internal class Arm64Disassembler : ClrMdV3Disassembler + internal class Arm64Disassembler : ClrMdDisassembler { internal sealed class RuntimeSpecificData { @@ -258,7 +258,8 @@ protected override IEnumerable Decode(byte[] code, ulong startAddress, Stat InstructionLength = instruction.Bytes.Length, Instruction = instruction, ReferencedAddress = (address > ushort.MaxValue) ? address : null, - IsReferencedAddressIndirect = isIndirect + IsReferencedAddressIndirect = isIndirect, + DisassembleSyntax = disassembler.DisassembleSyntax }; } } diff --git a/src/BenchmarkDotNet/Disassemblers/ClrMdV3Disassembler.cs b/src/BenchmarkDotNet/Disassemblers/ClrMdDisassembler.cs similarity index 69% rename from src/BenchmarkDotNet/Disassemblers/ClrMdV3Disassembler.cs rename to src/BenchmarkDotNet/Disassemblers/ClrMdDisassembler.cs index 6bd01426d8..ea6591bfcb 100644 --- a/src/BenchmarkDotNet/Disassemblers/ClrMdV3Disassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/ClrMdDisassembler.cs @@ -1,7 +1,6 @@ using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Filters; using Microsoft.Diagnostics.Runtime; -using Microsoft.Diagnostics.Runtime.Utilities; using System; using System.Collections.Generic; using System.IO; @@ -9,11 +8,41 @@ using System.Text.RegularExpressions; using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Portability; +using JetBrains.Annotations; +using System.ComponentModel; +using Microsoft.Diagnostics.NETCore.Client; namespace BenchmarkDotNet.Disassemblers { - // This Disassembler uses ClrMd v3x. Please keep it in sync with ClrMdV1Disassembler (if possible). - internal abstract class ClrMdV3Disassembler + [UsedImplicitly] + [EditorBrowsable(EditorBrowsableState.Never)] + public sealed class ClrMdArgs(int processId, string typeName, string methodName, bool printSource, int maxDepth, string syntax, string tfm, string[] filters, string resultsPath = null) + { + internal int ProcessId { get; } = processId; + internal string TypeName { get; } = typeName; + internal string MethodName { get; } = methodName; + internal bool PrintSource { get; } = printSource; + internal int MaxDepth { get; } = methodName == DisassemblerConstants.DisassemblerEntryMethodName && maxDepth != int.MaxValue ? maxDepth + 1 : maxDepth; + internal string[] Filters { get; } = filters; + internal string Syntax { get; } = syntax; + internal string TargetFrameworkMoniker { get; } = tfm; + internal string ResultsPath { get; } = resultsPath; + + internal static ClrMdArgs FromArgs(string[] args) + => new( + processId: int.Parse(args[0]), + typeName: args[1], + methodName: args[2], + printSource: bool.Parse(args[3]), + maxDepth: int.Parse(args[4]), + resultsPath: args[5], + syntax: args[6], + tfm: args[7], + filters: [.. args.Skip(8)] + ); + } + + internal abstract class ClrMdDisassembler { private static readonly ulong MinValidAddress = GetMinValidAddress(); @@ -30,7 +59,7 @@ private static ulong GetMinValidAddress() { Environments.Platform.X86 or Environments.Platform.X64 => 4096, Environments.Platform.Arm64 => 0x100000000, - _ => throw new NotSupportedException($"{RuntimeInformation.GetCurrentPlatform()} is not supported") + var platform => throw new NotSupportedException($"{platform} is not supported") }; throw new NotSupportedException($"{System.Runtime.InteropServices.RuntimeInformation.OSDescription} is not supported"); } @@ -43,47 +72,88 @@ private static bool IsValidAddress(ulong address) && address != 0 && address >= MinValidAddress; - internal DisassemblyResult AttachAndDisassemble(Settings settings) + private DataTarget Attach(int processId) { - using (var dataTarget = DataTarget.AttachToProcess( - settings.ProcessId, - suspend: false)) + bool isSelf = processId == System.Diagnostics.Process.GetCurrentProcess().Id; + if (OsDetector.IsWindows()) { - var runtime = dataTarget.ClrVersions.Single().CreateRuntime(); - - ConfigureSymbols(dataTarget); - - var state = new State(runtime, settings.TargetFrameworkMoniker); - - if (settings.Filters.Length > 0) + // Windows CoreCLR fails to disassemble generic types when using CreateSnapshotAndAttach, and succeeds with AttachToProcess. https://github.com/microsoft/clrmd/issues/1334 + return isSelf && !RuntimeInformation.IsNetCore + ? DataTarget.CreateSnapshotAndAttach(processId) + : DataTarget.AttachToProcess(processId, suspend: false); + } + if (OsDetector.IsLinux()) + { + // Linux crashes when using AttachToProcess in the same process. + return isSelf + ? DataTarget.CreateSnapshotAndAttach(processId) + : DataTarget.AttachToProcess(processId, suspend: false); + } + if (OsDetector.IsMacOS()) + { + // ClrMD does not support CreateSnapshotAndAttach on MacOS, and AttachToProcess is unreliable, so we have to create a dump file and load it. + string? dumpPath = Path.GetTempFileName(); + try { - FilterAndEnqueue(state, settings); + try + { + new DiagnosticsClient(processId).WriteDump(DumpType.Full, dumpPath, logDumpGeneration: false); + } + catch (ServerErrorException sxe) + { + throw new ArgumentException($"Unable to create a snapshot of process {processId:x}.", sxe); + } + return DataTarget.LoadDump(dumpPath); } - else + finally { - ClrType typeWithBenchmark = state.Runtime.EnumerateModules().Select(module => module.GetTypeByName(settings.TypeName)).First(type => type != null); - - state.Todo.Enqueue( - new MethodInfo( - // the Disassembler Entry Method is always parameterless, so check by name is enough - typeWithBenchmark.Methods.Single(method => method.Attributes.HasFlag(System.Reflection.MethodAttributes.Public) && method.Name == settings.MethodName), - 0)); + if (dumpPath != null) + { + File.Delete(dumpPath); + } } + } + throw new NotSupportedException($"{System.Runtime.InteropServices.RuntimeInformation.OSDescription} is not supported"); + } - var disassembledMethods = Disassemble(settings, state); + internal DisassemblyResult AttachAndDisassemble(ClrMdArgs settings) + { + using var dataTarget = Attach(settings.ProcessId); - // we don't want to export the disassembler entry point method which is just an artificial method added to get generic types working - var filteredMethods = disassembledMethods.Length == 1 - ? disassembledMethods // if there is only one method we want to return it (most probably benchmark got inlined) - : disassembledMethods.Where(method => !method.Name.Contains(DisassemblerConstants.DisassemblerEntryMethodName)).ToArray(); + var runtime = dataTarget.ClrVersions.Single().CreateRuntime(); - return new DisassemblyResult - { - Methods = filteredMethods, - SerializedAddressToNameMapping = state.AddressToNameMapping.Select(x => new DisassemblyResult.MutablePair { Key = x.Key, Value = x.Value }).ToArray(), - PointerSize = (uint)IntPtr.Size - }; + ConfigureSymbols(dataTarget); + + var state = new State(runtime, settings.TargetFrameworkMoniker); + + if (settings.Filters.Length > 0) + { + FilterAndEnqueue(state, settings); } + else + { + ClrType typeWithBenchmark = state.Runtime.EnumerateModules().Select(module => module.GetTypeByName(settings.TypeName)).First(type => type != null); + + state.Todo.Enqueue( + new MethodInfo( + // the Disassembler Entry Method is always parameterless, so check by name is enough + typeWithBenchmark.Methods.Single(method => method.Attributes.HasFlag(System.Reflection.MethodAttributes.Public) && method.Name == settings.MethodName), + 0)); + } + + var disassembledMethods = Disassemble(settings, state); + + // we don't want to export the disassembler entry point method which is just an artificial method added to get generic types working + var filteredMethods = disassembledMethods.Length == 1 + ? disassembledMethods // if there is only one method we want to return it (most probably benchmark got inlined) + : disassembledMethods.Where(method => !method.Name.Contains(DisassemblerConstants.DisassemblerEntryMethodName)).ToArray(); + + return new DisassemblyResult + { + Methods = filteredMethods, + SerializedAddressToNameMapping = state.AddressToNameMapping.Select(x => new DisassemblyResult.MutablePair { Key = x.Key, Value = x.Value }).ToArray(), + PointerSize = (uint) IntPtr.Size + }; } private static void ConfigureSymbols(DataTarget dataTarget) @@ -92,7 +162,7 @@ private static void ConfigureSymbols(DataTarget dataTarget) dataTarget.SetSymbolPath("http://msdl.microsoft.com/download/symbols"); } - private static void FilterAndEnqueue(State state, Settings settings) + private static void FilterAndEnqueue(State state, ClrMdArgs settings) { Regex[] filters = GlobFilter.ToRegex(settings.Filters); @@ -123,7 +193,7 @@ private static void FilterAndEnqueue(State state, Settings settings) } } - private DisassembledMethod[] Disassemble(Settings settings, State state) + private DisassembledMethod[] Disassemble(ClrMdArgs settings, State state) { var result = new List(); DisassemblySyntax syntax = (DisassemblySyntax)Enum.Parse(typeof(DisassemblySyntax), settings.Syntax); @@ -145,7 +215,7 @@ private DisassembledMethod[] Disassemble(Settings settings, State state) private static bool CanBeDisassembled(ClrMethod method) => method.ILOffsetMap.Length > 0 && method.NativeCode > 0; - private DisassembledMethod DisassembleMethod(MethodInfo methodInfo, State state, Settings settings, DisassemblySyntax syntax, SourceCodeProvider sourceCodeProvider) + private DisassembledMethod DisassembleMethod(MethodInfo methodInfo, State state, ClrMdArgs settings, DisassemblySyntax syntax, SourceCodeProvider sourceCodeProvider) { var method = methodInfo.Method; diff --git a/src/BenchmarkDotNet/Disassemblers/DataContracts.cs b/src/BenchmarkDotNet/Disassemblers/DataContracts.cs new file mode 100644 index 0000000000..9c755f4ec0 --- /dev/null +++ b/src/BenchmarkDotNet/Disassemblers/DataContracts.cs @@ -0,0 +1,471 @@ +using Gee.External.Capstone; +using Gee.External.Capstone.Arm64; +using Iced.Intel; +using Microsoft.Diagnostics.Runtime; +using SimpleJson; +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Xml.Serialization; + +#pragma warning disable CS3001 // Argument type 'ulong' is not CLS-compliant +#pragma warning disable CS3003 // Type is not CLS-compliant +#pragma warning disable CS1591 // XML comments for public types... +namespace BenchmarkDotNet.Disassemblers +{ + public abstract class SourceCode + { + // Closed hierarchy. + internal SourceCode() { } + + public ulong InstructionPointer { get; set; } + + internal JsonObject Serialize() + { + var json = new JsonObject { ["$type"] = GetType().Name }; + Serialize(json); + return json; + } + + private protected virtual void Serialize(JsonObject json) + { + json[nameof(InstructionPointer)] = InstructionPointer.ToString(); + } + + internal virtual void Deserialize(JsonObject json) + { + InstructionPointer = ulong.Parse((string) json[nameof(InstructionPointer)]); + } + } + + public sealed class Sharp : SourceCode + { + public string Text { get; set; } + public string FilePath { get; set; } + public int LineNumber { get; set; } + + private protected override void Serialize(JsonObject json) + { + base.Serialize(json); + + json[nameof(Text)] = Text; + json[nameof(FilePath)] = FilePath; + json[nameof(LineNumber)] = LineNumber; + } + + internal override void Deserialize(JsonObject json) + { + base.Deserialize(json); + + Text = (string) json[nameof(Text)]; + FilePath = (string) json[nameof(FilePath)]; + LineNumber = Convert.ToInt32(json[nameof(LineNumber)]); + } + } + + public abstract class Asm : SourceCode + { + // Closed hierarchy. + internal Asm() { } + + public int InstructionLength { get; set; } + public ulong? ReferencedAddress { get; set; } + public bool IsReferencedAddressIndirect { get; set; } + + private protected override void Serialize(JsonObject json) + { + base.Serialize(json); + json[nameof(InstructionLength)] = InstructionLength; + if (ReferencedAddress.HasValue) + { + json[nameof(ReferencedAddress)] = ReferencedAddress.ToString(); + } + json[nameof(IsReferencedAddressIndirect)] = IsReferencedAddressIndirect; + } + + internal override void Deserialize(JsonObject json) + { + base.Deserialize(json); + + InstructionLength = Convert.ToInt32(json[nameof(InstructionLength)]); + if (json.TryGetValue(nameof(ReferencedAddress), out var ra)) + { + ReferencedAddress = ulong.Parse((string) ra); + } + IsReferencedAddressIndirect = (bool) json[nameof(IsReferencedAddressIndirect)]; + } + } + +#if NET6_0_OR_GREATER + // There are way too many properties to serialize them manually. + // Ensure the Instruction's properties are not trimmed. + [method: DynamicDependency(DynamicallyAccessedMemberTypes.PublicProperties, typeof(Instruction))] +#endif + public sealed class IntelAsm() : Asm + { + public Instruction Instruction { get; set; } + + public override string ToString() => Instruction.ToString(); + + private protected override void Serialize(JsonObject json) + { + base.Serialize(json); + + var instructionJson = new JsonObject(); + foreach (var property in typeof(Instruction).GetProperties()) + { + if (property.GetSetMethod() is not null && property.GetGetMethod() is not null) + { + instructionJson[property.Name] = property.GetValue(Instruction) switch + { + ulong l => l.ToString(), + long l => l.ToString(), + Enum e => e.ToString(), + var propertyValue => propertyValue + }; + } + } + json[nameof(Instruction)] = instructionJson; + } + + internal override void Deserialize(JsonObject json) + { + base.Deserialize(json); + + object instruction = new Instruction(); + foreach (var kvp in (JsonObject) json[nameof(Instruction)]) + { + object value = kvp.Value; + var property = typeof(Instruction).GetProperty(kvp.Key); + var propertyType = property.PropertyType; + if (propertyType == typeof(ulong)) + { + value = ulong.Parse((string) value); + } + else if (propertyType == typeof(long)) + { + value = long.Parse((string) value); + } + else if (typeof(Enum).IsAssignableFrom(propertyType)) + { + value = Enum.Parse(propertyType, (string) value); + } + else if (propertyType.IsPrimitive) + { + value = Convert.ChangeType(value, propertyType); + } + property.SetValue(instruction, value); + } + Instruction = (Instruction) instruction; + } + } + + public sealed class Arm64Asm : Asm + { + private const string AddressKey = "Arm64Address"; + private const string BytesKey = "Arm64Bytes"; + private const string SyntaxKey = "Arm64Syntax"; + + public Arm64Instruction Instruction { get; set; } + internal DisassembleSyntax DisassembleSyntax { get; set; } + + public override string ToString() => Instruction.ToString(); + + private protected override void Serialize(JsonObject json) + { + base.Serialize(json); + + // We only need the address, bytes, and syntax to reconstruct the instruction. + if (Instruction?.Bytes?.Length > 0) + { + json[AddressKey] = Instruction.Address.ToString(); + json[BytesKey] = Convert.ToBase64String(Instruction.Bytes); + json[SyntaxKey] = (int) DisassembleSyntax; + } + } + + internal override void Deserialize(JsonObject json) + { + base.Deserialize(json); + + if (json.TryGetValue(BytesKey, out var bytes64)) + { + // Use the Capstone disassembler to recreate the instruction from the bytes. + using var disassembler = CapstoneDisassembler.CreateArm64Disassembler(Arm64DisassembleMode.Arm); + disassembler.EnableInstructionDetails = true; + disassembler.DisassembleSyntax = (DisassembleSyntax) Convert.ToInt32(json[SyntaxKey]); + byte[] bytes = Convert.FromBase64String((string) bytes64); + Instruction = disassembler.Disassemble(bytes, long.Parse((string) json[AddressKey])).Single(); + } + } + } + + public sealed class MonoCode : SourceCode + { + public string Text { get; set; } + + private protected override void Serialize(JsonObject json) + { + base.Serialize(json); + + json[nameof(Text)] = Text; + } + + internal override void Deserialize(JsonObject json) + { + base.Deserialize(json); + + Text = (string) json[nameof(Text)]; + } + } + + public sealed class Map + { + [XmlArray("Instructions")] + [XmlArrayItem(nameof(SourceCode), typeof(SourceCode))] + [XmlArrayItem(nameof(Sharp), typeof(Sharp))] + [XmlArrayItem(nameof(IntelAsm), typeof(IntelAsm))] + public SourceCode[] SourceCodes { get; set; } + + internal JsonObject Serialize() + { + var sourceCodes = new JsonArray(SourceCodes.Length); + foreach (var sourceCode in SourceCodes) + { + sourceCodes.Add(sourceCode.Serialize()); + } + return new JsonObject + { + [nameof(SourceCodes)] = sourceCodes, + }; + } + + internal void Deserialize(JsonObject json) + { + var sourceCodes = (JsonArray) json[nameof(SourceCodes)]; + SourceCodes = new SourceCode[sourceCodes.Count]; + for (int i = 0; i < sourceCodes.Count; i++) + { + var sourceJson = (JsonObject) sourceCodes[i]; + SourceCodes[i] = sourceJson["$type"] switch + { + nameof(Sharp) => new Sharp(), + nameof(IntelAsm) => new IntelAsm(), + nameof(Arm64Asm) => new Arm64Asm(), + nameof(MonoCode) => new MonoCode(), + var unhandledType => throw new NotSupportedException($"Unexpected type: {unhandledType}") + }; + SourceCodes[i].Deserialize(sourceJson); + } + } + } + + public sealed class DisassembledMethod + { + public string Name { get; set; } + + public ulong NativeCode { get; set; } + + public string Problem { get; set; } + + public Map[] Maps { get; set; } = []; + + public string CommandLine { get; set; } + + public static DisassembledMethod Empty(string fullSignature, ulong nativeCode, string problem) + => new() + { + Name = fullSignature, + NativeCode = nativeCode, + Problem = problem + }; + + internal JsonObject Serialize() + { + var maps = new JsonArray(Maps.Length); + foreach (var map in Maps) + { + maps.Add(map.Serialize()); + } + return new JsonObject + { + [nameof(Name)] = Name, + [nameof(NativeCode)] = NativeCode.ToString(), + [nameof(Problem)] = Problem, + [nameof(Maps)] = maps, + [nameof(CommandLine)] = CommandLine + }; + } + + internal void Deserialize(JsonObject json) + { + Name = (string) json[nameof(Name)]; + NativeCode = ulong.Parse((string) json[nameof(NativeCode)]); + Problem = (string) json[nameof(Problem)]; + + var maps = (JsonArray) json[nameof(Maps)]; + Maps = new Map[maps.Count]; + for (int i = 0; i < maps.Count; i++) + { + Maps[i] = new Map(); + Maps[i].Deserialize((JsonObject) maps[i]); + } + + CommandLine = (string) json[nameof(CommandLine)]; + } + } + + public sealed class DisassemblyResult + { + public DisassembledMethod[] Methods { get; set; } = []; + public string[] Errors { get; set; } = []; + public MutablePair[] SerializedAddressToNameMapping { get; set; } = []; + public uint PointerSize { get; set; } + + [XmlIgnore] // XmlSerializer does not support dictionaries ;) + public Dictionary AddressToNameMapping + => _addressToNameMapping ??= SerializedAddressToNameMapping.ToDictionary(x => x.Key, x => x.Value); + + [XmlIgnore] + private Dictionary _addressToNameMapping; + + // KeyValuePair is not serializable, because it has read-only properties + // so we need to define our own... + [Serializable] + [XmlType(TypeName = "Workaround")] + public struct MutablePair + { + public ulong Key { get; set; } + public string Value { get; set; } + } + + internal JsonObject Serialize() + { + var methods = new JsonArray(Methods.Length); + foreach (var method in Methods) + { + methods.Add(method.Serialize()); + } + var errors = new JsonArray(Errors.Length); + foreach (var error in Errors) + { + errors.Add(error); + } + var addressToNameMapping = new JsonObject(); + foreach (var kvp in SerializedAddressToNameMapping) + { + addressToNameMapping[kvp.Key.ToString()] = kvp.Value; + } + return new JsonObject + { + [nameof(Methods)] = methods, + [nameof(Errors)] = errors, + [nameof(AddressToNameMapping)] = addressToNameMapping, + [nameof(PointerSize)] = PointerSize.ToString() + }; + } + + internal void Deserialize(JsonObject json) + { + var methods = (JsonArray) json[nameof(Methods)]; + Methods = new DisassembledMethod[methods.Count]; + for (int i = 0; i < methods.Count; i++) + { + Methods[i] = new DisassembledMethod(); + Methods[i].Deserialize((JsonObject) methods[i]); + } + + var errors = (JsonArray) json[nameof(Errors)]; + Errors = new string[errors.Count]; + for (int i = 0; i < errors.Count; i++) + { + Errors[i] = (string) errors[i]; + } + + var addressToNameMapping = (JsonObject) json[nameof(AddressToNameMapping)]; + var serializedAddressToNameMapping = new MutablePair[addressToNameMapping.Count]; + int addressIndex = 0; + foreach (var kvp in addressToNameMapping) + { + serializedAddressToNameMapping[addressIndex].Key = ulong.Parse(kvp.Key); + serializedAddressToNameMapping[addressIndex].Value = (string) kvp.Value; + ++addressIndex; + } + + PointerSize = uint.Parse((string) json[nameof(PointerSize)]); + } + } + + public static class DisassemblerConstants + { + public const string DisassemblerEntryMethodName = "__ForDisassemblyDiagnoser__"; + } + + internal sealed class State + { + internal State(ClrRuntime runtime, string targetFrameworkMoniker) + { + Runtime = runtime; + Todo = new Queue(); + HandledMethods = new HashSet(new ClrMethodComparer()); + AddressToNameMapping = new Dictionary(); + RuntimeVersion = ParseVersion(targetFrameworkMoniker); + } + + internal ClrRuntime Runtime { get; } + internal string TargetFrameworkMoniker { get; } + internal Queue Todo { get; } + internal HashSet HandledMethods { get; } + internal Dictionary AddressToNameMapping { get; } + internal Version RuntimeVersion { get; } + + internal static Version ParseVersion(string targetFrameworkMoniker) + { + int firstDigit = -1, lastDigit = -1; + for (int i = 0; i < targetFrameworkMoniker.Length; i++) + { + if (char.IsDigit(targetFrameworkMoniker[i])) + { + if (firstDigit == -1) + firstDigit = i; + + lastDigit = i; + } + else if (targetFrameworkMoniker[i] == '-') + { + break; // it can be platform specific like net7.0-windows8 + } + } + + string versionToParse = targetFrameworkMoniker.Substring(firstDigit, lastDigit - firstDigit + 1); + if (!versionToParse.Contains(".")) // Full .NET Framework (net48 etc) + versionToParse = string.Join(".", versionToParse.ToCharArray()); + + return Version.Parse(versionToParse); + } + + private sealed class ClrMethodComparer : IEqualityComparer + { + public bool Equals(ClrMethod x, ClrMethod y) => x.NativeCode == y.NativeCode; + + public int GetHashCode(ClrMethod obj) => (int) obj.NativeCode; + } + } + + internal readonly struct MethodInfo // I am not using ValueTuple here (would be perfect) to keep the number of dependencies as low as possible + { + internal ClrMethod Method { get; } + internal int Depth { get; } + + internal MethodInfo(ClrMethod method, int depth) + { + Method = method; + Depth = depth; + } + } +} +#pragma warning restore CS1591 // XML comments for public types... +#pragma warning restore CS3003 // Type is not CLS-compliant +#pragma warning restore CS3001 // Argument type 'ulong' is not CLS-compliant \ No newline at end of file diff --git a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs index 7d0f95326c..8b8a366bae 100644 --- a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs +++ b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; using System.Linq; using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Columns; @@ -9,6 +11,7 @@ using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; @@ -17,27 +20,36 @@ using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.InProcess.NoEmit; using BenchmarkDotNet.Validators; +using JetBrains.Annotations; using Perfolizer.Metrology; +using SimpleJson; namespace BenchmarkDotNet.Diagnosers { - public class DisassemblyDiagnoser : IDiagnoser + public class DisassemblyDiagnoser : IInProcessDiagnoser { - private static readonly Lazy ptrace_scope = new Lazy(() => ProcessHelper.RunAndReadOutput("cat", "/proc/sys/kernel/yama/ptrace_scope").Trim()); + private static readonly Lazy ptrace_scope = new(() => ProcessHelper.RunAndReadOutput("cat", "/proc/sys/kernel/yama/ptrace_scope").Trim()); - private readonly WindowsDisassembler windowsDifferentArchitectureDisassembler; - private readonly SameArchitectureDisassembler sameArchitectureDisassembler; - private readonly MonoDisassembler monoDisassembler; - private readonly Dictionary results; + private ClrMdDisassembler? _clrMdDisassembler; + + // Lazy create to avoid exceptions at Disassembler ctor + private ClrMdDisassembler ClrMdDisassembler => _clrMdDisassembler ??= GetClrMdDisassembler(); + + internal static ClrMdDisassembler GetClrMdDisassembler() => + RuntimeInformation.GetCurrentPlatform() switch + { + Platform.X86 or Platform.X64 => new IntelDisassembler(), + Platform.Arm64 => new Arm64Disassembler(), + var platform => throw new NotSupportedException($"{platform} is not supported") + }; + + private readonly MonoDisassembler monoDisassembler = new(); + private readonly Dictionary results = []; public DisassemblyDiagnoser(DisassemblyDiagnoserConfig config) { Config = config; - windowsDifferentArchitectureDisassembler = new WindowsDisassembler(config); - sameArchitectureDisassembler = new SameArchitectureDisassembler(config); - monoDisassembler = new MonoDisassembler(config); - results = new Dictionary(); Exporters = GetExporters(results, config); } @@ -45,11 +57,11 @@ public DisassemblyDiagnoser(DisassemblyDiagnoserConfig config) public IReadOnlyDictionary Results => results; - public IEnumerable Ids => new[] { nameof(DisassemblyDiagnoser) }; + public IEnumerable Ids => [nameof(DisassemblyDiagnoser)]; public IEnumerable Exporters { get; } - public IEnumerable Analysers => new IAnalyser[] { new DisassemblyAnalyzer(results) }; + public IEnumerable Analysers => [new DisassemblyAnalyzer(results)]; public IEnumerable ProcessResults(DiagnoserResults diagnoserResults) { @@ -67,17 +79,29 @@ public RunMode GetRunMode(BenchmarkCase benchmarkCase) return RunMode.None; } + private ClrMdArgs BuildDisassemblerSettings(BenchmarkCase benchmarkCase, string typeName, int processId) + => new( + processId: processId, + typeName: typeName, + methodName: DisassemblerConstants.DisassemblerEntryMethodName, + printSource: Config.PrintSource, + maxDepth: Config.MaxDepth, + filters: Config.Filters, + syntax: Config.Syntax.ToString(), + tfm: benchmarkCase.Job.Environment.GetRuntime().MsBuildMoniker + ); + public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { var benchmark = parameters.BenchmarkCase; + bool isInProcess = parameters.BenchmarkCase.Job.Infrastructure.TryGetToolchain(out var toolchain) && toolchain.IsInProcess; switch (signal) { - case HostSignal.AfterAll when ShouldUseSameArchitectureDisassembler(benchmark, parameters): - results.Add(benchmark, sameArchitectureDisassembler.Disassemble(parameters)); - break; - case HostSignal.AfterAll when OsDetector.IsWindows() && !ShouldUseMonoDisassembler(benchmark): - results.Add(benchmark, windowsDifferentArchitectureDisassembler.Disassemble(parameters)); + case HostSignal.AfterAll when (Config.RunInHost || isInProcess) && ShouldUseClrMdDisassembler(benchmark): + results.Add(benchmark, ClrMdDisassembler.AttachAndDisassemble( + BuildDisassemblerSettings(parameters.BenchmarkCase, $"BenchmarkDotNet.Autogenerated.Runnable_{parameters.BenchmarkId.Value}", parameters.Process.Id)) + ); break; case HostSignal.SeparateLogic when ShouldUseMonoDisassembler(benchmark): results.Add(benchmark, monoDisassembler.Disassemble(benchmark, benchmark.Job.Environment.Runtime as MonoRuntime)); @@ -87,7 +111,7 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters) public void DisplayResults(ILogger logger) => logger.WriteInfo( - results.Any() + results.Count > 0 ? "Disassembled benchmarks got exported to \".\\BenchmarkDotNet.Artifacts\\results\\*asm.md\"" : "No benchmarks were disassembled"); @@ -100,6 +124,12 @@ public IEnumerable Validate(ValidationParameters validationPara yield break; } + if (Config.RunInHost && OsDetector.IsMacOS()) + { + yield return new ValidationError(true, "Disassembling in the host process is not supported on MacOS"); + yield break; + } + foreach (var benchmark in validationParameters.Benchmarks) { if (benchmark.Job.Infrastructure.TryGetToolchain(out var toolchain) && toolchain is InProcessNoEmitToolchain) @@ -113,6 +143,11 @@ public IEnumerable Validate(ValidationParameters validationPara if (ShouldUseClrMdDisassembler(benchmark)) { + if (Config.RunInHost && toolchain?.IsInProcess != true && !PlatformsMatch(currentPlatform, benchmark.Job.Environment.Platform)) + { + yield return new ValidationError(true, "DisassemblyDiagnoser cannot run in host for a job that targets a different platform", benchmark); + } + if (OsDetector.IsLinux()) { var runtime = benchmark.Job.ResolveValue(EnvironmentMode.RuntimeCharacteristic, EnvironmentResolver.Instance); @@ -139,31 +174,25 @@ public IEnumerable Validate(ValidationParameters validationPara } } - private static bool ShouldUseMonoDisassembler(BenchmarkCase benchmarkCase) - => benchmarkCase.Job.Environment.Runtime is MonoRuntime || RuntimeInformation.IsMono; - - // when we add macOS support, RuntimeInformation.IsMacOS() needs to be added here - private static bool ShouldUseClrMdDisassembler(BenchmarkCase benchmarkCase) - => !ShouldUseMonoDisassembler(benchmarkCase) && (OsDetector.IsWindows() || OsDetector.IsLinux()); - - private static bool ShouldUseSameArchitectureDisassembler(BenchmarkCase benchmarkCase, DiagnoserActionParameters parameters) + private static bool PlatformsMatch(Platform currentPlatform, Platform targetPlatform) { - if (ShouldUseClrMdDisassembler(benchmarkCase)) - { - if (OsDetector.IsWindows()) - { - return WindowsDisassembler.GetDisassemblerArchitecture(parameters.Process, benchmarkCase.Job.Environment.Platform) - == RuntimeInformation.GetCurrentPlatform(); - } - - // on Unix currently host process architecture is always the same as benchmark process architecture - // (no official x86 support) - return true; - } + Debug.Assert(currentPlatform != Platform.AnyCpu); - return false; + return targetPlatform == Platform.AnyCpu + // AnyCpu compiles to the bit-ness of the operating system. + // Legacy Framework also supports in Visual Studio, + // but we don't need to bother checking for it since we use dotnet sdk to build and we don't use or copy that property in generated csproj. + ? Environment.Is64BitProcess == Environment.Is64BitOperatingSystem + : currentPlatform == targetPlatform; } + private static bool ShouldUseMonoDisassembler(BenchmarkCase benchmarkCase) + => benchmarkCase.Job.Environment.Runtime is MonoRuntime + || (RuntimeInformation.IsMono && benchmarkCase.Job.Infrastructure.TryGetToolchain(out var toolchain) && toolchain.IsInProcess); + + private static bool ShouldUseClrMdDisassembler(BenchmarkCase benchmarkCase) + => !ShouldUseMonoDisassembler(benchmarkCase) && (OsDetector.IsWindows() || OsDetector.IsLinux() || OsDetector.IsMacOS()); + private static IEnumerable GetExporters(Dictionary results, DisassemblyDiagnoserConfig config) { if (config.ExportGithubMarkdown) @@ -187,6 +216,49 @@ private static IEnumerable GetExporters(Dictionary disassembly.Methods.Sum(method => method.Maps.Sum(map => map.SourceCodes.OfType().Sum(asm => asm.InstructionLength))); + string IInProcessDiagnoser.GetHandlerSourceCode(BenchmarkCase benchmarkCase, int index) + { + // Mono disassembler always runs another process. + if (Config.RunInHost || ShouldUseMonoDisassembler(benchmarkCase)) + { + return null; + } + var runMode = GetRunMode(benchmarkCase); + if (runMode == RunMode.None) + { + return null; + } + + var clrMdArgs = BuildDisassemblerSettings(benchmarkCase, null, 0); + return $$""" + new {{typeof(DisassemblyDiagnoserInProcessHandler).GetCorrectCSharpTypeName()}}() { + {{nameof(DisassemblyDiagnoserInProcessHandler.Index)}} = {{index}}, + {{nameof(DisassemblyDiagnoserInProcessHandler.RunMode)}} = {{SourceCodeHelper.ToSourceCode(runMode)}}, + {{nameof(DisassemblyDiagnoserInProcessHandler.ClrMdArgs)}} = new {{typeof(ClrMdArgs).GetCorrectCSharpTypeName()}}( + {{typeof(Process).GetCorrectCSharpTypeName()}}.{{nameof(Process.GetCurrentProcess)}}().{{nameof(Process.Id)}}, + instance.GetType().FullName, + {{SourceCodeHelper.ToSourceCode(clrMdArgs.MethodName)}}, + {{SourceCodeHelper.ToSourceCode(clrMdArgs.PrintSource)}}, + {{clrMdArgs.MaxDepth}}, + {{SourceCodeHelper.ToSourceCode(clrMdArgs.Syntax)}}, + {{SourceCodeHelper.ToSourceCode(clrMdArgs.TargetFrameworkMoniker)}}, + {{SourceCodeHelper.ToSourceCode(clrMdArgs.Filters)}} + ) + } + """; + } + + // We don't use handler for InProcess toolchains, the host diagnoser already handles it without needing to serialize data. + IInProcessDiagnoserHandler IInProcessDiagnoser.GetHandler(BenchmarkCase benchmarkCase, int index) => null; + + void IInProcessDiagnoser.DeserializeResults(BenchmarkCase benchmarkCase, string results) + { + var json = SimpleJsonSerializer.DeserializeObject(results); + var result = new DisassemblyResult(); + result.Deserialize(json); + this.results.Add(benchmarkCase, result); + } + private class NativeCodeSizeMetricDescriptor : IMetricDescriptor { internal static readonly IMetricDescriptor Instance = new NativeCodeSizeMetricDescriptor(); @@ -202,4 +274,29 @@ private class NativeCodeSizeMetricDescriptor : IMetricDescriptor public bool GetIsAvailable(Metric metric) => true; } } + + [UsedImplicitly] + [EditorBrowsable(EditorBrowsableState.Never)] + public sealed class DisassemblyDiagnoserInProcessHandler : IInProcessDiagnoserHandler + { + private DisassemblyResult _result; + + public int Index { get; set; } + public RunMode RunMode { get; set; } + public ClrMdArgs ClrMdArgs { get; set; } + + void IInProcessDiagnoserHandler.Handle(BenchmarkSignal signal, InProcessDiagnoserActionArgs parameters) + { + if (signal == BenchmarkSignal.AfterEngine) + { + _result = DisassemblyDiagnoser.GetClrMdDisassembler().AttachAndDisassemble(ClrMdArgs); + } + } + + string IInProcessDiagnoserHandler.SerializeResults() + { + SimpleJsonSerializer.CurrentJsonSerializerStrategy.Indent = false; + return _result.Serialize().ToString(); + } + } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoserConfig.cs b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoserConfig.cs index feff76ef99..fd1d8e78b3 100644 --- a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoserConfig.cs +++ b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoserConfig.cs @@ -18,6 +18,10 @@ public class DisassemblyDiagnoserConfig /// Exports to HTML with clickable links. False by default. /// Exports all benchmarks to a single HTML report. Makes it easy to compare different runtimes or methods (each becomes a column in HTML table). /// Exports a diff of the assembly code to the Github markdown format. False by default. + /// + /// If , disassembly will be ran in the host process; otherwise it will be ran in the benchmark process. + /// Requires host and benchmark processes to target the same platform (must have the same bit-ness) for CoreCLR. + /// [PublicAPI] public DisassemblyDiagnoserConfig( int maxDepth = 1, @@ -29,7 +33,8 @@ public DisassemblyDiagnoserConfig( bool exportGithubMarkdown = true, bool exportHtml = false, bool exportCombinedDisassemblyReport = false, - bool exportDiff = false) + bool exportDiff = false, + bool runInHost = false) { if (!(syntax is DisassemblySyntax.Masm or DisassemblySyntax.Intel or DisassemblySyntax.Att)) { @@ -46,6 +51,7 @@ public DisassemblyDiagnoserConfig( ExportHtml = exportHtml; ExportCombinedDisassemblyReport = exportCombinedDisassemblyReport; ExportDiff = exportDiff; + RunInHost = runInHost; } public bool PrintSource { get; } @@ -58,6 +64,7 @@ public DisassemblyDiagnoserConfig( public bool ExportHtml { get; } public bool ExportCombinedDisassemblyReport { get; } public bool ExportDiff { get; } + public bool RunInHost { get; } // user can specify a formatter without symbol solver // so we need to clone the formatter with settings and provide our symbols solver diff --git a/src/BenchmarkDotNet/Disassemblers/IntelDisassembler.cs b/src/BenchmarkDotNet/Disassemblers/IntelDisassembler.cs index eb0d5ce3f6..160b86a81d 100644 --- a/src/BenchmarkDotNet/Disassemblers/IntelDisassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/IntelDisassembler.cs @@ -7,7 +7,7 @@ namespace BenchmarkDotNet.Disassemblers { - internal class IntelDisassembler : ClrMdV3Disassembler + internal class IntelDisassembler : ClrMdDisassembler { internal sealed class RuntimeSpecificData { diff --git a/src/BenchmarkDotNet/Disassemblers/MonoDisassembler.cs b/src/BenchmarkDotNet/Disassemblers/MonoDisassembler.cs index de0e35b6dd..4bff7cd4bc 100644 --- a/src/BenchmarkDotNet/Disassemblers/MonoDisassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/MonoDisassembler.cs @@ -4,7 +4,6 @@ using System.Linq; using System.Reflection; using System.Text.RegularExpressions; -using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; @@ -13,10 +12,8 @@ namespace BenchmarkDotNet.Disassemblers { - internal class MonoDisassembler + internal sealed class MonoDisassembler { - internal MonoDisassembler(DisassemblyDiagnoserConfig _) { } - internal DisassemblyResult Disassemble(BenchmarkCase benchmarkCase, MonoRuntime mono) { Debug.Assert(mono == null || !RuntimeInformation.IsMono, "Must never be called for Non-Mono benchmarks"); diff --git a/src/BenchmarkDotNet/Disassemblers/SameArchitectureDisassembler.cs b/src/BenchmarkDotNet/Disassemblers/SameArchitectureDisassembler.cs deleted file mode 100644 index 1a4835484d..0000000000 --- a/src/BenchmarkDotNet/Disassemblers/SameArchitectureDisassembler.cs +++ /dev/null @@ -1,41 +0,0 @@ -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Portability; -using System; - -namespace BenchmarkDotNet.Disassemblers -{ - internal class SameArchitectureDisassembler - { - private readonly DisassemblyDiagnoserConfig config; - private ClrMdV3Disassembler? clrMdV3Disassembler; - - internal SameArchitectureDisassembler(DisassemblyDiagnoserConfig config) => this.config = config; - - internal DisassemblyResult Disassemble(DiagnoserActionParameters parameters) - // delay the creation to avoid exceptions at DisassemblyDiagnoser ctor - => (clrMdV3Disassembler ??= CreateDisassemblerForCurrentArchitecture()) - .AttachAndDisassemble(BuildDisassemblerSettings(parameters)); - - private static ClrMdV3Disassembler CreateDisassemblerForCurrentArchitecture() - => RuntimeInformation.GetCurrentPlatform() switch - { - Platform.X86 or Platform.X64 => new IntelDisassembler(), - Platform.Arm64 => new Arm64Disassembler(), - _ => throw new NotSupportedException($"{RuntimeInformation.GetCurrentPlatform()} is not supported") - }; - - private Settings BuildDisassemblerSettings(DiagnoserActionParameters parameters) - => new( - processId: parameters.Process.Id, - typeName: $"BenchmarkDotNet.Autogenerated.Runnable_{parameters.BenchmarkId.Value}", - methodName: DisassemblerConstants.DisassemblerEntryMethodName, - printSource: config.PrintSource, - maxDepth: config.MaxDepth, - filters: config.Filters, - syntax: config.Syntax.ToString(), - tfm: parameters.BenchmarkCase.Job.Environment.GetRuntime().MsBuildMoniker, - resultsPath: default - ); - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Disassemblers/WindowsDisassembler.cs b/src/BenchmarkDotNet/Disassemblers/WindowsDisassembler.cs deleted file mode 100644 index 8051ba3463..0000000000 --- a/src/BenchmarkDotNet/Disassemblers/WindowsDisassembler.cs +++ /dev/null @@ -1,184 +0,0 @@ -using System; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Reflection; -using System.Runtime.InteropServices; -using System.Text; -using System.Xml; -using System.Xml.Serialization; -using BenchmarkDotNet.Detectors; -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Extensions; -using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Properties; -using JetBrains.Annotations; -using RuntimeInformation = BenchmarkDotNet.Portability.RuntimeInformation; - -namespace BenchmarkDotNet.Disassemblers -{ - [PublicAPI] - public class WindowsDisassembler - { - private readonly DisassemblyDiagnoserConfig config; - - [PublicAPI] - public WindowsDisassembler(DisassemblyDiagnoserConfig config) => this.config = config; - - [PublicAPI] - public DisassemblyResult Disassemble(DiagnoserActionParameters parameters) - { - string resultsPath = Path.GetTempFileName(); - - string disassemblerPath = GetDisassemblerPath(parameters.Process, parameters.BenchmarkCase.Job.Environment.Platform); - string arguments = BuildArguments(parameters, resultsPath); - string errors = ProcessHelper.RunAndReadOutput(disassemblerPath, arguments); - - if (!string.IsNullOrEmpty(errors)) - { - parameters.Config.GetCompositeLogger().WriteError(errors); - return new DisassemblyResult { Errors = new[] { errors } }; - } - - try - { - using (var stream = new FileStream(resultsPath, FileMode.Open, FileAccess.Read)) - using (var reader = XmlReader.Create(stream)) - { - var serializer = new XmlSerializer(typeof(DisassemblyResult)); - - return (DisassemblyResult)serializer.Deserialize(reader); - } - } - catch (Exception e) - { - throw new Exception($"Can't read disassembly diagnostic file (DisassemblerPath = '{disassemblerPath}', Arguments = '{arguments}')", e); - } - finally - { - File.Delete(resultsPath); - } - } - - internal static Platform GetDisassemblerArchitecture(Process process, Platform platform) - => platform switch - { - Platform.AnyCpu when System.Runtime.InteropServices.RuntimeInformation.OSArchitecture is Architecture.Arm or Architecture.Arm64 => RuntimeInformation.GetCurrentPlatform(), - Platform.AnyCpu => NativeMethods.Is64Bit(process) ? Platform.X64 : Platform.X86, - _ => platform - }; - - private static string GetDisassemblerPath(Process process, Platform platform) - => GetDisassemblerArchitecture(process, platform) switch - { - Platform.X86 => GetDisassemblerPath("x86"), - Platform.X64 => GetDisassemblerPath("x64"), - _ => throw new NotSupportedException($"Platform {platform} not supported!") - }; - - private static string GetDisassemblerPath(string architectureName) - { - // one can only attach to a process of same target architecture, this is why we need exe for x64 and for x86 - string exeName = $"BenchmarkDotNet.Disassembler.{architectureName}.exe"; - var assemblyWithDisassemblersInResources = typeof(WindowsDisassembler).GetTypeInfo().Assembly; - - var dir = new FileInfo(assemblyWithDisassemblersInResources.Location).Directory ?? throw new DirectoryNotFoundException(); - string disassemblerPath = Path.Combine( - dir.FullName, - FolderNameHelper.ToFolderName(BenchmarkDotNetInfo.Instance.FullVersion), // possible update - exeName); // separate process per architecture!! - - Path.GetDirectoryName(disassemblerPath).CreateIfNotExists(); - - // for development we always want to copy the file to not omit any dev changes - if (!BenchmarkDotNetInfo.Instance.IsDevelop) - { - if (File.Exists(disassemblerPath)) - return disassemblerPath; - } - - // the disassembler has not been yet retrieved from the resources - CopyFromResources( - assemblyWithDisassemblersInResources, - $"BenchmarkDotNet.Disassemblers.net462.win7_{architectureName}.{exeName}", - disassemblerPath); - - CopyAllRequiredDependencies(assemblyWithDisassemblersInResources, Path.GetDirectoryName(disassemblerPath)); - - return disassemblerPath; - } - - private static void CopyAllRequiredDependencies(Assembly assemblyWithDisassemblersInResources, string destinationFolder) - { - // ClrMD and Iced are also embedded in the resources, we need to copy them as well - foreach (string dependency in assemblyWithDisassemblersInResources.GetManifestResourceNames().Where(name => name.EndsWith(".dll"))) - { - // dependency is sth like "BenchmarkDotNet.Disassemblers.net462.win7_x64.Microsoft.Diagnostics.Runtime.dll" - string fileName = dependency.Replace("BenchmarkDotNet.Disassemblers.net462.win7_x64.", string.Empty); - string dllPath = Path.Combine(destinationFolder, fileName); - - if (!File.Exists(dllPath)) - CopyFromResources( - assemblyWithDisassemblersInResources, - dependency, - dllPath); - } - } - - private static void CopyFromResources(Assembly assembly, string resourceName, string destinationPath) - { - using (var resourceStream = assembly.GetManifestResourceStream(resourceName)) - using (var exeStream = File.Create(destinationPath)) - { - if (resourceStream == null) - throw new InvalidOperationException($"{nameof(resourceName)} is null"); - resourceStream.CopyTo(exeStream); - } - } - - // if the benchmark requires jitting we use disassembler entry method, if not we use benchmark method name - private string BuildArguments(DiagnoserActionParameters parameters, string resultsPath) - => new StringBuilder(200) - .Append(parameters.Process.Id).Append(' ') - .Append("BenchmarkDotNet.Autogenerated.Runnable_").Append(parameters.BenchmarkId.Value).Append(' ') - .Append(DisassemblerConstants.DisassemblerEntryMethodName).Append(' ') - .Append(config.PrintSource).Append(' ') - .Append(config.MaxDepth).Append(' ') - .Append(Escape(resultsPath)) - .Append(' ') - .Append(config.Syntax.ToString()) - .Append(' ') - .Append(parameters.BenchmarkCase.Job.Environment.GetRuntime().MsBuildMoniker) - .Append(' ') - .Append(string.Join(" ", config.Filters.Select(Escape))) - .ToString(); - - private static string Escape(string value) => $"\"{value}\""; - - // code copied from https://stackoverflow.com/a/33206186/5852046 - private static class NativeMethods - { - // see https://msdn.microsoft.com/en-us/library/windows/desktop/ms684139%28v=vs.85%29.aspx - public static bool Is64Bit(Process process) - { - if (Environment.GetEnvironmentVariable("PROCESSOR_ARCHITECTURE") == "x86") - return false; - - if (OsDetector.IsWindows()) - { - IsWow64Process(process.Handle, out bool isWow64); - - return !isWow64; - } - - return RuntimeInformation.Is64BitPlatform(); // todo: find the way to cover all scenarios for .NET Core - } - - [DllImport("kernel32.dll", SetLastError = true, CallingConvention = CallingConvention.Winapi)] - [return: MarshalAs(UnmanagedType.Bool)] - private static extern bool IsWow64Process([In] IntPtr process, [Out] out bool wow64Process); - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/Engine.cs b/src/BenchmarkDotNet/Engines/Engine.cs index 1fc1b96dfd..8b655567aa 100644 --- a/src/BenchmarkDotNet/Engines/Engine.cs +++ b/src/BenchmarkDotNet/Engines/Engine.cs @@ -45,13 +45,14 @@ public class Engine : IEngine private readonly List jittingMeasurements = new(10); private readonly bool includeExtraStats; private readonly Random random; + private readonly Diagnosers.CompositeInProcessDiagnoserHandler inProcessDiagnoserHandler; internal Engine( IHost host, IResolver resolver, Action dummy1Action, Action dummy2Action, Action dummy3Action, Action overheadAction, Action workloadAction, Job targetJob, Action globalSetupAction, Action globalCleanupAction, Action iterationSetupAction, Action iterationCleanupAction, long operationsPerInvoke, - bool includeExtraStats, string benchmarkName) + bool includeExtraStats, string benchmarkName, Diagnosers.CompositeInProcessDiagnoserHandler inProcessDiagnoserHandler) { Host = host; @@ -68,6 +69,7 @@ internal Engine( OperationsPerInvoke = operationsPerInvoke; this.includeExtraStats = includeExtraStats; BenchmarkName = benchmarkName; + this.inProcessDiagnoserHandler = inProcessDiagnoserHandler; Resolver = resolver; @@ -117,6 +119,7 @@ public RunResults Run() if (stage.Stage == IterationStage.Actual && stage.Mode == IterationMode.Workload) { Host.BeforeMainRun(); + inProcessDiagnoserHandler.Handle(BenchmarkSignal.BeforeActualRun); } var stageMeasurements = stage.GetMeasurementList(); @@ -135,6 +138,7 @@ public RunResults Run() if (stage.Stage == IterationStage.Actual && stage.Mode == IterationMode.Workload) { Host.AfterMainRun(); + inProcessDiagnoserHandler.Handle(BenchmarkSignal.AfterActualRun); } } diff --git a/src/BenchmarkDotNet/Engines/EngineFactory.cs b/src/BenchmarkDotNet/Engines/EngineFactory.cs index 0588218522..1a4e1cb635 100644 --- a/src/BenchmarkDotNet/Engines/EngineFactory.cs +++ b/src/BenchmarkDotNet/Engines/EngineFactory.cs @@ -125,6 +125,7 @@ private static Engine CreateEngine(EngineParameters engineParameters, Job job, A engineParameters.IterationCleanupAction, engineParameters.OperationsPerInvoke, engineParameters.MeasureExtraStats, - engineParameters.BenchmarkName); + engineParameters.BenchmarkName, + engineParameters.InProcessDiagnoserHandler); } } diff --git a/src/BenchmarkDotNet/Engines/EngineParameters.cs b/src/BenchmarkDotNet/Engines/EngineParameters.cs index ec61582529..ad1b8cd127 100644 --- a/src/BenchmarkDotNet/Engines/EngineParameters.cs +++ b/src/BenchmarkDotNet/Engines/EngineParameters.cs @@ -28,6 +28,7 @@ public class EngineParameters public bool MeasureExtraStats { get; set; } [PublicAPI] public string BenchmarkName { get; set; } + public Diagnosers.CompositeInProcessDiagnoserHandler InProcessDiagnoserHandler { get; set; } public bool NeedsJitting => TargetJob.ResolveValue(RunMode.RunStrategyCharacteristic, DefaultResolver).NeedsJitting(); diff --git a/src/BenchmarkDotNet/Engines/InProcessSignal.cs b/src/BenchmarkDotNet/Engines/InProcessSignal.cs new file mode 100644 index 0000000000..f5630381bf --- /dev/null +++ b/src/BenchmarkDotNet/Engines/InProcessSignal.cs @@ -0,0 +1,28 @@ +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Engines +{ + [UsedImplicitly] + public enum BenchmarkSignal + { + /// + /// before the engine is created + /// + BeforeEngine, + + /// + /// after globalSetup, warmup and pilot but before the main run + /// + BeforeActualRun, + + /// + /// after main run, but before global Cleanup + /// + AfterActualRun, + + /// + /// after the engine has completed the run + /// + AfterEngine, + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Helpers/SourceCodeHelper.cs b/src/BenchmarkDotNet/Helpers/SourceCodeHelper.cs index 3883de7546..524878b2c1 100644 --- a/src/BenchmarkDotNet/Helpers/SourceCodeHelper.cs +++ b/src/BenchmarkDotNet/Helpers/SourceCodeHelper.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Numerics; @@ -33,6 +34,16 @@ public static string ToSourceCode(object value) return $"System.DateTime.Parse(\"{dateTime.ToString(CultureInfo.InvariantCulture)}\", System.Globalization.CultureInfo.InvariantCulture)"; case Guid guid: return $"System.Guid.Parse(\"{guid.ToString()}\")"; + // Multi-dimensional arrays are more complex, we only need single-dimension support for now. + case Array { Rank: 1 } array: + { + var elementsSourceCode = new string[array.Length]; + for (int i = 0; i < array.Length; ++i) + { + elementsSourceCode[i] = ToSourceCode(array.GetValue(i)); + } + return $"new {array.GetType().GetElementType().GetCorrectCSharpTypeName()}[] {{ {string.Join(", ", elementsSourceCode)} }}"; + } } if (ReflectionUtils.GetTypeInfo(value.GetType()).IsEnum) return $"({value.GetType().GetCorrectCSharpTypeName()})({ToInvariantCultureString(value)})"; diff --git a/src/BenchmarkDotNet/Loggers/Broker.cs b/src/BenchmarkDotNet/Loggers/Broker.cs index 277cd3c430..9af2e9797b 100644 --- a/src/BenchmarkDotNet/Loggers/Broker.cs +++ b/src/BenchmarkDotNet/Loggers/Broker.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.IO; using System.IO.Pipes; +using System.Text; using System.Threading; using System.Threading.Tasks; using BenchmarkDotNet.Diagnosers; @@ -15,34 +16,33 @@ internal class Broker { private readonly ILogger logger; private readonly Process process; + private readonly CompositeInProcessDiagnoser compositeInProcessDiagnoser; private readonly AnonymousPipeServerStream inputFromBenchmark, acknowledgments; private readonly ManualResetEvent finished; - public Broker(ILogger logger, Process process, IDiagnoser diagnoser, + public Broker(ILogger logger, Process process, IDiagnoser? diagnoser, CompositeInProcessDiagnoser compositeInProcessDiagnoser, BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, AnonymousPipeServerStream inputFromBenchmark, AnonymousPipeServerStream acknowledgments) { this.logger = logger; this.process = process; this.Diagnoser = diagnoser; + this.compositeInProcessDiagnoser = compositeInProcessDiagnoser; this.inputFromBenchmark = inputFromBenchmark; this.acknowledgments = acknowledgments; DiagnoserActionParameters = new DiagnoserActionParameters(process, benchmarkCase, benchmarkId); finished = new ManualResetEvent(false); - Results = new List(); - PrefixedOutput = new List(); - process.EnableRaisingEvents = true; process.Exited += OnProcessExited; } - internal IDiagnoser Diagnoser { get; } + internal IDiagnoser? Diagnoser { get; } internal DiagnoserActionParameters DiagnoserActionParameters { get; } - internal List Results { get; } + internal List Results { get; } = []; - internal List PrefixedOutput { get; } + internal List PrefixedOutput { get; } = []; internal void ProcessData() { @@ -90,6 +90,26 @@ private void ProcessDataBlocking() { Results.Add(line); } + // Keep in sync with WasmExecutor and InProcessHost. + else if (line.StartsWith(CompositeInProcessDiagnoser.HeaderKey)) + { + // Something like "// InProcessDiagnoser 0 1" + string[] lineItems = line.Split(' '); + int diagnoserIndex = int.Parse(lineItems[2]); + int resultsLinesCount = int.Parse(lineItems[3]); + var resultsStringBuilder = new StringBuilder(); + for (int i = 0; i < resultsLinesCount;) + { + // Strip the prepended "// InProcessDiagnoserResults ". + line = reader.ReadLine().Substring(CompositeInProcessDiagnoser.ResultsKey.Length + 1); + resultsStringBuilder.Append(line); + if (++i < resultsLinesCount) + { + resultsStringBuilder.AppendLine(); + } + } + compositeInProcessDiagnoser.DeserializeResults(diagnoserIndex, DiagnoserActionParameters.BenchmarkCase, resultsStringBuilder.ToString()); + } else if (Engine.Signals.TryGetSignal(line, out var signal)) { Diagnoser?.Handle(signal, DiagnoserActionParameters); diff --git a/src/BenchmarkDotNet/Properties/AssemblyInfo.cs b/src/BenchmarkDotNet/Properties/AssemblyInfo.cs index bec0dbbc48..a385a155f5 100644 --- a/src/BenchmarkDotNet/Properties/AssemblyInfo.cs +++ b/src/BenchmarkDotNet/Properties/AssemblyInfo.cs @@ -12,6 +12,7 @@ [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.Windows,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotTrace,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotMemory,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] +[assembly: InternalsVisibleTo("BenchmarkDotNet.Disassembler,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.TestAdapter,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] diff --git a/src/BenchmarkDotNet/Running/BenchmarkBuildInfo.cs b/src/BenchmarkDotNet/Running/BenchmarkBuildInfo.cs index 6dde6284a0..855e4c7e3c 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkBuildInfo.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkBuildInfo.cs @@ -1,14 +1,16 @@ using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; namespace BenchmarkDotNet.Running { public class BenchmarkBuildInfo { - public BenchmarkBuildInfo(BenchmarkCase benchmarkCase, ImmutableConfig config, int id) + public BenchmarkBuildInfo(BenchmarkCase benchmarkCase, ImmutableConfig config, int id, CompositeInProcessDiagnoser compositeInProcessDiagnoser) { BenchmarkCase = benchmarkCase; Config = config; Id = new BenchmarkId(id, benchmarkCase); + CompositeInProcessDiagnoser = compositeInProcessDiagnoser; } public BenchmarkCase BenchmarkCase { get; } @@ -16,5 +18,7 @@ public BenchmarkBuildInfo(BenchmarkCase benchmarkCase, ImmutableConfig config, i public ImmutableConfig Config { get; } public BenchmarkId Id { get; } + + public CompositeInProcessDiagnoser CompositeInProcessDiagnoser { get; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Running/BenchmarkConverter.cs b/src/BenchmarkDotNet/Running/BenchmarkConverter.cs index 5c9250a579..1d9d23043d 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkConverter.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkConverter.cs @@ -79,8 +79,9 @@ from parameterInstance in parameterInstances } var orderedBenchmarks = configPerType.Orderer.GetExecutionOrder(benchmarks.ToImmutableArray()).ToArray(); + var compositeInProcessDiagnoser = new Diagnosers.CompositeInProcessDiagnoser([.. configPerType.GetDiagnosers().OfType()]); - return new BenchmarkRunInfo(orderedBenchmarks, type, configPerType); + return new BenchmarkRunInfo(orderedBenchmarks, type, configPerType, compositeInProcessDiagnoser); } private static ImmutableConfig GetFullTypeConfig(Type type, IConfig? config) diff --git a/src/BenchmarkDotNet/Running/BenchmarkId.cs b/src/BenchmarkDotNet/Running/BenchmarkId.cs index 52085b905a..98f8d710b7 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkId.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkId.cs @@ -30,9 +30,11 @@ public BenchmarkId(int value, BenchmarkCase benchmarkCase) public override int GetHashCode() => Value; - public string ToArguments() => $"--benchmarkName {FullBenchmarkName.EscapeCommandLine()} --job {JobId.EscapeCommandLine()} --benchmarkId {Value}"; + public string ToArguments(Diagnosers.RunMode diagnoserRunMode) + => $"--benchmarkName {FullBenchmarkName.EscapeCommandLine()} --job {JobId.EscapeCommandLine()} --diagnoserRunMode {(int) diagnoserRunMode} --benchmarkId {Value}"; - public string ToArguments(string fromBenchmark, string toBenchmark) => $"{AnonymousPipesHost.AnonymousPipesDescriptors} {fromBenchmark} {toBenchmark} {ToArguments()}"; + public string ToArguments(string fromBenchmark, string toBenchmark, Diagnosers.RunMode diagnoserRunMode) + => $"{AnonymousPipesHost.AnonymousPipesDescriptors} {fromBenchmark} {toBenchmark} {ToArguments(diagnoserRunMode)}"; public override string ToString() => Value.ToString(); diff --git a/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs b/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs index 4892de7f0c..8602991d18 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs @@ -13,9 +13,9 @@ public static class BenchmarkPartitioner { public static BuildPartition[] CreateForBuild(BenchmarkRunInfo[] supportedBenchmarks, IResolver resolver) => supportedBenchmarks - .SelectMany(info => info.BenchmarksCases.Select(benchmark => (benchmark, benchmark.Config))) + .SelectMany(info => info.BenchmarksCases.Select(benchmark => (benchmark, benchmark.Config, info.CompositeInProcessDiagnoser))) .GroupBy(tuple => tuple.benchmark, BenchmarkRuntimePropertiesComparer.Instance) - .Select(group => new BuildPartition(group.Select((item, index) => new BenchmarkBuildInfo(item.benchmark, item.Config, index)).ToArray(), resolver)) + .Select(group => new BuildPartition([.. group.Select((item, index) => new BenchmarkBuildInfo(item.benchmark, item.Config, index, item.CompositeInProcessDiagnoser))], resolver)) .ToArray(); internal class BenchmarkRuntimePropertiesComparer : IEqualityComparer diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunInfo.cs b/src/BenchmarkDotNet/Running/BenchmarkRunInfo.cs index e12543cc59..1d5cd46eb6 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunInfo.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunInfo.cs @@ -1,15 +1,17 @@ using System; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; namespace BenchmarkDotNet.Running { public class BenchmarkRunInfo : IDisposable { - public BenchmarkRunInfo(BenchmarkCase[] benchmarksCase, Type type, ImmutableConfig config) + public BenchmarkRunInfo(BenchmarkCase[] benchmarksCase, Type type, ImmutableConfig config, CompositeInProcessDiagnoser compositeInProcessDiagnoser) { BenchmarksCases = benchmarksCase; Type = type; Config = config; + CompositeInProcessDiagnoser = compositeInProcessDiagnoser; } public void Dispose() @@ -23,5 +25,6 @@ public void Dispose() public BenchmarkCase[] BenchmarksCases { get; } public Type Type { get; } public ImmutableConfig Config { get; } + public CompositeInProcessDiagnoser CompositeInProcessDiagnoser { get; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index fefd88a906..bc2a6c1e1c 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -244,7 +244,7 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, artifactsToCleanup.AddRange(buildResult.ArtifactsToCleanup); eventProcessor.OnStartRunBenchmark(benchmark); - var report = RunCore(benchmark, info.benchmarkId, logger, resolver, buildResult); + var report = RunCore(benchmark, info.benchmarkId, logger, resolver, buildResult, benchmarkRunInfo.CompositeInProcessDiagnoser); eventProcessor.OnEndRunBenchmark(benchmark, report); if (report.AllMeasurements.Any(m => m.Operations == 0)) @@ -476,20 +476,22 @@ private static BuildResult Build(BuildPartition buildPartition, string rootArtif } } - private static BenchmarkReport RunCore(BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, ILogger logger, IResolver resolver, BuildResult buildResult) + private static BenchmarkReport RunCore(BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, ILogger logger, IResolver resolver, BuildResult buildResult, + CompositeInProcessDiagnoser compositeInProcessDiagnoser) { var toolchain = benchmarkCase.GetToolchain(); logger.WriteLineHeader("// **************************"); logger.WriteLineHeader("// Benchmark: " + benchmarkCase.DisplayInfo); - var (success, executeResults, metrics) = Execute(logger, benchmarkCase, benchmarkId, toolchain, buildResult, resolver); + var (success, executeResults, metrics) = Execute(logger, benchmarkCase, benchmarkId, toolchain, buildResult, resolver, compositeInProcessDiagnoser); return new BenchmarkReport(success, benchmarkCase, buildResult, buildResult, executeResults, metrics); } private static (bool success, List executeResults, List metrics) Execute( - ILogger logger, BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, IToolchain toolchain, BuildResult buildResult, IResolver resolver) + ILogger logger, BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, IToolchain toolchain, BuildResult buildResult, IResolver resolver, + CompositeInProcessDiagnoser compositeInProcessDiagnoser) { var executeResults = new List(); var metrics = new List(); @@ -522,7 +524,9 @@ private static (bool success, List executeResults, List m buildResult, resolver, useDiagnoser ? noOverheadCompositeDiagnoser : null, - launchIndex); + compositeInProcessDiagnoser, + launchIndex, + useDiagnoser ? Diagnosers.RunMode.NoOverhead : Diagnosers.RunMode.None); executeResults.Add(executeResult); @@ -563,7 +567,9 @@ private static (bool success, List executeResults, List m buildResult, resolver, extraRunCompositeDiagnoser, - launchCount + 1); + compositeInProcessDiagnoser, + launchCount + 1, + Diagnosers.RunMode.ExtraRun); if (executeResult.IsSuccess) { @@ -585,7 +591,7 @@ private static (bool success, List executeResults, List m } private static ExecuteResult RunExecute(ILogger logger, BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, IToolchain toolchain, - BuildResult buildResult, IResolver resolver, IDiagnoser diagnoser, int launchIndex) + BuildResult buildResult, IResolver resolver, IDiagnoser? diagnoser, CompositeInProcessDiagnoser compositeInProcessDiagnoser, int launchIndex, Diagnosers.RunMode diagnoserRunMode) { var executeResult = toolchain.Executor.Execute( new ExecuteParameters( @@ -595,7 +601,9 @@ private static ExecuteResult RunExecute(ILogger logger, BenchmarkCase benchmarkC logger, resolver, launchIndex, - diagnoser)); + compositeInProcessDiagnoser, + diagnoser, + diagnoserRunMode)); if (!executeResult.IsSuccess) { @@ -657,8 +665,8 @@ private static (BenchmarkRunInfo[], List) GetSupportedBenchmark new BenchmarkRunInfo( validBenchmarks, benchmarkRunInfo.Type, - benchmarkRunInfo.Config - + benchmarkRunInfo.Config, + benchmarkRunInfo.CompositeInProcessDiagnoser )); diff --git a/src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs b/src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs index ea280ab054..24a2860e6b 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs @@ -216,7 +216,7 @@ private IEnumerable ApplesToApples(ImmutableConfig effectiveConfig, IRe .WithUnrollFactor(dictionary[(benchmark.Descriptor, benchmark.Parameters)].Operations % 16 == 0 ? 16 : 1), benchmark.Parameters, benchmark.Config)).ToArray(), - benchmarkInfo.Type, benchmarkInfo.Config)) + benchmarkInfo.Type, benchmarkInfo.Config, benchmarkInfo.CompositeInProcessDiagnoser)) .ToArray(); logger.WriteLineHeader("Actual benchmarking is going to happen now!"); diff --git a/src/BenchmarkDotNet/Running/ClassicBenchmarkConverter.cs b/src/BenchmarkDotNet/Running/ClassicBenchmarkConverter.cs index 6ec87e8f92..d7fd7cb8b9 100644 --- a/src/BenchmarkDotNet/Running/ClassicBenchmarkConverter.cs +++ b/src/BenchmarkDotNet/Running/ClassicBenchmarkConverter.cs @@ -125,7 +125,7 @@ public static BenchmarkRunInfo[] SourceToBenchmarks(string source, IConfig? conf b.Config); }); resultBenchmarks.Add( - new BenchmarkRunInfo(benchmarks.ToArray(), runInfo.Type, runInfo.Config)); + new BenchmarkRunInfo(benchmarks.ToArray(), runInfo.Type, runInfo.Config, runInfo.CompositeInProcessDiagnoser)); } return resultBenchmarks.ToArray(); diff --git a/src/BenchmarkDotNet/Templates/BenchmarkProgram.txt b/src/BenchmarkDotNet/Templates/BenchmarkProgram.txt index 65dc0a24e6..fa830ecd09 100644 --- a/src/BenchmarkDotNet/Templates/BenchmarkProgram.txt +++ b/src/BenchmarkDotNet/Templates/BenchmarkProgram.txt @@ -42,6 +42,7 @@ namespace BenchmarkDotNet.Autogenerated // this variable name is used by CodeGenerator.GetCoreRtSwitch, do NOT change it System.String benchmarkName = System.Linq.Enumerable.FirstOrDefault(System.Linq.Enumerable.Skip(System.Linq.Enumerable.SkipWhile(args, arg => arg != "--benchmarkName"), 1)) ?? "not provided"; + BenchmarkDotNet.Diagnosers.RunMode diagnoserRunMode = (BenchmarkDotNet.Diagnosers.RunMode) System.Int32.Parse(System.Linq.Enumerable.FirstOrDefault(System.Linq.Enumerable.Skip(System.Linq.Enumerable.SkipWhile(args, arg => arg != "--diagnoserRunMode"), 1)) ?? "0"); System.Int32 id = args.Length > 0 ? System.Int32.Parse(args[args.Length - 1]) // this variable name is used by CodeGenerator.GetCoreRtSwitch, do NOT change it : 0; // used when re-using generated exe without BDN (typically to repro a bug) @@ -50,12 +51,11 @@ namespace BenchmarkDotNet.Autogenerated { host.WriteLine("You have not specified benchmark id (an integer) so the first benchmark will be executed."); } - #if NATIVEAOT $NativeAotSwitch$ #else System.Type type = typeof(BenchmarkDotNet.Autogenerated.UniqueProgramName).Assembly.GetType($"BenchmarkDotNet.Autogenerated.Runnable_{id}"); - type.GetMethod("Run", System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Static).Invoke(null, new System.Object[] { host, benchmarkName }); + type.GetMethod("Run", System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Static).Invoke(null, new System.Object[] { host, benchmarkName, diagnoserRunMode }); #endif return 0; } diff --git a/src/BenchmarkDotNet/Templates/BenchmarkType.txt b/src/BenchmarkDotNet/Templates/BenchmarkType.txt index ea1bbaa858..449c252ea6 100644 --- a/src/BenchmarkDotNet/Templates/BenchmarkType.txt +++ b/src/BenchmarkDotNet/Templates/BenchmarkType.txt @@ -1,7 +1,7 @@ // the type name must be in sync with WindowsDisassembler.BuildArguments public unsafe class Runnable_$ID$ : global::$WorkloadTypeName$ { - public static void Run(BenchmarkDotNet.Engines.IHost host, System.String benchmarkName) + public static void Run(BenchmarkDotNet.Engines.IHost host, System.String benchmarkName, BenchmarkDotNet.Diagnosers.RunMode diagnoserRunMode) { BenchmarkDotNet.Autogenerated.Runnable_$ID$ instance = new BenchmarkDotNet.Autogenerated.Runnable_$ID$ { $ParamsInitializer$ }; // do NOT change name "instance" (used in SmartParamameter) $ParamsContent$ @@ -21,6 +21,17 @@ if (BenchmarkDotNet.Validators.ValidationErrorReporter.ReportIfAny(errors, host)) return; + BenchmarkDotNet.Diagnosers.CompositeInProcessDiagnoserHandler compositeInProcessDiagnoserHandler = new BenchmarkDotNet.Diagnosers.CompositeInProcessDiagnoserHandler( + new BenchmarkDotNet.Diagnosers.IInProcessDiagnoserHandler[] + { + $InProcessDiagnosers$ + }, + host, + diagnoserRunMode, + new BenchmarkDotNet.Diagnosers.InProcessDiagnoserActionArgs(instance) + ); + compositeInProcessDiagnoserHandler.Handle(BenchmarkDotNet.Engines.BenchmarkSignal.BeforeEngine); + BenchmarkDotNet.Engines.EngineParameters engineParameters = new BenchmarkDotNet.Engines.EngineParameters() { Host = host, @@ -38,7 +49,8 @@ TargetJob = job, OperationsPerInvoke = $OperationsPerInvoke$, MeasureExtraStats = $MeasureExtraStats$, - BenchmarkName = benchmarkName + BenchmarkName = benchmarkName, + InProcessDiagnoserHandler = compositeInProcessDiagnoserHandler }; using (BenchmarkDotNet.Engines.IEngine engine = new $EngineFactoryType$().CreateReadyToRun(engineParameters)) @@ -49,6 +61,7 @@ instance.__TrickTheJIT__(); // compile the method for disassembler, but without actual run of the benchmark ;) } + compositeInProcessDiagnoserHandler.Handle(BenchmarkDotNet.Engines.BenchmarkSignal.AfterEngine); } public delegate $OverheadMethodReturnTypeName$ OverheadDelegate($ArgumentsDefinition$); diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs index d0595fe166..044f169588 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs @@ -41,9 +41,11 @@ public ExecuteResult Execute(ExecuteParameters executeParameters) executeParameters.Logger, executeParameters.BuildResult.ArtifactsPaths, executeParameters.Diagnoser, + executeParameters.CompositeInProcessDiagnoser, Path.GetFileName(executeParameters.BuildResult.ArtifactsPaths.ExecutablePath), executeParameters.Resolver, - executeParameters.LaunchIndex); + executeParameters.LaunchIndex, + executeParameters.DiagnoserRunMode); } finally { @@ -57,10 +59,12 @@ private ExecuteResult Execute(BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, ILogger logger, ArtifactsPaths artifactsPaths, - IDiagnoser diagnoser, + IDiagnoser? diagnoser, + CompositeInProcessDiagnoser compositeInProcessDiagnoser, string executableName, IResolver resolver, - int launchIndex) + int launchIndex, + Diagnosers.RunMode diagnoserRunMode) { using AnonymousPipeServerStream inputFromBenchmark = new AnonymousPipeServerStream(PipeDirection.In, HandleInheritability.Inheritable); using AnonymousPipeServerStream acknowledgments = new AnonymousPipeServerStream(PipeDirection.Out, HandleInheritability.Inheritable); @@ -68,7 +72,7 @@ private ExecuteResult Execute(BenchmarkCase benchmarkCase, var startInfo = DotNetCliCommandExecutor.BuildStartInfo( CustomDotNetCliPath, artifactsPaths.BinariesDirectoryPath, - $"{executableName.EscapeCommandLine()} {benchmarkId.ToArguments(inputFromBenchmark.GetClientHandleAsString(), acknowledgments.GetClientHandleAsString())}", + $"{executableName.EscapeCommandLine()} {benchmarkId.ToArguments(inputFromBenchmark.GetClientHandleAsString(), acknowledgments.GetClientHandleAsString(), diagnoserRunMode)}", redirectStandardOutput: true, redirectStandardInput: false, redirectStandardError: false); // #1629 @@ -79,7 +83,7 @@ private ExecuteResult Execute(BenchmarkCase benchmarkCase, using (ConsoleExitHandler consoleExitHandler = new(process, logger)) using (AsyncProcessOutputReader processOutputReader = new(process, logOutput: true, logger, readStandardError: false)) { - Broker broker = new(logger, process, diagnoser, benchmarkCase, benchmarkId, inputFromBenchmark, acknowledgments); + Broker broker = new(logger, process, diagnoser, compositeInProcessDiagnoser, benchmarkCase, benchmarkId, inputFromBenchmark, acknowledgments); logger.WriteLineInfo($"// Execute: {process.StartInfo.FileName} {process.StartInfo.Arguments} in {process.StartInfo.WorkingDirectory}"); diff --git a/src/BenchmarkDotNet/Toolchains/Executor.cs b/src/BenchmarkDotNet/Toolchains/Executor.cs index adc05e12e6..70e844ef22 100644 --- a/src/BenchmarkDotNet/Toolchains/Executor.cs +++ b/src/BenchmarkDotNet/Toolchains/Executor.cs @@ -33,24 +33,26 @@ public ExecuteResult Execute(ExecuteParameters executeParameters) } return Execute(executeParameters.BenchmarkCase, executeParameters.BenchmarkId, executeParameters.Logger, executeParameters.BuildResult.ArtifactsPaths, - executeParameters.Diagnoser, executeParameters.Resolver, executeParameters.LaunchIndex); + executeParameters.Diagnoser, executeParameters.CompositeInProcessDiagnoser, executeParameters.Resolver, executeParameters.LaunchIndex, + executeParameters.DiagnoserRunMode); } private static ExecuteResult Execute(BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, ILogger logger, ArtifactsPaths artifactsPaths, - IDiagnoser diagnoser, IResolver resolver, int launchIndex) + IDiagnoser? diagnoser, CompositeInProcessDiagnoser compositeInProcessDiagnoser, IResolver resolver, int launchIndex, + Diagnosers.RunMode diagnoserRunMode) { try { using AnonymousPipeServerStream inputFromBenchmark = new(PipeDirection.In, HandleInheritability.Inheritable); using AnonymousPipeServerStream acknowledgments = new(PipeDirection.Out, HandleInheritability.Inheritable); - string args = benchmarkId.ToArguments(inputFromBenchmark.GetClientHandleAsString(), acknowledgments.GetClientHandleAsString()); + string args = benchmarkId.ToArguments(inputFromBenchmark.GetClientHandleAsString(), acknowledgments.GetClientHandleAsString(), diagnoserRunMode); using (Process process = new() { StartInfo = CreateStartInfo(benchmarkCase, artifactsPaths, args, resolver) }) using (ConsoleExitHandler consoleExitHandler = new(process, logger)) using (AsyncProcessOutputReader processOutputReader = new(process, logOutput: true, logger, readStandardError: false)) { - Broker broker = new(logger, process, diagnoser, benchmarkCase, benchmarkId, inputFromBenchmark, acknowledgments); + Broker broker = new(logger, process, diagnoser, compositeInProcessDiagnoser, benchmarkCase, benchmarkId, inputFromBenchmark, acknowledgments); diagnoser?.Handle(HostSignal.BeforeProcessStart, new DiagnoserActionParameters(process, benchmarkCase, benchmarkId)); diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs index 4796c84a77..5e0c222591 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs @@ -1051,8 +1051,8 @@ private MethodBuilder EmitRunMethod() /* .method public hidebysig static void Run ( - class [BenchmarkDotNet]BenchmarkDotNet.Running.BenchmarkCase benchmarkCase, - class [BenchmarkDotNet]BenchmarkDotNet.Engines.IHost host + class [BenchmarkDotNet]BenchmarkDotNet.Engines.IHost host, + class [BenchmarkDotNet]BenchmarkDotNet.Toolchains.Parameters.ExecuteParameters parameters, ) cil managed */ var argsExceptInstance = prepareForRunMethodTemplate @@ -1066,8 +1066,8 @@ class [BenchmarkDotNet]BenchmarkDotNet.Engines.IHost host EmitParameterInfo.CreateReturnVoidParameter(), argsExceptInstance); argsExceptInstance = methodBuilder.GetEmitParameters(argsExceptInstance); - var benchmarkCaseArg = argsExceptInstance[0]; - var hostArg = argsExceptInstance[1]; + var hostArg = argsExceptInstance[0]; + var parametersArg = argsExceptInstance[1]; var ilBuilder = methodBuilder.GetILGenerator(); @@ -1097,15 +1097,15 @@ [5] valuetype [BenchmarkDotNet]BenchmarkDotNet.Engines.RunResults ilBuilder.EmitStloc(instanceLocal); /* - // (Job, EngineParameters, IEngineFactory) valueTuple = RunnableReuse.PrepareForRun(instance, benchmarkCase, host); + // (Job, EngineParameters, IEngineFactory) valueTuple = RunnableReuse.PrepareForRun(instance, host, parameters); IL_0006: ldloc.0 IL_0007: ldarg.0 IL_0008: ldarg.1 - IL_0009: call valuetype [mscorlib]System.ValueTuple`3 [BenchmarkDotNet]BenchmarkDotNet.Toolchains.InProcess.Emit.Implementation.RunnableReuse::PrepareForRun(!!0, class [BenchmarkDotNet]BenchmarkDotNet.Running.BenchmarkCase, class [BenchmarkDotNet]BenchmarkDotNet.Engines.IHost) + IL_0009: call valuetype [mscorlib]System.ValueTuple`3 [BenchmarkDotNet]BenchmarkDotNet.Toolchains.InProcess.Emit.Implementation.RunnableReuse::PrepareForRun(!!0, class [BenchmarkDotNet]BenchmarkDotNet.Engines.IHost, class [BenchmarkDotNet]BenchmarkDotNet.Toolchains.Parameters.ExecuteParameters) */ ilBuilder.EmitLdloc(instanceLocal); - ilBuilder.EmitLdarg(benchmarkCaseArg); ilBuilder.EmitLdarg(hostArg); + ilBuilder.EmitLdarg(parametersArg); ilBuilder.Emit(OpCodes.Call, prepareForRunMethodTemplate.MakeGenericMethod(runnableBuilder)); /* @@ -1216,6 +1216,18 @@ [5] valuetype [BenchmarkDotNet]BenchmarkDotNet.Engines.RunResults ilBuilder.EndExceptionBlock(); } + /* + // engineParameters.InProcessDiagnoserHandler.Handle(BenchmarkSignal.AfterEngine); + IL_0054: ldloc.2 + IL_0055: callvirt instance class [BenchmarkDotNet]BenchmarkDotNet.Diagnosers.CompositeInProcessDiagnoserHandler [BenchmarkDotNet]BenchmarkDotNet.Engines.EngineParameters::get_InProcessDiagnoserHandler() + IL_005a: ldc.i4.3 + IL_005b: callvirt instance void [BenchmarkDotNet]BenchmarkDotNet.Diagnosers.CompositeInProcessDiagnoserHandler::Handle(valuetype [BenchmarkDotNet]BenchmarkDotNet.Engines.BenchmarkSignal) + */ + ilBuilder.EmitLdloc(engineParametersLocal); + ilBuilder.Emit(OpCodes.Callvirt, typeof(EngineParameters).GetProperty(nameof(EngineParameters.InProcessDiagnoserHandler)).GetGetMethod()); + ilBuilder.Emit(OpCodes.Ldc_I4_3); + ilBuilder.Emit(OpCodes.Callvirt, typeof(Diagnosers.CompositeInProcessDiagnoserHandler).GetMethod(nameof(Diagnosers.CompositeInProcessDiagnoserHandler.Handle))); + ilBuilder.EmitVoidReturn(methodBuilder); return methodBuilder; diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Runnable/RunnableProgram.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Runnable/RunnableProgram.cs index 08584e9c50..2516f4e6d5 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Runnable/RunnableProgram.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Runnable/RunnableProgram.cs @@ -2,18 +2,15 @@ using System.Reflection; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.Parameters; using static BenchmarkDotNet.Toolchains.InProcess.Emit.Implementation.RunnableConstants; using static BenchmarkDotNet.Toolchains.InProcess.Emit.Implementation.RunnableReflectionHelpers; namespace BenchmarkDotNet.Toolchains.InProcess.Emit.Implementation { - public class RunnableProgram + internal class RunnableProgram { - public static int Run( - BenchmarkId benchmarkId, - Assembly partitionAssembly, - BenchmarkCase benchmarkCase, - IHost host) + internal static int Run(Assembly partitionAssembly, IHost host, ExecuteParameters parameters) { // the first thing to do is to let diagnosers hook in before anything happens // so all jit-related diagnosers can catch first jit compilation! @@ -25,9 +22,9 @@ public static int Run( // which could cause the jitting/assembly loading to happen before we do anything // we have some jitting diagnosers and we want them to catch all the informations!! - var runCallback = GetRunCallback(benchmarkId, partitionAssembly); + var runCallback = GetRunCallback(parameters.BenchmarkId, partitionAssembly); - runCallback.Invoke(null, new object[] { benchmarkCase, host }); + runCallback.Invoke(null, [host, parameters]); return 0; } catch (Exception oom) when ( diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Runnable/RunnableReuse.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Runnable/RunnableReuse.cs index 7067650ec1..68176fe2b9 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Runnable/RunnableReuse.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Runnable/RunnableReuse.cs @@ -1,9 +1,11 @@ using System.Linq; +using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.Parameters; using BenchmarkDotNet.Validators; using static BenchmarkDotNet.Toolchains.InProcess.Emit.Implementation.RunnableConstants; using static BenchmarkDotNet.Toolchains.InProcess.Emit.Implementation.RunnableReflectionHelpers; @@ -12,11 +14,9 @@ namespace BenchmarkDotNet.Toolchains.InProcess.Emit.Implementation { public static class RunnableReuse { - public static (Job, EngineParameters, IEngineFactory) PrepareForRun( - T instance, - BenchmarkCase benchmarkCase, - IHost host) + public static (Job, EngineParameters, IEngineFactory) PrepareForRun(T instance, IHost host, ExecuteParameters parameters) { + var benchmarkCase = parameters.BenchmarkCase; FillObjectMembers(instance, benchmarkCase); DumpEnvironment(host); @@ -28,7 +28,15 @@ public static (Job, EngineParameters, IEngineFactory) PrepareForRun( if (ValidationErrorReporter.ReportIfAny(errors, host)) return (null, null, null); - var engineParameters = CreateEngineParameters(instance, benchmarkCase, host); + var compositeInProcessDiagnoserHandler = new CompositeInProcessDiagnoserHandler( + parameters.CompositeInProcessDiagnoser.GetInProcessHandlers(benchmarkCase), + host, + parameters.DiagnoserRunMode, + new InProcessDiagnoserActionArgs(instance) + ); + compositeInProcessDiagnoserHandler.Handle(BenchmarkSignal.BeforeEngine); + + var engineParameters = CreateEngineParameters(instance, benchmarkCase, host, compositeInProcessDiagnoserHandler); var engineFactory = GetEngineFactory(benchmarkCase); return (job, engineParameters, engineFactory); @@ -80,12 +88,8 @@ private static IEngineFactory GetEngineFactory(BenchmarkCase benchmarkCase) InfrastructureResolver.Instance); } - private static EngineParameters CreateEngineParameters( - T instance, - BenchmarkCase benchmarkCase, - IHost host) - { - var engineParameters = new EngineParameters + private static EngineParameters CreateEngineParameters(T instance, BenchmarkCase benchmarkCase, IHost host, CompositeInProcessDiagnoserHandler inProcessDiagnoserHandler) + => new() { Host = host, WorkloadActionUnroll = LoopCallbackFromMethod(instance, WorkloadActionUnrollMethodName), @@ -102,9 +106,8 @@ private static EngineParameters CreateEngineParameters( TargetJob = benchmarkCase.Job, OperationsPerInvoke = benchmarkCase.Descriptor.OperationsPerInvoke, MeasureExtraStats = benchmarkCase.Config.HasExtraStatsDiagnoser(), - BenchmarkName = FullNameProvider.GetBenchmarkName(benchmarkCase) + BenchmarkName = FullNameProvider.GetBenchmarkName(benchmarkCase), + InProcessDiagnoserHandler = inProcessDiagnoserHandler }; - return engineParameters; - } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitExecutor.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitExecutor.cs index 60d6f8a6ae..a87034b34d 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitExecutor.cs @@ -76,6 +76,8 @@ public ExecuteResult Execute(ExecuteParameters executeParameters) $"Benchmark {executeParameters.BenchmarkCase.DisplayInfo} takes too long to run. " + "Prefer to use out-of-process toolchains for long-running benchmarks."); + host.HandleInProcessDiagnoserResults(executeParameters.BenchmarkCase, executeParameters.CompositeInProcessDiagnoser); + return ExecuteResult.FromRunResults(host.RunResults, exitCode); } @@ -111,11 +113,7 @@ private int ExecuteCore(IHost host, ExecuteParameters parameters) var generatedAssembly = ((InProcessEmitArtifactsPath)parameters.BuildResult.ArtifactsPaths) .GeneratedAssembly; - exitCode = RunnableProgram.Run( - parameters.BenchmarkId, - generatedAssembly, - parameters.BenchmarkCase, - host); + exitCode = RunnableProgram.Run(generatedAssembly, host, parameters); } catch (Exception ex) { diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/InProcessHost.cs b/src/BenchmarkDotNet/Toolchains/InProcess/InProcessHost.cs index 90043dded6..ee0dc1ddf3 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/InProcessHost.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/InProcessHost.cs @@ -1,7 +1,8 @@ using System; +using System.Collections.Generic; using System.Diagnostics; using System.IO; - +using System.Text; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; @@ -9,19 +10,16 @@ using BenchmarkDotNet.Running; using BenchmarkDotNet.Validators; -using JetBrains.Annotations; - namespace BenchmarkDotNet.Toolchains.InProcess { /// Host API for in-process benchmarks. /// - public sealed class InProcessHost : IHost + internal sealed class InProcessHost : IHost { private readonly ILogger logger; - private readonly IDiagnoser? diagnoser; - private readonly DiagnoserActionParameters? diagnoserActionParameters; + private readonly List inProcessDiagnoserLines = []; /// Creates a new instance of . /// Current benchmark. @@ -34,7 +32,6 @@ public InProcessHost(BenchmarkCase benchmarkCase, ILogger logger, IDiagnoser dia this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); this.diagnoser = diagnoser; - IsDiagnoserAttached = diagnoser != null; Config = benchmarkCase.Config; if (diagnoser != null) @@ -44,16 +41,12 @@ public InProcessHost(BenchmarkCase benchmarkCase, ILogger logger, IDiagnoser dia default); } - /// True if there are diagnosers attached. - /// True if there are diagnosers attached. - [PublicAPI] public bool IsDiagnoserAttached { get; } - /// Results of the run. /// Results of the run. public RunResults RunResults { get; private set; } /// Current config - [PublicAPI] public IConfig Config { get; set; } + public IConfig Config { get; set; } /// Passes text to the host. /// Text to write. @@ -64,20 +57,18 @@ public InProcessHost(BenchmarkCase benchmarkCase, ILogger logger, IDiagnoser dia /// Passes text (new line appended) to the host. /// Text to write. - public void WriteLine(string message) => logger.WriteLine(message); + public void WriteLine(string message) + { + logger.WriteLine(message); + if (message.StartsWith(CompositeInProcessDiagnoser.HeaderKey)) // Captures both header and results + { + inProcessDiagnoserLines.Add(message); + } + } /// Sends notification signal to the host. /// The signal to send. - public void SendSignal(HostSignal hostSignal) - { - if (!IsDiagnoserAttached) // no need to send the signal, nobody is listening for it - return; - - if (diagnoser == null) - throw new NullReferenceException(nameof(diagnoser)); - - diagnoser.Handle(hostSignal, diagnoserActionParameters); - } + public void SendSignal(HostSignal hostSignal) => diagnoser?.Handle(hostSignal, diagnoserActionParameters); public void SendError(string message) => logger.WriteLine(LogKind.Error, $"{ValidationErrorReporter.ConsoleErrorPrefix} {message}"); @@ -94,6 +85,34 @@ public void ReportResults(RunResults runResults) } } + // Keep in sync with Broker and WasmExecutor. + internal void HandleInProcessDiagnoserResults(BenchmarkCase benchmarkCase, CompositeInProcessDiagnoser compositeInProcessDiagnoser) + { + var linesEnumerator = inProcessDiagnoserLines.GetEnumerator(); + while (linesEnumerator.MoveNext()) + { + // Something like "// InProcessDiagnoser 0 1" + var line = linesEnumerator.Current; + string[] lineItems = line.Split(' '); + int diagnoserIndex = int.Parse(lineItems[2]); + int resultsLinesCount = int.Parse(lineItems[3]); + var resultsStringBuilder = new StringBuilder(); + for (int i = 0; i < resultsLinesCount;) + { + // Strip the prepended "// InProcessDiagnoserResults ". + bool movedNext = linesEnumerator.MoveNext(); + Debug.Assert(movedNext); + line = linesEnumerator.Current.Substring(CompositeInProcessDiagnoser.ResultsKey.Length + 1); + resultsStringBuilder.Append(line); + if (++i < resultsLinesCount) + { + resultsStringBuilder.AppendLine(); + } + } + compositeInProcessDiagnoser.DeserializeResults(diagnoserIndex, benchmarkCase, resultsStringBuilder.ToString()); + } + } + public void Dispose() { // do nothing on purpose diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitExecutor.cs b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitExecutor.cs index b50ba77c1c..424ea4a31b 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitExecutor.cs @@ -76,6 +76,8 @@ public ExecuteResult Execute(ExecuteParameters executeParameters) $"Benchmark {executeParameters.BenchmarkCase.DisplayInfo} takes too long to run. " + "Prefer to use out-of-process toolchains for long-running benchmarks."); + host.HandleInProcessDiagnoserResults(executeParameters.BenchmarkCase, executeParameters.CompositeInProcessDiagnoser); + return ExecuteResult.FromRunResults(host.RunResults, exitCode); } @@ -99,7 +101,7 @@ private int ExecuteCore(IHost host, ExecuteParameters parameters) process.TrySetAffinity(affinity.Value, parameters.Logger); } - exitCode = InProcessNoEmitRunner.Run(host, parameters.BenchmarkCase); + exitCode = InProcessNoEmitRunner.Run(host, parameters); } catch (Exception ex) { diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitRunner.cs b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitRunner.cs index 18ee40c30a..73c4778fb3 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitRunner.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitRunner.cs @@ -6,7 +6,7 @@ using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; - +using BenchmarkDotNet.Toolchains.Parameters; using JetBrains.Annotations; namespace BenchmarkDotNet.Toolchains.InProcess.NoEmit @@ -19,7 +19,7 @@ internal class InProcessNoEmitRunner #if NET6_0_OR_GREATER [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Runnable))] #endif - public static int Run(IHost host, BenchmarkCase benchmarkCase) + public static int Run(IHost host, ExecuteParameters parameters) { // the first thing to do is to let diagnosers hook in before anything happens // so all jit-related diagnosers can catch first jit compilation! @@ -37,7 +37,7 @@ public static int Run(IHost host, BenchmarkCase benchmarkCase) var methodInfo = type.GetMethod(nameof(Runnable.RunCore), BindingFlags.Public | BindingFlags.Static) ?? throw new InvalidOperationException($"Bug: method {nameof(Runnable.RunCore)} in {inProcessRunnableTypeName} not found."); - methodInfo.Invoke(null, new object[] { host, benchmarkCase }); + methodInfo.Invoke(null, [host, parameters]); return 0; } @@ -104,8 +104,9 @@ internal static void FillMembers(object instance, BenchmarkCase benchmarkCase) [UsedImplicitly] private static class Runnable { - public static void RunCore(IHost host, BenchmarkCase benchmarkCase) + public static void RunCore(IHost host, ExecuteParameters parameters) { + var benchmarkCase = parameters.BenchmarkCase; var target = benchmarkCase.Descriptor; var job = benchmarkCase.Job; // TODO: filter job (same as SourceCodePresenter does)? int unrollFactor = benchmarkCase.Job.ResolveValue(RunMode.UnrollFactorCharacteristic, EnvironmentResolver.Instance); @@ -130,6 +131,14 @@ public static void RunCore(IHost host, BenchmarkCase benchmarkCase) host.WriteLine("// Job: {0}", job.DisplayInfo); host.WriteLine(); + var compositeInProcessDiagnoserHandler = new Diagnosers.CompositeInProcessDiagnoserHandler( + parameters.CompositeInProcessDiagnoser.GetInProcessHandlers(benchmarkCase), + host, + parameters.DiagnoserRunMode, + new Diagnosers.InProcessDiagnoserActionArgs(instance) + ); + compositeInProcessDiagnoserHandler.Handle(BenchmarkSignal.BeforeEngine); + var engineParameters = new EngineParameters { Host = host, @@ -147,7 +156,8 @@ public static void RunCore(IHost host, BenchmarkCase benchmarkCase) TargetJob = job, OperationsPerInvoke = target.OperationsPerInvoke, MeasureExtraStats = benchmarkCase.Config.HasExtraStatsDiagnoser(), - BenchmarkName = FullNameProvider.GetBenchmarkName(benchmarkCase) + BenchmarkName = FullNameProvider.GetBenchmarkName(benchmarkCase), + InProcessDiagnoserHandler = compositeInProcessDiagnoserHandler }; using (var engine = job @@ -158,6 +168,7 @@ public static void RunCore(IHost host, BenchmarkCase benchmarkCase) host.ReportResults(results); // printing costs memory, do this after runs } + compositeInProcessDiagnoserHandler.Handle(BenchmarkSignal.AfterEngine); } } } diff --git a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs index e6756ebcbe..f58eb786a5 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs @@ -10,10 +10,11 @@ using BenchmarkDotNet.Toolchains.Parameters; using BenchmarkDotNet.Toolchains.Results; using System; +using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; using System.IO; -using System.Linq; +using System.Text; namespace BenchmarkDotNet.Toolchains.MonoWasm { @@ -29,22 +30,22 @@ public ExecuteResult Execute(ExecuteParameters executeParameters) } return Execute(executeParameters.BenchmarkCase, executeParameters.BenchmarkId, executeParameters.Logger, executeParameters.BuildResult.ArtifactsPaths, - executeParameters.Diagnoser, executeParameters.Resolver, executeParameters.LaunchIndex); + executeParameters.Diagnoser, executeParameters.CompositeInProcessDiagnoser, executeParameters.Resolver, executeParameters.LaunchIndex, + executeParameters.DiagnoserRunMode); } private static ExecuteResult Execute(BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, ILogger logger, ArtifactsPaths artifactsPaths, - IDiagnoser diagnoser, IResolver resolver, int launchIndex) + IDiagnoser? diagnoser, CompositeInProcessDiagnoser compositeInProcessDiagnoser, IResolver resolver, int launchIndex, + Diagnosers.RunMode diagnoserRunMode) { try { - using (Process process = CreateProcess(benchmarkCase, artifactsPaths, benchmarkId.ToArguments(), resolver)) - using (ConsoleExitHandler consoleExitHandler = new(process, logger)) - using (AsyncProcessOutputReader processOutputReader = new(process, logOutput: true, logger, readStandardError: false)) - { - diagnoser?.Handle(HostSignal.BeforeProcessStart, new DiagnoserActionParameters(process, benchmarkCase, benchmarkId)); + using Process process = CreateProcess(benchmarkCase, artifactsPaths, benchmarkId.ToArguments(diagnoserRunMode), resolver); + using ConsoleExitHandler consoleExitHandler = new(process, logger); + using AsyncProcessOutputReader processOutputReader = new(process, logOutput: true, logger, readStandardError: false); - return Execute(process, benchmarkCase, processOutputReader, logger, consoleExitHandler, launchIndex); - } + diagnoser?.Handle(HostSignal.BeforeProcessStart, new DiagnoserActionParameters(process, benchmarkCase, benchmarkId)); + return Execute(process, benchmarkCase, processOutputReader, logger, consoleExitHandler, launchIndex, compositeInProcessDiagnoser); } finally { @@ -75,7 +76,7 @@ private static Process CreateProcess(BenchmarkCase benchmarkCase, ArtifactsPaths } private static ExecuteResult Execute(Process process, BenchmarkCase benchmarkCase, AsyncProcessOutputReader processOutputReader, - ILogger logger, ConsoleExitHandler consoleExitHandler, int launchIndex) + ILogger logger, ConsoleExitHandler consoleExitHandler, int launchIndex, CompositeInProcessDiagnoser compositeInProcessDiagnoser) { logger.WriteLineInfo($"// Execute: {process.StartInfo.FileName} {process.StartInfo.Arguments} in {process.StartInfo.WorkingDirectory}"); @@ -101,12 +102,49 @@ private static ExecuteResult Execute(Process process, BenchmarkCase benchmarkCas } ImmutableArray outputLines = processOutputReader.GetOutputLines(); + var prefixedLines = new List(); + var resultLines = new List(); + var outputEnumerator = outputLines.GetEnumerator(); + while (outputEnumerator.MoveNext()) + { + var line = outputEnumerator.Current; + if (!line.StartsWith("//")) + { + resultLines.Add(line); + continue; + } + + prefixedLines.Add(line); + + // Keep in sync with Broker and InProcessHost. + if (line.StartsWith(CompositeInProcessDiagnoser.HeaderKey)) + { + // Something like "// InProcessDiagnoser 0 1" + string[] lineItems = line.Split(' '); + int diagnoserIndex = int.Parse(lineItems[2]); + int resultsLinesCount = int.Parse(lineItems[3]); + var resultsStringBuilder = new StringBuilder(); + for (int i = 0; i < resultsLinesCount;) + { + // Strip the prepended "// InProcessDiagnoserResults ". + bool movedNext = outputEnumerator.MoveNext(); + Debug.Assert(movedNext); + line = outputEnumerator.Current.Substring(CompositeInProcessDiagnoser.ResultsKey.Length + 1); + resultsStringBuilder.Append(line); + if (++i < resultsLinesCount) + { + resultsStringBuilder.AppendLine(); + } + } + compositeInProcessDiagnoser.DeserializeResults(diagnoserIndex, benchmarkCase, resultsStringBuilder.ToString()); + } + } return new ExecuteResult(true, process.HasExited ? process.ExitCode : null, process.Id, - outputLines.Where(line => !line.StartsWith("//")).ToArray(), - outputLines.Where(line => line.StartsWith("//")).ToArray(), + [.. resultLines], + [.. prefixedLines], outputLines, launchIndex); } diff --git a/src/BenchmarkDotNet/Toolchains/Parameters/ExecuteParameters.cs b/src/BenchmarkDotNet/Toolchains/Parameters/ExecuteParameters.cs index e10c81b49b..34ae665a83 100644 --- a/src/BenchmarkDotNet/Toolchains/Parameters/ExecuteParameters.cs +++ b/src/BenchmarkDotNet/Toolchains/Parameters/ExecuteParameters.cs @@ -7,33 +7,29 @@ namespace BenchmarkDotNet.Toolchains.Parameters { - public class ExecuteParameters + public class ExecuteParameters(BuildResult buildResult, BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, + ILogger logger, IResolver resolver, int launchIndex, + CompositeInProcessDiagnoser compositeInProcessDiagnoser, + IDiagnoser diagnoser = null, RunMode diagnoserRunMode = RunMode.None) { internal static readonly TimeSpan ProcessExitTimeout = TimeSpan.FromSeconds(2); - public ExecuteParameters(BuildResult buildResult, BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, ILogger logger, IResolver resolver, int launchIndex, IDiagnoser? diagnoser = null) - { - BuildResult = buildResult; - BenchmarkCase = benchmarkCase; - BenchmarkId = benchmarkId; - Logger = logger; - Resolver = resolver; - Diagnoser = diagnoser; - LaunchIndex = launchIndex; - } + public BuildResult BuildResult { get; } = buildResult; - public BuildResult BuildResult { get; } + public BenchmarkCase BenchmarkCase { get; } = benchmarkCase; - public BenchmarkCase BenchmarkCase { get; } + public BenchmarkId BenchmarkId { get; } = benchmarkId; - public BenchmarkId BenchmarkId { get; } + public ILogger Logger { get; } = logger; - public ILogger Logger { get; } + public IResolver Resolver { get; } = resolver; - public IResolver Resolver { get; } + public CompositeInProcessDiagnoser CompositeInProcessDiagnoser { get; } = compositeInProcessDiagnoser; - public IDiagnoser Diagnoser { get; } + public IDiagnoser? Diagnoser { get; } = diagnoser; - public int LaunchIndex { get; } + public int LaunchIndex { get; } = launchIndex; + + public RunMode DiagnoserRunMode { get; } = diagnoserRunMode; } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/Diagnosers/MockInProcessDiagnoser.cs b/tests/BenchmarkDotNet.IntegrationTests/Diagnosers/MockInProcessDiagnoser.cs new file mode 100644 index 0000000000..9098431a9b --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/Diagnosers/MockInProcessDiagnoser.cs @@ -0,0 +1,53 @@ +using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Validators; +using BenchmarkDotNet.Extensions; +using System.Collections.Generic; +using BenchmarkDotNet.Helpers; + +namespace BenchmarkDotNet.IntegrationTests.Diagnosers +{ + public sealed class MockInProcessDiagnoser : IInProcessDiagnoser + { + public Dictionary Results { get; } = []; + + public IEnumerable Ids => [nameof(MockInProcessDiagnoser)]; + + public IEnumerable Exporters => []; + + public IEnumerable Analysers => []; + + public void DeserializeResults(BenchmarkCase benchmarkCase, string results) => Results.Add(benchmarkCase, results); + + public void DisplayResults(ILogger logger) => logger.WriteLine($"{nameof(MockInProcessDiagnoser)} results: [{string.Join(", ", Results.Values)}]"); + + public IInProcessDiagnoserHandler GetHandler(BenchmarkCase benchmarkCase, int index) => new MockInProcessDiagnoserHandler(index, GetRunMode(benchmarkCase)); + + public string GetHandlerSourceCode(BenchmarkCase benchmarkCase, int index) + => $"new {typeof(MockInProcessDiagnoserHandler).GetCorrectCSharpTypeName()}({index}, {SourceCodeHelper.ToSourceCode(GetRunMode(benchmarkCase))})"; + + public RunMode GetRunMode(BenchmarkCase benchmarkCase) => RunMode.NoOverhead; + + public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { } + + public IEnumerable ProcessResults(DiagnoserResults results) => []; + + public IEnumerable Validate(ValidationParameters validationParameters) => []; + } + + public sealed class MockInProcessDiagnoserHandler(int index, RunMode runMode) : IInProcessDiagnoserHandler + { + public int Index { get; } = index; + + public RunMode RunMode { get; } = runMode; + + public void Handle(BenchmarkSignal signal, InProcessDiagnoserActionArgs parameters) { } + + public string SerializeResults() => $"DummyResult{Index}"; + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs index 488d76ca31..8c3a05429a 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs @@ -14,6 +14,9 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Tests.Loggers; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.InProcess.Emit; using Xunit; using Xunit.Abstractions; @@ -25,31 +28,26 @@ public DisassemblyDiagnoserTests(ITestOutputHelper output) : base(output) { } public static IEnumerable GetAllJits() { + yield return [RuntimeInformation.GetCurrentJit(), RuntimeInformation.GetCurrentPlatform(), InProcessEmitToolchain.DontLogOutput]; // InProcess + if (RuntimeInformation.IsFullFramework) { - yield return new object[] { Jit.LegacyJit, Platform.X86, ClrRuntime.Net462 }; // 32bit LegacyJit for desktop .NET - yield return new object[] { Jit.LegacyJit, Platform.X64, ClrRuntime.Net462 }; // 64bit LegacyJit for desktop .NET - yield return new object[] { Jit.RyuJit, Platform.X64, ClrRuntime.Net462 }; // RyuJit for desktop .NET + yield return [Jit.LegacyJit, Platform.X86, CsProjClassicNetToolchain.Net462]; // 32bit LegacyJit for desktop .NET + yield return [Jit.LegacyJit, Platform.X64, CsProjClassicNetToolchain.Net462]; // 64bit LegacyJit for desktop .NET + yield return [Jit.RyuJit, Platform.X64, CsProjClassicNetToolchain.Net462]; // RyuJit for desktop .NET } else if (RuntimeInformation.IsNetCore) { if (RuntimeInformation.GetCurrentPlatform() is Platform.X86 or Platform.X64) { - yield return new object[] { Jit.RyuJit, Platform.X64, CoreRuntime.Core80 }; // .NET Core x64 + yield return [Jit.RyuJit, Platform.X64, CsProjCoreToolchain.NetCoreApp80]; // .NET Core x64 + // We could add Platform.X86 here, but it would make our CI more complicated. } - else if (RuntimeInformation.GetCurrentPlatform() is Platform.Arm64 && OsDetector.IsLinux()) + else if (RuntimeInformation.GetCurrentPlatform() is Platform.Arm64) { - yield return new object[] { Jit.RyuJit, Platform.Arm64, CoreRuntime.Core80 }; // .NET Core arm64 + yield return [Jit.RyuJit, Platform.Arm64, CsProjCoreToolchain.NetCoreApp80]; // .NET Core arm64 } } - if (OsDetector.IsMacOS()) - { - // This scope of tests is not supported on macOS - // However, when the MemberData method provides no data, xUnit throws an "No data found" InvalidOperationException - // In order to fix the problem, we should provide at least one input data set - // All the tests check the OS on the first line and stop the test if it's macOS - yield return new object[] { Jit.Default, Platform.AnyCpu, CoreRuntime.Latest }; - } // we could add new object[] { Jit.Llvm, Platform.X64, new MonoRuntime() } here but our CI would need to have Mono installed.. } @@ -90,15 +88,12 @@ public void Recursive() [Theory] [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] - public void CanDisassembleAllMethodCalls(Jit jit, Platform platform, Runtime runtime) + public void CanDisassembleAllMethodCalls(Jit jit, Platform platform, IToolchain toolchain) { - if (OsDetector.IsMacOS()) return; // currently not supported - - var printSource = IsPrintSourceSupported(platform); var disassemblyDiagnoser = new DisassemblyDiagnoser( - new DisassemblyDiagnoserConfig(printSource: printSource, maxDepth: 3)); + new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 3)); - CanExecute(CreateConfig(jit, platform, runtime, disassemblyDiagnoser, RunStrategy.ColdStart)); + CanExecute(CreateConfig(jit, platform, toolchain, disassemblyDiagnoser, RunStrategy.ColdStart)); DisassemblyResult result = disassemblyDiagnoser.Results.Single().Value; @@ -113,15 +108,12 @@ public void CanDisassembleAllMethodCalls(Jit jit, Platform platform, Runtime run [Theory] [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] - public void CanDisassembleAllMethodCallsUsingFilters(Jit jit, Platform platform, Runtime runtime) + public void CanDisassembleAllMethodCallsUsingFilters(Jit jit, Platform platform, IToolchain toolchain) { - if (OsDetector.IsMacOS()) return; // currently not supported - - var printSource = IsPrintSourceSupported(platform); var disassemblyDiagnoser = new DisassemblyDiagnoser( - new DisassemblyDiagnoserConfig(printSource: printSource, maxDepth: 1, filters: new[] { "*WithCalls*" })); + new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 1, filters: new[] { "*WithCalls*" })); - CanExecute(CreateConfig(jit, platform, runtime, disassemblyDiagnoser, RunStrategy.ColdStart)); + CanExecute(CreateConfig(jit, platform, toolchain, disassemblyDiagnoser, RunStrategy.ColdStart)); DisassemblyResult result = disassemblyDiagnoser.Results.Single().Value; @@ -142,15 +134,12 @@ public void CanDisassembleAllMethodCallsUsingFilters(Jit jit, Platform platform, [Theory] [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] - public void CanDisassembleGenericTypes(Jit jit, Platform platform, Runtime runtime) + public void CanDisassembleGenericTypes(Jit jit, Platform platform, IToolchain toolchain) { - if (OsDetector.IsMacOS()) return; // currently not supported - - var printSource = IsPrintSourceSupported(platform); var disassemblyDiagnoser = new DisassemblyDiagnoser( - new DisassemblyDiagnoserConfig(printSource: printSource, maxDepth: 3)); + new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 3)); - CanExecute>(CreateConfig(jit, platform, runtime, disassemblyDiagnoser, RunStrategy.Monitoring)); + CanExecute>(CreateConfig(jit, platform, toolchain, disassemblyDiagnoser, RunStrategy.Monitoring)); var result = disassemblyDiagnoser.Results.Values.Single(); @@ -166,15 +155,12 @@ [Benchmark] public void JustReturn() { } [Theory] [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] - public void CanDisassembleInlinableBenchmarks(Jit jit, Platform platform, Runtime runtime) + public void CanDisassembleInlinableBenchmarks(Jit jit, Platform platform, IToolchain toolchain) { - if (OsDetector.IsMacOS()) return; // currently not supported - - var printSource = IsPrintSourceSupported(platform); var disassemblyDiagnoser = new DisassemblyDiagnoser( - new DisassemblyDiagnoserConfig(printSource: printSource, maxDepth: 3)); + new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 3)); - CanExecute(CreateConfig(jit, platform, runtime, disassemblyDiagnoser, RunStrategy.Monitoring)); + CanExecute(CreateConfig(jit, platform, toolchain, disassemblyDiagnoser, RunStrategy.Monitoring)); var disassemblyResult = disassemblyDiagnoser.Results.Values.Single(result => result.Methods.Count(method => method.Name.Contains(nameof(WithInlineable.JustReturn))) == 1); @@ -182,11 +168,11 @@ public void CanDisassembleInlinableBenchmarks(Jit jit, Platform platform, Runtim Assert.Contains(disassemblyResult.Methods, method => method.Maps.Any(map => map.SourceCodes.OfType().All(asm => asm.ToString().Contains("ret")))); } - private IConfig CreateConfig(Jit jit, Platform platform, Runtime runtime, IDiagnoser disassemblyDiagnoser, RunStrategy runStrategy) + private IConfig CreateConfig(Jit jit, Platform platform, IToolchain toolchain, IDiagnoser disassemblyDiagnoser, RunStrategy runStrategy) => ManualConfig.CreateEmpty() .AddJob(Job.Dry.WithJit(jit) .WithPlatform(platform) - .WithRuntime(runtime) + .WithToolchain(toolchain) .WithStrategy(runStrategy)) .AddLogger(DefaultConfig.Instance.GetLoggers().ToArray()) .AddColumnProvider(DefaultColumnProviders.Instance) @@ -198,8 +184,5 @@ private void AssertDisassemblyResult(DisassemblyResult result, string methodSign Assert.Contains(methodSignature, result.Methods.Select(m => m.Name.Split('.').Last()).ToArray()); Assert.Contains(result.Methods.Single(m => m.Name.EndsWith(methodSignature)).Maps, map => map.SourceCodes.Any()); } - - private static bool IsPrintSourceSupported(Platform platform) - => platform != Platform.X86; // Workaround for https://github.com/dotnet/BenchmarkDotNet/issues/2789 } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcess.EmitTests/Runnable_0.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcess.EmitTests/Runnable_0.cs index 4fe4fd8a2e..e4086c383f 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcess.EmitTests/Runnable_0.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcess.EmitTests/Runnable_0.cs @@ -1,19 +1,19 @@ using BenchmarkDotNet.Engines; -using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.InProcess.Emit.Implementation; +using BenchmarkDotNet.Toolchains.Parameters; // ReSharper disable once CheckNamespace namespace BenchmarkDotNet.Autogenerated.ReplaceMe { // Stub for diff for RunMethod // Used as a template for EmitRunMethod() - internal class Runnable0 + internal class Runnable_0 { - public static void Run(BenchmarkCase benchmarkCase, IHost host) + public static void Run(IHost host, ExecuteParameters parameters) { - var instance = new Runnable0(); + var instance = new Runnable_0(); - var (job, engineParameters, engineFactory) = RunnableReuse.PrepareForRun(instance, benchmarkCase, host); + var (job, engineParameters, engineFactory) = RunnableReuse.PrepareForRun(instance, host, parameters); if (job == null) return; @@ -26,6 +26,7 @@ public static void Run(BenchmarkCase benchmarkCase, IHost host) instance.__TrickTheJIT__(); // compile the method for disassembler, but without actual run of the benchmark ;) } + engineParameters.InProcessDiagnoserHandler.Handle(BenchmarkSignal.AfterEngine); } public void __TrickTheJIT__() diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs index 4b9ea2005b..0fb878f96c 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs @@ -6,6 +6,7 @@ using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Engines; +using BenchmarkDotNet.IntegrationTests.Diagnosers; using BenchmarkDotNet.IntegrationTests.InProcess.EmitTests; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; @@ -130,6 +131,19 @@ public void InProcessBenchmarkEmitsSameIL(Type benchmarkType) Assert.DoesNotContain("No benchmarks found", logger.GetLog()); } + [Fact] + public void InProcessEmitSupportsInProcessDiagnosers() + { + var logger = new OutputLogger(Output); + var diagnoser = new MockInProcessDiagnoser(); + var config = CreateInProcessConfig(logger).AddDiagnoser(diagnoser); + + var summary = CanExecute(config); + + var expected = Enumerable.Repeat("DummyResult0", summary.BenchmarksCases.Length); + Assert.Equal(expected, diagnoser.Results.Values); + } + [UsedImplicitly(ImplicitUseTargetFlags.WithMembers)] public class BenchmarkAllCases { diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs index 97c467a2bd..1e52fc5158 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs @@ -9,6 +9,7 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; +using BenchmarkDotNet.IntegrationTests.Diagnosers; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; @@ -217,6 +218,19 @@ public void InProcessBenchmarkAllCasesSupported() } } + [Fact] + public void InProcessNoEmitSupportsInProcessDiagnosers() + { + var logger = new OutputLogger(Output); + var diagnoser = new MockInProcessDiagnoser(); + var config = CreateInProcessConfig(logger).AddDiagnoser(diagnoser); + + var summary = CanExecute(config); + + var expected = Enumerable.Repeat("DummyResult0", summary.BenchmarksCases.Length); + Assert.Equal(expected, diagnoser.Results.Values); + } + [UsedImplicitly(ImplicitUseTargetFlags.WithMembers)] public class BenchmarkAllCases { diff --git a/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs b/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs index 169fb3a0b3..1909fef395 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs @@ -1,42 +1,51 @@ using System; +using System.Linq; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; +using BenchmarkDotNet.IntegrationTests.Diagnosers; using BenchmarkDotNet.IntegrationTests.Xunit; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains.NativeAot; +using Xunit; using Xunit.Abstractions; namespace BenchmarkDotNet.IntegrationTests { - public class NativeAotTests : BenchmarkTestExecutor + public class NativeAotTests(ITestOutputHelper outputHelper) : BenchmarkTestExecutor(outputHelper) { - public NativeAotTests(ITestOutputHelper outputHelper) : base(outputHelper) { } - - [FactEnvSpecific("It's impossible to reliably detect the version of NativeAOT if the process is not a .NET Core or NativeAOT process", EnvRequirement.DotNetCoreOnly)] - public void LatestNativeAotVersionIsSupported() + private bool IsAvx2Supported() { - if (!RuntimeInformation.Is64BitPlatform()) // NativeAOT does not support 32bit yet - return; - if (ContinuousIntegration.IsGitHubActionsOnWindows()) // no native dependencies installed - return; - if (OsDetector.IsMacOS()) - return; // currently not supported +#if NET6_0_OR_GREATER + return System.Runtime.Intrinsics.X86.Avx2.IsSupported; +#else + return false; +#endif + } + private ManualConfig GetConfig() + { var toolchain = NativeAotToolchain.CreateBuilder().UseNuGet().IlcInstructionSet(IsAvx2Supported() ? "avx2" : "").ToToolchain(); - var config = ManualConfig.CreateEmpty() + return ManualConfig.CreateEmpty() .AddJob(Job.Dry .WithRuntime(NativeAotRuntime.GetCurrentVersion()) // we test against latest version for current TFM to make sure we avoid issues like #1055 .WithToolchain(toolchain) .WithEnvironmentVariable(NativeAotBenchmark.EnvVarKey, IsAvx2Supported().ToString().ToLower())); + } + + [FactEnvSpecific("It's impossible to reliably detect the version of NativeAOT if the process is not a .NET Core or NativeAOT process", EnvRequirement.DotNetCoreOnly)] + public void LatestNativeAotVersionIsSupported() + { + if (!GetShouldRunTest()) + return; try { - CanExecute(config); + CanExecute(GetConfig()); } catch (MisconfiguredEnvironmentException e) { @@ -47,14 +56,36 @@ public void LatestNativeAotVersionIsSupported() } } - private bool IsAvx2Supported() + [FactEnvSpecific("It's impossible to reliably detect the version of NativeAOT if the process is not a .NET Core or NativeAOT process", EnvRequirement.DotNetCoreOnly)] + public void NativeAotSupportsInProcessDiagnosers() { -#if NET6_0_OR_GREATER - return System.Runtime.Intrinsics.X86.Avx2.IsSupported; -#else - return false; -#endif + if (!GetShouldRunTest()) + return; + + var diagnoser = new MockInProcessDiagnoser(); + var config = GetConfig().AddDiagnoser(diagnoser); + + try + { + CanExecute(config); + } + catch (MisconfiguredEnvironmentException e) + { + if (ContinuousIntegration.IsLocalRun()) + { + Output.WriteLine(e.SkipMessage); + return; + } + throw; + } + + Assert.Equal(["DummyResult0"], diagnoser.Results.Values); } + + private static bool GetShouldRunTest() + => RuntimeInformation.Is64BitPlatform() // NativeAOT does not support 32bit yet + && !ContinuousIntegration.IsGitHubActionsOnWindows() // no native dependencies installed + && !OsDetector.IsMacOS(); // currently not supported } public class NativeAotBenchmark diff --git a/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs index 6d3519a9c5..b4d1b8024e 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs @@ -1,14 +1,17 @@ using System; using System.IO; +using System.Linq; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; +using BenchmarkDotNet.IntegrationTests.Diagnosers; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Tests.Loggers; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains.DotNetCli; using BenchmarkDotNet.Toolchains.MonoWasm; +using Xunit; using Xunit.Abstractions; namespace BenchmarkDotNet.IntegrationTests @@ -23,23 +26,37 @@ namespace BenchmarkDotNet.IntegrationTests /// public class WasmTests(ITestOutputHelper output) : BenchmarkTestExecutor(output) { - [FactEnvSpecific("WASM is only supported on Unix", EnvRequirement.NonWindows)] - public void WasmIsSupported() + private ManualConfig GetConfig() { var dotnetVersion = "net8.0"; var logger = new OutputLogger(Output); var netCoreAppSettings = new NetCoreAppSettings(dotnetVersion, null, "Wasm"); var mainJsPath = Path.Combine(AppContext.BaseDirectory, "AppBundle", "test-main.js"); - var config = ManualConfig.CreateEmpty() + return ManualConfig.CreateEmpty() .AddLogger(logger) .AddJob(Job.Dry .WithArguments([new MsBuildArgument($"/p:WasmMainJSPath={mainJsPath}")]) .WithRuntime(new WasmRuntime(dotnetVersion, moniker: RuntimeMoniker.WasmNet80, javaScriptEngineArguments: "--expose_wasm --module")) .WithToolchain(WasmToolchain.From(netCoreAppSettings))) .WithOption(ConfigOptions.GenerateMSBuildBinLog, true); + } + + [FactEnvSpecific("WASM is only supported on Unix", EnvRequirement.NonWindows)] + public void WasmIsSupported() + { + CanExecute(GetConfig()); + } + + [FactEnvSpecific("WASM is only supported on Unix", EnvRequirement.NonWindows)] + public void WasmSupportsInProcessDiagnosers() + { + var diagnoser = new MockInProcessDiagnoser(); + var config = GetConfig().AddDiagnoser(diagnoser); CanExecute(config); + + Assert.Equal(["DummyResult0"], diagnoser.Results.Values); } public class WasmBenchmark diff --git a/tests/BenchmarkDotNet.Tests/CodeGeneratorTests.cs b/tests/BenchmarkDotNet.Tests/CodeGeneratorTests.cs index 0ff68a578f..f009f065b9 100644 --- a/tests/BenchmarkDotNet.Tests/CodeGeneratorTests.cs +++ b/tests/BenchmarkDotNet.Tests/CodeGeneratorTests.cs @@ -26,7 +26,10 @@ public void AsyncVoidIsNotSupported() var target = new Descriptor(typeof(CodeGeneratorTests), asyncVoidMethod); var benchmark = BenchmarkCase.Create(target, Job.Default, null, ManualConfig.CreateEmpty().CreateImmutableConfig()); - Assert.Throws(() => CodeGenerator.Generate(new BuildPartition(new[] { new BenchmarkBuildInfo(benchmark, ManualConfig.CreateEmpty().CreateImmutableConfig(), 0) }, BenchmarkRunnerClean.DefaultResolver))); + Assert.Throws(() => CodeGenerator.Generate(new BuildPartition( + [new BenchmarkBuildInfo(benchmark, ManualConfig.CreateEmpty().CreateImmutableConfig(), 0, new([]))], + BenchmarkRunnerClean.DefaultResolver) + )); } [Fact] @@ -41,7 +44,10 @@ public void UsingStatementsInTheAutoGeneratedCodeAreProhibited() var target = new Descriptor(typeof(CodeGeneratorTests), fineMethod); var benchmark = BenchmarkCase.Create(target, Job.Default, new ParameterInstances(Array.Empty()), ManualConfig.CreateEmpty().CreateImmutableConfig()); - var generatedSourceFile = CodeGenerator.Generate(new BuildPartition(new[] { new BenchmarkBuildInfo(benchmark, ManualConfig.CreateEmpty().CreateImmutableConfig(), 0) }, BenchmarkRunnerClean.DefaultResolver)); + var generatedSourceFile = CodeGenerator.Generate(new BuildPartition( + [new BenchmarkBuildInfo(benchmark, ManualConfig.CreateEmpty().CreateImmutableConfig(), 0, new([]))], + BenchmarkRunnerClean.DefaultResolver) + ); using (StringReader stringReader = new StringReader(generatedSourceFile)) { diff --git a/tests/BenchmarkDotNet.Tests/CsProjGeneratorTests.cs b/tests/BenchmarkDotNet.Tests/CsProjGeneratorTests.cs index ef1225432e..955298a634 100644 --- a/tests/BenchmarkDotNet.Tests/CsProjGeneratorTests.cs +++ b/tests/BenchmarkDotNet.Tests/CsProjGeneratorTests.cs @@ -195,7 +195,7 @@ public void TheDefaultFilePathShouldBeUsedWhenAnAssemblyLocationIsEmpty() var target = new Descriptor(assemblyType, MockFactory.MockMethodInfo); var benchmarkCase = BenchmarkCase.Create(target, Job.Default, null, config); - var benchmarks = new[] { new BenchmarkBuildInfo(benchmarkCase, config.CreateImmutableConfig(), 999) }; + var benchmarks = new[] { new BenchmarkBuildInfo(benchmarkCase, config.CreateImmutableConfig(), 999, new([])) }; var projectGenerator = new SteamLoadedBuildPartition("netcoreapp3.1", null, null, null, true); string binariesPath = projectGenerator.ResolvePathForBinaries(new BuildPartition(benchmarks, new Resolver()), programName); @@ -209,7 +209,7 @@ public void TestAssemblyFilePathIsUsedWhenTheAssemblyLocationIsNotEmpty() const string programName = "testProgram"; var target = new Descriptor(MockFactory.MockType, MockFactory.MockMethodInfo); var benchmarkCase = BenchmarkCase.Create(target, Job.Default, null, ManualConfig.CreateEmpty().CreateImmutableConfig()); - var benchmarks = new[] { new BenchmarkBuildInfo(benchmarkCase, ManualConfig.CreateEmpty().CreateImmutableConfig(), 0) }; + var benchmarks = new[] { new BenchmarkBuildInfo(benchmarkCase, ManualConfig.CreateEmpty().CreateImmutableConfig(), 0, new([])) }; var projectGenerator = new SteamLoadedBuildPartition("netcoreapp3.1", null, null, null, true); var buildPartition = new BuildPartition(benchmarks, new Resolver()); string binariesPath = projectGenerator.ResolvePathForBinaries(buildPartition, programName);