diff --git a/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs b/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs index 4ff5bf012a..d94087239c 100644 --- a/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs +++ b/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs @@ -1,4 +1,5 @@ using Perfolizer.Mathematics.SignificanceTesting; +using Perfolizer.Mathematics.Thresholds; namespace BenchmarkDotNet.Analysers { @@ -19,11 +20,11 @@ public static bool CheckZeroMeasurementOneSample(double[] results, double thresh /// Checks distribution against Zero Measurement hypothesis in case of two samples /// /// True if measurement is ZeroMeasurement - public static bool CheckZeroMeasurementTwoSamples(double[] workload, double[] overhead) + public static bool CheckZeroMeasurementTwoSamples(double[] workload, double[] overhead, Threshold threshold = null) { if (workload.Length < 3 || overhead.Length < 3) return false; - return !WelchTest.Instance.IsGreater(workload, overhead).NullHypothesisIsRejected; + return !WelchTest.Instance.IsGreater(workload, overhead, threshold).NullHypothesisIsRejected; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Properties/AssemblyInfo.cs b/src/BenchmarkDotNet/Properties/AssemblyInfo.cs index 21e7364a24..596c220645 100644 --- a/src/BenchmarkDotNet/Properties/AssemblyInfo.cs +++ b/src/BenchmarkDotNet/Properties/AssemblyInfo.cs @@ -14,9 +14,11 @@ [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.Windows,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotTrace,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] +[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] #else [assembly: InternalsVisibleTo("BenchmarkDotNet.Tests")] [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests")] [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.Windows")] [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotTrace")] +[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning")] #endif \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj index 77e3aa5cff..b0259a941e 100755 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj @@ -16,7 +16,6 @@ - @@ -26,6 +25,7 @@ + diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs new file mode 100644 index 0000000000..e36e43e0c3 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs @@ -0,0 +1,235 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Tests.XUnit; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using Perfolizer.Horology; +using Perfolizer.Mathematics.Thresholds; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.IntegrationTests.ManualRunning +{ + public class ExpectedBenchmarkResultsTests : BenchmarkTestExecutor + { + // NativeAot takes a long time to build, so not including it in these tests. + // We also don't test InProcessNoEmitToolchain because it is known to be less accurate than code-gen toolchains. + + private static readonly TimeInterval FallbackCpuResolutionValue = TimeInterval.FromNanoseconds(0.2d); + + public ExpectedBenchmarkResultsTests(ITestOutputHelper output) : base(output) { } + + private static IEnumerable EmptyBenchmarkTypes() => + new[] + { + typeof(EmptyVoid), + typeof(EmptyByte), + typeof(EmptySByte), + typeof(EmptyShort), + typeof(EmptyUShort), + typeof(EmptyChar), + typeof(EmptyInt32), + typeof(EmptyUInt32), + typeof(EmptyInt64), + typeof(EmptyUInt64), + typeof(EmptyIntPtr), + typeof(EmptyUIntPtr), + typeof(EmptyVoidPointer), + typeof(EmptyClass) + }; + + public static IEnumerable InProcessData() + { + foreach (var type in EmptyBenchmarkTypes()) + { + yield return new object[] { type }; + } + } + + public static IEnumerable CoreData() + { + foreach (var type in EmptyBenchmarkTypes()) + { + yield return new object[] { type, RuntimeMoniker.Net70 }; + yield return new object[] { type, RuntimeMoniker.Mono70 }; + } + } + + public static IEnumerable FrameworkData() + { + foreach (var type in EmptyBenchmarkTypes()) + { + yield return new object[] { type, RuntimeMoniker.Net462 }; + yield return new object[] { type, RuntimeMoniker.Mono }; + } + } + + [Theory] + [MemberData(nameof(InProcessData))] + public void EmptyBenchmarksReportZeroTimeAndAllocated_InProcess(Type benchmarkType) + { + AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty() + .AddJob(Job.Default + .WithToolchain(InProcessEmitToolchain.Instance) + )); + } + + [TheoryNetCoreOnly("To not repeat tests in both full Framework and Core")] + [MemberData(nameof(CoreData))] + public void EmptyBenchmarksReportZeroTimeAndAllocated_Core(Type benchmarkType, RuntimeMoniker runtimeMoniker) + { + AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty() + .AddJob(Job.Default + .WithRuntime(runtimeMoniker.GetRuntime()) + )); + } + + [TheoryFullFrameworkOnly("Can only run full Framework and Mono tests from Framework host")] + [MemberData(nameof(FrameworkData))] + public void EmptyBenchmarksReportZeroTimeAndAllocated_Framework(Type benchmarkType, RuntimeMoniker runtimeMoniker) + { + AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty() + .AddJob(Job.Default + .WithRuntime(runtimeMoniker.GetRuntime()) + )); + } + + private void AssertZeroResults(Type benchmarkType, IConfig config) + { + var summary = CanExecute(benchmarkType, config + .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(TimeUnit.Nanosecond)) + .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) + ); + + var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue; + var threshold = Threshold.Create(ThresholdUnit.Nanoseconds, cpuResolution.Nanoseconds); + + foreach (var report in summary.Reports) + { + var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().WithoutOutliers(); + var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().WithoutOutliers(); + + bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements, threshold); + Assert.True(isZero, $"Actual time was not 0."); + + isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements, threshold); + Assert.True(isZero, "Overhead took more time than workload."); + + Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0."); + } + } + + [Fact] + public void DifferentSizedStructsBenchmarksReportsNonZeroTimeAndZeroAllocated_InProcess() + { + AssertDifferentSizedStructsResults(ManualConfig.CreateEmpty() + .AddJob(Job.Default + .WithToolchain(InProcessEmitToolchain.Instance) + )); + } + + [TheoryNetCoreOnly("To not repeat tests in both full Framework and Core")] + [InlineData(RuntimeMoniker.Net70)] + [InlineData(RuntimeMoniker.Mono70)] + public void DifferentSizedStructsBenchmarksReportsNonZeroTimeAndZeroAllocated_Core(RuntimeMoniker runtimeMoniker) + { + AssertDifferentSizedStructsResults(ManualConfig.CreateEmpty() + .AddJob(Job.Default + .WithRuntime(runtimeMoniker.GetRuntime()) + )); + } + + [TheoryFullFrameworkOnly("Can only run full Framework and Mono tests from Framework host")] + [InlineData(RuntimeMoniker.Net462)] + [InlineData(RuntimeMoniker.Mono)] + public void DifferentSizedStructsBenchmarksReportsNonZeroTimeAndZeroAllocated_Framework(RuntimeMoniker runtimeMoniker) + { + AssertDifferentSizedStructsResults(ManualConfig.CreateEmpty() + .AddJob(Job.Default + .WithRuntime(runtimeMoniker.GetRuntime()) + )); + } + + private void AssertDifferentSizedStructsResults(IConfig config) + { + var summary = CanExecute(config + .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(TimeUnit.Nanosecond)) + .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) + ); + + var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue; + var threshold = Threshold.Create(ThresholdUnit.Nanoseconds, cpuResolution.Nanoseconds); + + foreach (var report in summary.Reports) + { + var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().WithoutOutliers(); + var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().WithoutOutliers(); + + bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements, threshold); + Assert.False(isZero, $"Actual time was 0."); + + isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements, threshold); + Assert.True(isZero, "Overhead took more time than workload."); + + Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0."); + } + } + } + + public struct Struct16 + { + public long l1, l2; + } + + public struct Struct32 + { + public long l1, l2, l3, l4; + } + + public struct Struct64 + { + public long l1, l2, l3, l4, + l5, l6, l7, l8; + } + + public struct Struct128 + { + public long l1, l2, l3, l4, + l5, l6, l7, l8, + l9, l10, l11, l12, + l13, l14, l15, l16; + } + + public class DifferentSizedStructs + { + [Benchmark] public Struct16 Struct16() => default; + [Benchmark] public Struct32 Struct32() => default; + [Benchmark] public Struct64 Struct64() => default; + [Benchmark] public Struct128 Struct128() => default; + } +} + +public class EmptyVoid { [Benchmark] public void Void() { } } +public class EmptyByte { [Benchmark] public byte Byte() => default; } +public class EmptySByte { [Benchmark] public sbyte SByte() => default; } +public class EmptyShort { [Benchmark] public short Short() => default; } +public class EmptyUShort { [Benchmark] public ushort UShort() => default; } +public class EmptyChar { [Benchmark] public char Char() => default; } +public class EmptyInt32 { [Benchmark] public int Int32() => default; } +public class EmptyUInt32 { [Benchmark] public uint UInt32() => default; } +public class EmptyInt64 { [Benchmark] public long Int64() => default; } +public class EmptyUInt64 { [Benchmark] public ulong UInt64() => default; } +public class EmptyIntPtr { [Benchmark] public IntPtr IntPtr() => default; } +public class EmptyUIntPtr { [Benchmark] public UIntPtr UIntPtr() => default; } +public class EmptyVoidPointer { [Benchmark] public unsafe void* VoidPointer() => default; } +public class EmptyClass { [Benchmark] public object Class() => default; } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs index 032e2643d8..1171b0f014 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs @@ -50,7 +50,7 @@ public Reports.Summary CanExecute(IConfig config = null, bool fullVa /// Optional custom config to be used instead of the default /// Optional: disable validation (default = true/enabled) /// The summary from the benchmark run - protected Reports.Summary CanExecute(Type type, IConfig config = null, bool fullValidation = true) + public Reports.Summary CanExecute(Type type, IConfig config = null, bool fullValidation = true) { // Add logging, so the Benchmark execution is in the TestRunner output (makes Debugging easier) if (config == null)