Skip to content

Commit 6af9151

Browse files
timcasselltimcassell
committed
Added manual tests for expected results.
1 parent 1dd0ed5 commit 6af9151

File tree

4 files changed

+224
-2
lines changed

4 files changed

+224
-2
lines changed

‎src/BenchmarkDotNet/Properties/AssemblyInfo.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,11 @@
1414
[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests,PublicKey=" + BenchmarkDotNetInfo.PublicKey)]
1515
[assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.Windows,PublicKey=" + BenchmarkDotNetInfo.PublicKey)]
1616
[assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotTrace,PublicKey=" + BenchmarkDotNetInfo.PublicKey)]
17+
[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning,PublicKey=" + BenchmarkDotNetInfo.PublicKey)]
1718
#else
1819
[assembly: InternalsVisibleTo("BenchmarkDotNet.Tests")]
1920
[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests")]
2021
[assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.Windows")]
2122
[assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotTrace")]
23+
[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning")]
2224
#endif

‎tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
<Compile Include="..\BenchmarkDotNet.IntegrationTests\Xunit\MisconfiguredEnvironmentException.cs" Link="MisconfiguredEnvironmentException.cs" />
1717
<Compile Include="..\BenchmarkDotNet.IntegrationTests\Xunit\Extensions.cs" Link="Extensions.cs" />
1818
<Compile Include="..\BenchmarkDotNet.IntegrationTests\TestConfigs.cs" Link="TestConfigs.cs" />
19-
<Compile Include="..\BenchmarkDotNet.Tests\Loggers\OutputLogger.cs" Link="OutputLogger.cs" />
2019
</ItemGroup>
2120
<ItemGroup>
2221
<Content Include="xunit.runner.json">
@@ -26,6 +25,7 @@
2625
<ItemGroup>
2726
<ProjectReference Include="..\..\src\BenchmarkDotNet\BenchmarkDotNet.csproj" />
2827
<ProjectReference Include="..\..\src\BenchmarkDotNet.Diagnostics.Windows\BenchmarkDotNet.Diagnostics.Windows.csproj" />
28+
<ProjectReference Include="..\BenchmarkDotNet.Tests\BenchmarkDotNet.Tests.csproj" />
2929
</ItemGroup>
3030
<ItemGroup>
3131
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.2" />

‎tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs‎

Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using BenchmarkDotNet.Attributes;
5+
using BenchmarkDotNet.Configs;
6+
using BenchmarkDotNet.Diagnosers;
7+
using BenchmarkDotNet.Extensions;
8+
using BenchmarkDotNet.Jobs;
9+
using BenchmarkDotNet.Portability;
10+
using BenchmarkDotNet.Reports;
11+
using BenchmarkDotNet.Tests.XUnit;
12+
using BenchmarkDotNet.Toolchains.InProcess.Emit;
13+
using Xunit;
14+
using Xunit.Abstractions;
15+
16+
namespace BenchmarkDotNet.IntegrationTests.ManualRunning
17+
{
18+
public class ExpectedBenchmarkResultsTests : BenchmarkTestExecutor
19+
{
20+
// NativeAot takes a long time to build, so not including it in these tests.
21+
// We also don't test InProcessNoEmitToolchain because it is known to be less accurate than code-gen toolchains.
22+
23+
public ExpectedBenchmarkResultsTests(ITestOutputHelper output) : base(output) { }
24+
25+
private static IEnumerable<Type> EmptyBenchmarkTypes() =>
26+
new[]
27+
{
28+
typeof(EmptyVoid),
29+
typeof(EmptyByte),
30+
typeof(EmptySByte),
31+
typeof(EmptyShort),
32+
typeof(EmptyUShort),
33+
typeof(EmptyChar),
34+
typeof(EmptyInt32),
35+
typeof(EmptyUInt32),
36+
typeof(EmptyInt64),
37+
typeof(EmptyUInt64),
38+
typeof(EmptyIntPtr),
39+
typeof(EmptyUIntPtr),
40+
typeof(EmptyVoidPointer),
41+
typeof(EmptyClass)
42+
};
43+
44+
public static IEnumerable<object[]> InProcessData()
45+
{
46+
foreach (var type in EmptyBenchmarkTypes())
47+
{
48+
yield return new object[] { type };
49+
}
50+
}
51+
52+
public static IEnumerable<object[]> CoreData()
53+
{
54+
foreach (var type in EmptyBenchmarkTypes())
55+
{
56+
yield return new object[] { type, RuntimeMoniker.Net70 };
57+
yield return new object[] { type, RuntimeMoniker.Mono70 };
58+
}
59+
}
60+
61+
public static IEnumerable<object[]> FrameworkData()
62+
{
63+
foreach (var type in EmptyBenchmarkTypes())
64+
{
65+
yield return new object[] { type, RuntimeMoniker.Net462 };
66+
yield return new object[] { type, RuntimeMoniker.Mono };
67+
}
68+
}
69+
70+
[Theory]
71+
[MemberData(nameof(InProcessData))]
72+
public void EmptyBenchmarksReportZeroTimeAndAllocated_InProcess(Type benchmarkType)
73+
{
74+
AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty()
75+
.AddJob(Job.Default
76+
.WithToolchain(InProcessEmitToolchain.Instance)
77+
));
78+
}
79+
80+
[TheoryNetCoreOnly("To not repeat tests in both full Framework and Core")]
81+
[MemberData(nameof(CoreData))]
82+
public void EmptyBenchmarksReportZeroTimeAndAllocated_Core(Type benchmarkType, RuntimeMoniker runtimeMoniker)
83+
{
84+
AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty()
85+
.AddJob(Job.Default
86+
.WithRuntime(runtimeMoniker.GetRuntime())
87+
));
88+
}
89+
90+
[TheoryFullFrameworkOnly("Can only run full Framework and Mono tests from Framework host")]
91+
[MemberData(nameof(FrameworkData))]
92+
public void EmptyBenchmarksReportZeroTimeAndAllocated_Framework(Type benchmarkType, RuntimeMoniker runtimeMoniker)
93+
{
94+
AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty()
95+
.AddJob(Job.Default
96+
.WithRuntime(runtimeMoniker.GetRuntime())
97+
));
98+
}
99+
100+
private void AssertZeroResults(Type benchmarkType, IConfig config)
101+
{
102+
var summary = CanExecute(benchmarkType, config
103+
.WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(Perfolizer.Horology.TimeUnit.Nanosecond))
104+
.AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false)))
105+
);
106+
107+
var cpuGhz = RuntimeInformation.GetCpuInfo().MaxFrequency.Value.ToGHz();
108+
109+
foreach (var report in summary.Reports)
110+
{
111+
Assert.True(cpuGhz * report.ResultStatistics.Mean < 1, $"Actual time was greater than 1 clock cycle.");
112+
113+
var overheadTime = report.AllMeasurements
114+
.Where(m => m.IsOverhead() && m.IterationStage == Engines.IterationStage.Actual)
115+
.Select(m => m.GetAverageTime().Nanoseconds)
116+
.Average();
117+
118+
var workloadTime = report.AllMeasurements
119+
.Where(m => m.IsWorkload() && m.IterationStage == Engines.IterationStage.Actual)
120+
.Select(m => m.GetAverageTime().Nanoseconds)
121+
.Average();
122+
123+
// Allow for 1 cpu cycle variance
124+
Assert.True(overheadTime * cpuGhz < workloadTime * cpuGhz + 1, "Overhead took more time than workload.");
125+
126+
Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0.");
127+
}
128+
}
129+
130+
[Fact]
131+
public void LargeStructBenchmarksReportsNonZeroTimeAndZeroAllocated_InProcess()
132+
{
133+
AssertLargeStructResults(ManualConfig.CreateEmpty()
134+
.AddJob(Job.Default
135+
.WithToolchain(InProcessEmitToolchain.Instance)
136+
));
137+
}
138+
139+
[TheoryNetCoreOnly("To not repeat tests in both full Framework and Core")]
140+
[InlineData(RuntimeMoniker.Net70)]
141+
[InlineData(RuntimeMoniker.Mono70)]
142+
public void LargeStructBenchmarksReportsNonZeroTimeAndZeroAllocated_Core(RuntimeMoniker runtimeMoniker)
143+
{
144+
AssertLargeStructResults(ManualConfig.CreateEmpty()
145+
.AddJob(Job.Default
146+
.WithRuntime(runtimeMoniker.GetRuntime())
147+
));
148+
}
149+
150+
[TheoryFullFrameworkOnly("Can only run full Framework and Mono tests from Framework host")]
151+
[InlineData(RuntimeMoniker.Net462)]
152+
[InlineData(RuntimeMoniker.Mono)]
153+
public void LargeStructBenchmarksReportsNonZeroTimeAndZeroAllocated_Framework(RuntimeMoniker runtimeMoniker)
154+
{
155+
AssertLargeStructResults(ManualConfig.CreateEmpty()
156+
.AddJob(Job.Default
157+
.WithRuntime(runtimeMoniker.GetRuntime())
158+
));
159+
}
160+
161+
private void AssertLargeStructResults(IConfig config)
162+
{
163+
var summary = CanExecute<LargeStruct>(config
164+
.WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(Perfolizer.Horology.TimeUnit.Nanosecond))
165+
.AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false)))
166+
);
167+
168+
var cpuGhz = RuntimeInformation.GetCpuInfo().MaxFrequency.Value.ToGHz();
169+
170+
foreach (var report in summary.Reports)
171+
{
172+
Assert.True(cpuGhz * report.ResultStatistics.Mean >= 1, $"Actual time was less than 1 clock cycle.");
173+
174+
var overheadTime = report.AllMeasurements
175+
.Where(m => m.IsOverhead() && m.IterationStage == Engines.IterationStage.Actual)
176+
.Select(m => m.GetAverageTime().Nanoseconds)
177+
.Average();
178+
179+
var workloadTime = report.AllMeasurements
180+
.Where(m => m.IsWorkload() && m.IterationStage == Engines.IterationStage.Actual)
181+
.Select(m => m.GetAverageTime().Nanoseconds)
182+
.Average();
183+
184+
// Allow for 1 cpu cycle variance
185+
Assert.True(overheadTime * cpuGhz < workloadTime * cpuGhz + 1, "Overhead took more time than workload.");
186+
187+
Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0.");
188+
}
189+
}
190+
}
191+
192+
public class LargeStruct
193+
{
194+
public struct Struct
195+
{
196+
// 128 bits
197+
public long l1, l2, l3, l4,
198+
l5, l6, l7, l8,
199+
l9, l10, l11, l12,
200+
l13, l14, l15, l16;
201+
}
202+
203+
[Benchmark] public Struct Benchmark() => default;
204+
}
205+
}
206+
207+
public class EmptyVoid { [Benchmark] public void Benchmark() { } }
208+
public class EmptyByte { [Benchmark] public byte Benchmark() => default; }
209+
public class EmptySByte { [Benchmark] public sbyte Benchmark() => default; }
210+
public class EmptyShort { [Benchmark] public short Benchmark() => default; }
211+
public class EmptyUShort { [Benchmark] public ushort Benchmark() => default; }
212+
public class EmptyChar { [Benchmark] public char Benchmark() => default; }
213+
public class EmptyInt32 { [Benchmark] public int Benchmark() => default; }
214+
public class EmptyUInt32 { [Benchmark] public uint Benchmark() => default; }
215+
public class EmptyInt64 { [Benchmark] public long Benchmark() => default; }
216+
public class EmptyUInt64 { [Benchmark] public ulong Benchmark() => default; }
217+
public class EmptyIntPtr { [Benchmark] public IntPtr Benchmark() => default; }
218+
public class EmptyUIntPtr { [Benchmark] public UIntPtr Benchmark() => default; }
219+
public class EmptyVoidPointer { [Benchmark] public unsafe void* Benchmark() => default; }
220+
public class EmptyClass { [Benchmark] public object Class() => default; }

‎tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ public Reports.Summary CanExecute<TBenchmark>(IConfig config = null, bool fullVa
5050
/// <param name="config">Optional custom config to be used instead of the default</param>
5151
/// <param name="fullValidation">Optional: disable validation (default = true/enabled)</param>
5252
/// <returns>The summary from the benchmark run</returns>
53-
protected Reports.Summary CanExecute(Type type, IConfig config = null, bool fullValidation = true)
53+
public Reports.Summary CanExecute(Type type, IConfig config = null, bool fullValidation = true)
5454
{
5555
// Add logging, so the Benchmark execution is in the TestRunner output (makes Debugging easier)
5656
if (config == null)

0 commit comments

Comments
 (0)