Skip to content

Commit 5cc751c

Browse files
palimondopalimondo
authored
Merge pull request #21717 from palimondo/cells-interlinked
[benchmark] BenchmarkDoctor: Lower runtime bound + Set.Empty fixes
2 parents 511286d + b4f1f47 commit 5cc751c

File tree

3 files changed

+82
-22
lines changed

3 files changed

+82
-22
lines changed

‎benchmark/scripts/Benchmark_Driver‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -435,14 +435,28 @@ class BenchmarkDoctor(object):
435435
"Decrease the workload of '%s' by a factor of %d (%d), to be "
436436
"less than %d μs.", name, factor(2), factor(10), threshold)
437437

438+
threshold = 20
439+
if runtime < threshold:
440+
log = (BenchmarkDoctor.log_runtime.error if runtime == 0 else
441+
BenchmarkDoctor.log_runtime.warning)
442+
log("'%s' execution took %d μs.", name, runtime)
443+
444+
BenchmarkDoctor.log_runtime.info(
445+
"Ensure the workload of '%s' has a properly measurable size"
446+
" (runtime > %d μs) and is not eliminated by the compiler (use"
447+
" `blackHole` function if necessary)." if runtime == 0 else
448+
"Increase the workload of '%s' to be more than %d μs.",
449+
name, threshold)
450+
438451
@staticmethod
439452
def _setup_overhead(measurements):
440453
select = BenchmarkDoctor._select
441454
ti1, ti2 = [float(min(mins)) for mins in
442455
[[result.samples.min for result in i_series]
443456
for i_series in
444457
[select(measurements, num_iters=i) for i in [1, 2]]]]
445-
setup = int(round(2.0 * (ti1 - ti2)))
458+
setup = (int(round(2.0 * (ti1 - ti2))) if ti2 > 20 # limit of accuracy
459+
else 0)
446460
ratio = (setup / ti1) if ti1 > 0 else 0
447461
return (setup, ratio)
448462

‎benchmark/scripts/test_Benchmark_Driver.py‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -667,7 +667,7 @@ def test_benchmark_name_is_at_most_40_chars_long(self):
667667
self.logs['info'])
668668

669669
def test_benchmark_runtime_range(self):
670-
"""Optimized benchmark should run in less then 1000 μs.
670+
"""Optimized benchmark should have runtime between 20 μs and 1000 μs.
671671
672672
Even on calm machine, benchmark with runtime of 2500 μs has 1:4 chance
673673
of being interrupted in the middle of measurement due to elapsed 10 ms
@@ -687,6 +687,8 @@ def measurements(name, runtime):
687687

688688
with captured_output() as (out, _):
689689
doctor = BenchmarkDoctor(self.args, BenchmarkDriverMock([]))
690+
doctor.analyze(measurements('Sylph', 0))
691+
doctor.analyze(measurements('Unicorn', 3))
690692
doctor.analyze(measurements('Cheetah', 200))
691693
doctor.analyze(measurements('Hare', 1001))
692694
doctor.analyze(measurements('Tortoise', 500000))
@@ -697,6 +699,18 @@ def measurements(name, runtime):
697699

698700
self.assertIn('runtime: ', output)
699701
self.assertNotIn('Cheetah', output)
702+
self.assert_contains(["'Sylph' execution took 0 μs."],
703+
self.logs['error'])
704+
self.assert_contains(
705+
["Ensure the workload of 'Sylph' has a properly measurable size"
706+
" (runtime > 20 μs) and is not eliminated by the compiler (use "
707+
"`blackHole` function if necessary)."],
708+
self.logs['info'])
709+
self.assert_contains(["'Unicorn' execution took 3 μs."],
710+
self.logs['warning'])
711+
self.assert_contains(
712+
["Increase the workload of 'Unicorn' to be more than 20 μs."],
713+
self.logs['info'])
700714
self.assert_contains(["'Hare' execution took at least 1001 μs."],
701715
self.logs['warning'])
702716
self.assert_contains(
@@ -728,11 +742,18 @@ def test_benchmark_has_no_significant_setup_overhead(self):
728742
'SO O i2a': _PTR(min=67), 'SO O i2b': _PTR(min=68)})
729743
doctor.analyze({'name': 'Zero', 'Zero O i1a': _PTR(min=0),
730744
'Zero O i2a': _PTR(min=0)})
745+
doctor.analyze({
746+
'name': 'LOA', # Limit of Accuracy
747+
# Impossible to detect overhead:
748+
# Even 1μs change in 20μs runtime is 5%.
749+
'LOA O i1a': _PTR(min=21),
750+
'LOA O i2a': _PTR(min=20)})
731751
output = out.getvalue()
732752

733753
self.assertIn('runtime: ', output)
734754
self.assertNotIn('NoOverhead', output)
735755
self.assertNotIn('ZeroRuntime', output)
756+
self.assertNotIn('LOA', output)
736757
self.assert_contains(
737758
["'SO' has setup overhead of 4 μs (5.8%)."],
738759
self.logs['error'])

‎benchmark/single-source/SetTests.swift‎

Lines changed: 45 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -58,10 +58,15 @@ let setQ: Set<Int> = {
5858
public let SetTests = [
5959
// Mnemonic: number after name is percentage of common elements in input sets.
6060
BenchmarkInfo(
61-
name: "Set.Empty.IsSubsetInt0",
61+
name: "Set.isSubset.Empty.Int",
6262
runFunction: { n in run_SetIsSubsetInt(setE, setAB, true, 5000 * n) },
6363
tags: [.validation, .api, .Set],
6464
setUpFunction: { blackHole([setE, setAB]) }),
65+
BenchmarkInfo(
66+
name: "Set.isSubset.Int.Empty",
67+
runFunction: { n in run_SetIsSubsetInt(setAB, setE, false, 5000 * n) },
68+
tags: [.validation, .api, .Set],
69+
setUpFunction: { blackHole([setAB, setE]) }),
6570
BenchmarkInfo(
6671
name: "SetIsSubsetInt0",
6772
runFunction: { n in run_SetIsSubsetInt(setAB, setCD, false, 5000 * n) },
@@ -94,46 +99,56 @@ public let SetTests = [
9499
setUpFunction: { blackHole([setP, setQ]) }),
95100

96101
BenchmarkInfo(
97-
name: "Set.Empty.IsDisjointInt0",
98-
runFunction: { n in run_SetIsDisjointInt(setE, setAB, true, 50 * n) },
102+
name: "Set.isDisjoint.Empty.Int",
103+
runFunction: { n in run_SetIsDisjointInt(setE, setAB, true, 5000 * n) },
99104
tags: [.validation, .api, .Set],
100105
setUpFunction: { blackHole([setE, setAB]) }),
101106
BenchmarkInfo(
102-
name: "Set.Empty.IsDisjointBox0",
103-
runFunction: { n in run_SetIsDisjointBox(setOE, setOAB, true, 50 * n) },
107+
name: "Set.isDisjoint.Int.Empty",
108+
runFunction: { n in run_SetIsDisjointInt(setAB, setE, true, 5000 * n) },
109+
tags: [.validation, .api, .Set],
110+
setUpFunction: { blackHole([setAB, setE]) }),
111+
BenchmarkInfo(
112+
name: "Set.isDisjoint.Empty.Box",
113+
runFunction: { n in run_SetIsDisjointBox(setOE, setOAB, true, 5000 * n) },
104114
tags: [.validation, .api, .Set],
105115
setUpFunction: { blackHole([setOE, setOAB]) }),
106116
BenchmarkInfo(
107-
name: "SetIsDisjointInt0",
117+
name: "Set.isDisjoint.Box.Empty",
118+
runFunction: { n in run_SetIsDisjointBox(setOAB, setOE, true, 5000 * n) },
119+
tags: [.validation, .api, .Set],
120+
setUpFunction: { blackHole([setOAB, setOE]) }),
121+
BenchmarkInfo(
122+
name: "Set.isDisjoint.Int0",
108123
runFunction: { n in run_SetIsDisjointInt(setAB, setCD, true, 50 * n) },
109124
tags: [.validation, .api, .Set],
110125
setUpFunction: { blackHole([setAB, setCD]) }),
111126
BenchmarkInfo(
112-
name: "SetIsDisjointBox0",
127+
name: "Set.isDisjoint.Box0",
113128
runFunction: { n in run_SetIsDisjointBox(setOAB, setOCD, true, 50 * n) },
114129
tags: [.validation, .api, .Set],
115130
setUpFunction: { blackHole([setOAB, setOCD]) }),
116131
BenchmarkInfo(
117-
name: "SetIsDisjointInt25",
118-
runFunction: { n in run_SetIsDisjointInt(setB, setAB, false, 50 * n) },
132+
name: "Set.isDisjoint.Int25",
133+
runFunction: { n in run_SetIsDisjointInt(setB, setAB, false, 5000 * n) },
119134
tags: [.validation, .api, .Set],
120135
setUpFunction: { blackHole([setB, setAB]) }),
121136
BenchmarkInfo(
122-
name: "SetIsDisjointBox25",
123-
runFunction: { n in run_SetIsDisjointBox(setOB, setOAB, false, 50 * n) },
137+
name: "Set.isDisjoint.Box25",
138+
runFunction: { n in run_SetIsDisjointBox(setOB, setOAB, false, 5000 * n) },
124139
tags: [.validation, .api, .Set],
125140
setUpFunction: { blackHole([setOB, setOAB]) }),
126141
BenchmarkInfo(
127-
name: "SetIsDisjointInt50",
128-
runFunction: { n in run_SetIsDisjointInt(setY, setXY, false, 50 * n) },
142+
name: "Set.isDisjoint.Int50",
143+
runFunction: { n in run_SetIsDisjointInt(setY, setXY, false, 5000 * n) },
129144
tags: [.validation, .api, .Set],
130145
setUpFunction: { blackHole([setY, setXY]) }),
131146
BenchmarkInfo(
132-
name: "SetIsDisjointInt100",
133-
runFunction: { n in run_SetIsDisjointInt(setP, setQ, false, 50 * n) },
147+
name: "Set.isDisjoint.Int100",
148+
runFunction: { n in run_SetIsDisjointInt(setP, setQ, false, 5000 * n) },
134149
tags: [.validation, .api, .Set],
135150
setUpFunction: { blackHole([setP, setQ]) }),
136-
151+
137152
BenchmarkInfo(
138153
name: "SetSymmetricDifferenceInt0",
139154
runFunction: { n in run_SetSymmetricDifferenceInt(setAB, setCD, countABCD, 10 * n) },
@@ -228,15 +243,25 @@ public let SetTests = [
228243
setUpFunction: { blackHole([setP, setQ]) }),
229244

230245
BenchmarkInfo(
231-
name: "Set.Empty.SubtractingInt0",
232-
runFunction: { n in run_SetSubtractingInt(setE, setAB, 0, 10 * n) },
246+
name: "Set.subtracting.Empty.Int",
247+
runFunction: { n in run_SetSubtractingInt(setE, setAB, 0, 1000 * n) },
233248
tags: [.validation, .api, .Set],
234249
setUpFunction: { blackHole([setE, setAB]) }),
235250
BenchmarkInfo(
236-
name: "Set.Empty.SubtractingBox0",
237-
runFunction: { n in run_SetSubtractingBox(setOE, setOAB, 0, 10 * n) },
251+
name: "Set.subtracting.Int.Empty",
252+
runFunction: { n in run_SetSubtractingInt(setAB, setE, countAB, 1000 * n) },
253+
tags: [.validation, .api, .Set],
254+
setUpFunction: { blackHole([setAB, setE]) }),
255+
BenchmarkInfo(
256+
name: "Set.subtracting.Empty.Box",
257+
runFunction: { n in run_SetSubtractingBox(setOE, setOAB, 0, 1000 * n) },
238258
tags: [.validation, .api, .Set],
239259
setUpFunction: { blackHole([setOE, setOAB]) }),
260+
BenchmarkInfo(
261+
name: "Set.subtracting.Box.Empty",
262+
runFunction: { n in run_SetSubtractingBox(setOAB, setOE, countAB, 1000 * n) },
263+
tags: [.validation, .api, .Set],
264+
setUpFunction: { blackHole([setOAB, setOE]) }),
240265
BenchmarkInfo(
241266
name: "SetSubtractingInt0",
242267
runFunction: { n in run_SetSubtractingInt(setAB, setCD, countAB, 10 * n) },

0 commit comments

Comments
 (0)