Skip to content

Commit 6910c00

Browse files
MaxDesiatovMaxDesiatov
committed
Add synthetic modules graph benchmarks with macros
Benchmarking modules graph is now generalized with `syntheticModulesGraph` function. It also uncovered a bug in the previous `SyntheticModulesGraph`, which didn't pass generated `TargetDescription`s array to `loadModulesGraph`, which is fixed now. New `SyntheticModulesGraphWithMacros` calls `syntheticModulesGraph` with `includeMacros: true` argument, which splits all modules in three parts: library modules, macros modules that library modules depend on, and macro dependencies that macros depend on. This allows us to track potential performance regressions in #7353.
1 parent d95de2e commit 6910c00

File tree

4 files changed

+107
-40
lines changed

4 files changed

+107
-40
lines changed

‎Benchmarks/Benchmarks/PackageGraphBenchmarks/PackageGraphBenchmarks.swift‎

Lines changed: 100 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,15 @@ let benchmarks = {
2727
let parsedValue = Int(envVar) {
2828
modulesGraphDepth = parsedValue
2929
} else {
30-
modulesGraphDepth = 100
30+
modulesGraphDepth = 150
3131
}
3232

3333
let modulesGraphWidth: Int
3434
if let envVar = ProcessInfo.processInfo.environment["SWIFTPM_BENCHMARK_MODULES_GRAPH_WIDTH"],
3535
let parsedValue = Int(envVar) {
3636
modulesGraphWidth = parsedValue
3737
} else {
38-
modulesGraphWidth = 100
38+
modulesGraphWidth = 150
3939
}
4040

4141
let packagesGraphDepth: Int
@@ -46,8 +46,6 @@ let benchmarks = {
4646
packagesGraphDepth = 10
4747
}
4848

49-
let noopObservability = ObservabilitySystem.NOOP
50-
5149
// Benchmarks computation of a resolved graph of modules for a package using `Workspace` as an entry point. It runs PubGrub to get
5250
// resolved concrete versions of dependencies, assigning all modules and products to each other as corresponding dependencies
5351
// with their build triples, but with the build plan not yet constructed. In this benchmark specifically we're loading `Package.swift`
@@ -67,55 +65,120 @@ let benchmarks = {
6765
let workspace = try Workspace(fileSystem: localFileSystem, location: .init(forRootPackage: path, fileSystem: localFileSystem))
6866

6967
for _ in benchmark.scaledIterations {
70-
try workspace.loadPackageGraph(rootPath: path, observabilityScope: noopObservability)
68+
try workspace.loadPackageGraph(rootPath: path, observabilityScope: ObservabilitySystem.NOOP)
7169
}
7270
}
7371

74-
75-
// Benchmarks computation of a resolved graph of modules for a synthesized package using `loadModulesGraph` as an
76-
// entry point, which almost immediately delegates to `ModulesGraph.load` under the hood.
72+
// Benchmarks computation of a resolved graph of modules for a trivial synthesized package using `loadModulesGraph`
73+
// as an entry point, which almost immediately delegates to `ModulesGraph.load` under the hood.
7774
Benchmark(
7875
"SyntheticModulesGraph",
7976
configuration: .init(
8077
metrics: defaultMetrics,
8178
maxDuration: .seconds(10),
8279
thresholds: [
83-
.mallocCountTotal: .init(absolute: [.p90: 2500]),
84-
.syscalls: .init(absolute: [.p90: 0]),
80+
.mallocCountTotal: .init(absolute: [.p90: 17000]),
81+
.syscalls: .init(absolute: [.p90: 5]),
8582
]
8683
)
8784
) { benchmark in
88-
let targets = try (0..<modulesGraphWidth).map { i in
89-
try TargetDescription(name: "Target\(i)", dependencies: (0..<min(i, modulesGraphDepth)).map {
90-
.target(name: "Target\($0)")
91-
})
92-
}
93-
let fileSystem = InMemoryFileSystem(
94-
emptyFiles: targets.map { "/benchmark/Sources/\($0.name)/empty.swift" }
85+
try syntheticModulesGraph(
86+
benchmark,
87+
modulesGraphDepth: modulesGraphDepth,
88+
modulesGraphWidth: modulesGraphWidth
9589
)
96-
let rootPackagePath = try AbsolutePath(validating: "/benchmark")
90+
}
9791

98-
let manifest = Manifest(
99-
displayName: "benchmark",
100-
path: rootPackagePath,
101-
packageKind: .root(rootPackagePath),
102-
packageLocation: rootPackagePath.pathString,
103-
defaultLocalization: nil,
104-
platforms: [],
105-
version: nil,
106-
revision: nil,
107-
toolsVersion: .v5_10,
108-
pkgConfig: nil,
109-
providers: nil,
110-
cLanguageStandard: nil,
111-
cxxLanguageStandard: nil,
112-
swiftLanguageVersions: nil
92+
// Benchmarks computation of a resolved graph of modules for a synthesized package that includes macros,
93+
// using `loadModulesGraph` as an entry point, which almost immediately delegates to `ModulesGraph.load` under
94+
// the hood.
95+
Benchmark(
96+
"SyntheticModulesGraphWithMacros",
97+
configuration: .init(
98+
metrics: defaultMetrics,
99+
maxDuration: .seconds(10),
100+
thresholds: [
101+
.mallocCountTotal: .init(absolute: [.p90: 8000]),
102+
.syscalls: .init(absolute: [.p90: 5]),
103+
]
113104
)
105+
) { benchmark in
106+
try syntheticModulesGraph(
107+
benchmark,
108+
modulesGraphDepth: modulesGraphDepth,
109+
modulesGraphWidth: modulesGraphWidth,
110+
includeMacros: true
111+
)
112+
}
113+
}
114114

115-
for _ in benchmark.scaledIterations {
116-
try blackHole(
117-
loadModulesGraph(fileSystem: fileSystem, manifests: [manifest], observabilityScope: noopObservability)
118-
)
115+
func syntheticModulesGraph(
116+
_ benchmark: Benchmark,
117+
modulesGraphDepth: Int,
118+
modulesGraphWidth: Int,
119+
includeMacros: Bool = false
120+
) throws {
121+
// If macros are included, modules are split in three parts:
122+
// 1. top-level modules
123+
// 2. macros
124+
// 3. dependencies of macros
125+
let macrosDenominator = includeMacros ? 3 : 1
126+
let libraryModules: [TargetDescription] = try (0..<(modulesGraphWidth / macrosDenominator)).map { i -> TargetDescription in
127+
let dependencies = (0..<min(i, modulesGraphDepth / macrosDenominator)).flatMap { i -> [TargetDescription.Dependency] in
128+
if includeMacros {
129+
[.target(name: "Module\(i)"), .target(name: "Macros\(i)")]
130+
} else {
131+
[.target(name: "Module\(i)")]
132+
}
133+
}
134+
return try TargetDescription(name: "Module\(i)", dependencies: dependencies)
135+
}
136+
137+
let macrosModules: [TargetDescription]
138+
let macrosDependenciesModules: [TargetDescription]
139+
if includeMacros {
140+
macrosModules = try (0..<modulesGraphWidth / macrosDenominator).map { i in
141+
try TargetDescription(name: "Macros\(i)", dependencies: (0..<min(i, modulesGraphDepth)).map {
142+
.target(name: "MacrosDependency\($0)")
143+
})
144+
}
145+
macrosDependenciesModules = try (0..<modulesGraphWidth / macrosDenominator).map { i in
146+
try TargetDescription(name: "MacrosDependency\(i)")
119147
}
148+
} else {
149+
macrosModules = []
150+
macrosDependenciesModules = []
151+
}
152+
153+
let modules = libraryModules + macrosModules + macrosDependenciesModules
154+
let fileSystem = InMemoryFileSystem(
155+
emptyFiles: modules.map {
156+
"/benchmark/Sources/\($0.name)/empty.swift"
157+
}
158+
)
159+
let rootPackagePath = try AbsolutePath(validating: "/benchmark")
160+
161+
let manifest = Manifest(
162+
displayName: "benchmark",
163+
path: rootPackagePath,
164+
packageKind: .root(rootPackagePath),
165+
packageLocation: rootPackagePath.pathString,
166+
defaultLocalization: nil,
167+
platforms: [],
168+
version: nil,
169+
revision: nil,
170+
toolsVersion: .v5_10,
171+
pkgConfig: nil,
172+
providers: nil,
173+
cLanguageStandard: nil,
174+
cxxLanguageStandard: nil,
175+
swiftLanguageVersions: nil,
176+
targets: modules
177+
)
178+
179+
for _ in benchmark.scaledIterations {
180+
try blackHole(
181+
loadModulesGraph(fileSystem: fileSystem, manifests: [manifest], observabilityScope: ObservabilitySystem.NOOP)
182+
)
120183
}
121184
}

‎Benchmarks/Thresholds/macosx-arm64/PackageGraphBenchmarks.SwiftPMWorkspaceModulesGraph.p90.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{
2-
"mallocCountTotal" : 10775,
3-
"syscalls" : 1508
2+
"mallocCountTotal" : 11255,
3+
"syscalls" : 1497
44
}

‎Benchmarks/Thresholds/macosx-arm64/PackageGraphBenchmarks.SyntheticModulesGraph.p90.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{
2-
"mallocCountTotal" : 2427,
2+
"mallocCountTotal" : 15679,
33
"syscalls" : 0
44
}

‎Benchmarks/Thresholds/macosx-arm64/PackageGraphBenchmarks.SyntheticModulesGraphWithMacros.p90.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"mallocCountTotal" : 7743,
3+
"syscalls" : 0
4+
}

0 commit comments

Comments
 (0)