Skip to content

Commit 3c22c00

Browse files
cachemeifyoucancachemeifyoucan
committed
[Caching] Support pluginCAS in swiftDriver
Add support for pluginCAS by adopting the new SwiftScan CAS creation APIs. The old CAS creation API is deprecated, and will be removed once all users are updated.
1 parent b921b97 commit 3c22c00

File tree

7 files changed

+149
-42
lines changed

7 files changed

+149
-42
lines changed

‎Sources/CSwiftScan/include/swiftscan_header.h‎

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
#include <stdint.h>
1919

2020
#define SWIFTSCAN_VERSION_MAJOR 0
21-
#define SWIFTSCAN_VERSION_MINOR 4
21+
#define SWIFTSCAN_VERSION_MINOR 5
2222

2323
//=== Public Scanner Data Types -------------------------------------------===//
2424

@@ -79,6 +79,7 @@ typedef void *swiftscan_scanner_t;
7979

8080
//=== CAS/Caching Specification -------------------------------------------===//
8181
typedef struct swiftscan_cas_s *swiftscan_cas_t;
82+
typedef struct swiftscan_cas_options_s *swiftscan_cas_options_t;
8283

8384
typedef enum {
8485
SWIFTSCAN_OUTPUT_TYPE_OBJECT = 0,
@@ -274,15 +275,24 @@ typedef struct {
274275
void (*swiftscan_scanner_cache_reset)(swiftscan_scanner_t scanner);
275276

276277
//=== Scanner CAS Operations ----------------------------------------------===//
277-
swiftscan_cas_t (*swiftscan_cas_create)(const char *path);
278+
swiftscan_cas_options_t (*swiftscan_cas_options_create)(void);
279+
void (*swiftscan_cas_options_dispose)(swiftscan_cas_options_t options);
280+
void (*swiftscan_cas_options_set_ondisk_path)(swiftscan_cas_options_t options,
281+
const char *path);
282+
void (*swiftscan_cas_options_set_plugin_path)(swiftscan_cas_options_t options,
283+
const char *path);
284+
bool (*swiftscan_cas_options_set_option)(swiftscan_cas_options_t options,
285+
const char *name, const char *value,
286+
swiftscan_string_ref_t *error);
287+
swiftscan_cas_t (*swiftscan_cas_create_from_options)(
288+
swiftscan_cas_options_t options, swiftscan_string_ref_t *error);
278289
void (*swiftscan_cas_dispose)(swiftscan_cas_t cas);
279290
swiftscan_string_ref_t (*swiftscan_cas_store)(swiftscan_cas_t cas,
280-
uint8_t *data, unsigned size);
281-
swiftscan_string_ref_t (*swiftscan_compute_cache_key)(swiftscan_cas_t cas,
282-
int argc,
283-
const char *argv,
284-
const char *input,
285-
swiftscan_output_kind_t);
291+
uint8_t *data, unsigned size,
292+
swiftscan_string_ref_t *error);
293+
swiftscan_string_ref_t (*swiftscan_compute_cache_key)(
294+
swiftscan_cas_t cas, int argc, const char *argv, const char *input,
295+
swiftscan_output_kind_t, swiftscan_string_ref_t *error);
286296

287297
} swiftscan_functions_t;
288298

‎Sources/SwiftDriver/Driver/Driver.swift‎

Lines changed: 29 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -270,7 +270,6 @@ public struct Driver {
270270
/// CAS/Caching related options.
271271
let enableCaching: Bool
272272
let useClangIncludeTree: Bool
273-
let casPath: String
274273

275274
/// Code & data for incremental compilation. Nil if not running in incremental mode.
276275
/// Set during planning because needs the jobs to look at outputs.
@@ -595,13 +594,6 @@ public struct Driver {
595594
let cachingEnableOverride = parsedOptions.hasArgument(.driverExplicitModuleBuild) && env.keys.contains("SWIFT_ENABLE_CACHING")
596595
self.enableCaching = parsedOptions.hasArgument(.cacheCompileJob) || cachingEnableOverride
597596
self.useClangIncludeTree = enableCaching && env.keys.contains("SWIFT_CACHING_USE_INCLUDE_TREE")
598-
if let casPathOpt = parsedOptions.getLastArgument(.casPath)?.asSingle {
599-
self.casPath = casPathOpt.description
600-
} else if let cacheEnv = env["CCHROOT"] {
601-
self.casPath = cacheEnv
602-
} else {
603-
self.casPath = ""
604-
}
605597

606598
// Compute the working directory.
607599
workingDirectory = try parsedOptions.getLastArgument(.workingDirectory).map { workingDirectoryArg in
@@ -3470,3 +3462,32 @@ extension Driver {
34703462
return try VirtualPath.intern(path: moduleName.appendingFileTypeExtension(type))
34713463
}
34723464
}
3465+
3466+
// CAS and Caching.
3467+
extension Driver {
3468+
mutating func getCASPluginPath() throws -> AbsolutePath? {
3469+
if let pluginOpt = parsedOptions.getLastArgument(.casPluginOption)?.asSingle {
3470+
return try AbsolutePath(validating: pluginOpt.description)
3471+
}
3472+
return try toolchain.lookupToolchainCASPluginLib()
3473+
}
3474+
3475+
mutating func getOnDiskCASPath() throws -> AbsolutePath? {
3476+
if let casPathOpt = parsedOptions.getLastArgument(.casPath)?.asSingle {
3477+
return try AbsolutePath(validating: casPathOpt.description)
3478+
}
3479+
return nil;
3480+
}
3481+
3482+
mutating func getCASPluginOptions() throws -> [String: String] {
3483+
var options : [String: String] = [:]
3484+
for opt in parsedOptions.arguments(for: .casPluginOption) {
3485+
let pluginArg = opt.argument.asSingle.split(separator: "=", maxSplits: 1)
3486+
if pluginArg.count != 2 {
3487+
throw Error.invalidArgumentValue(Option.casPluginOption.spelling, opt.argument.asSingle)
3488+
}
3489+
options[String(pluginArg[0])] = String(pluginArg[1])
3490+
}
3491+
return options
3492+
}
3493+
}

‎Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -171,11 +171,11 @@ public class InterModuleDependencyOracle {
171171
return diags.isEmpty ? nil : diags
172172
}
173173

174-
public func createCAS(path: String) throws {
174+
public func createCAS(pluginPath: AbsolutePath?, onDiskPath: AbsolutePath?, pluginOptions: [String: String]) throws {
175175
guard let swiftScan = swiftScanLibInstance else {
176176
fatalError("Attempting to reset scanner cache with no scanner instance.")
177177
}
178-
try swiftScan.createCAS(casPath: path)
178+
try swiftScan.createCAS(pluginPath: pluginPath?.pathString, onDiskPath: onDiskPath?.pathString, pluginOptions: pluginOptions)
179179
}
180180

181181
public func store(data: Data) throws -> String {

‎Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,9 @@ public extension Driver {
167167
}
168168
}
169169
if !fallbackToFrontend && enableCaching {
170-
try interModuleDependencyOracle.createCAS(path: casPath)
170+
try interModuleDependencyOracle.createCAS(pluginPath: try getCASPluginPath(),
171+
onDiskPath: try getOnDiskCASPath(),
172+
pluginOptions: try getCASPluginOptions())
171173
}
172174
return fallbackToFrontend
173175
}

‎Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -336,13 +336,16 @@ extension Driver {
336336
// CAS related options.
337337
if enableCaching {
338338
commandLine.appendFlag(.cacheCompileJob)
339-
if !casPath.isEmpty {
339+
if let casPath = try getOnDiskCASPath() {
340340
commandLine.appendFlag(.casPath)
341-
commandLine.appendFlag(casPath)
341+
commandLine.appendFlag(casPath.pathString)
342+
}
343+
if let pluginPath = try getCASPluginPath() {
344+
commandLine.appendFlag(.casPluginPath)
345+
commandLine.appendFlag(pluginPath.pathString)
342346
}
343-
try commandLine.appendLast(.cacheRemarks, from: &parsedOptions)
344-
try commandLine.appendLast(.casPluginPath, from: &parsedOptions)
345347
try commandLine.appendAll(.casPluginOption, from: &parsedOptions)
348+
try commandLine.appendLast(.cacheRemarks, from: &parsedOptions)
346349
}
347350
if useClangIncludeTree {
348351
commandLine.appendFlag(.clangIncludeTree)

‎Sources/SwiftDriver/SwiftScan/SwiftScan.swift‎

Lines changed: 61 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ public enum DependencyScanningError: Error, DiagnosticData {
2525
case missingRequiredSymbol(String)
2626
case dependencyScanFailed
2727
case failedToInstantiateScanner
28-
case failedToInstantiateCAS
28+
case casError(String)
2929
case missingField(String)
3030
case moduleNameDecodeFailure(String)
3131
case unsupportedDependencyDetailsKind(Int)
@@ -42,8 +42,8 @@ public enum DependencyScanningError: Error, DiagnosticData {
4242
return "libSwiftScan dependency scan query failed"
4343
case .failedToInstantiateScanner:
4444
return "libSwiftScan failed to create scanner instance"
45-
case .failedToInstantiateCAS:
46-
return "libSwiftScan failed to create CAS"
45+
case .casError(let reason):
46+
return "libSwiftScan CAS error: \(reason)"
4747
case .missingField(let fieldName):
4848
return "libSwiftScan scan result missing required field: `\(fieldName)`"
4949
case .moduleNameDecodeFailure(let encodedName):
@@ -281,7 +281,12 @@ internal extension swiftscan_diagnostic_severity_t {
281281
}
282282

283283
@_spi(Testing) public var supportsCaching : Bool {
284-
return api.swiftscan_cas_create != nil &&
284+
return api.swiftscan_cas_options_create != nil &&
285+
api.swiftscan_cas_options_dispose != nil &&
286+
api.swiftscan_cas_options_set_ondisk_path != nil &&
287+
api.swiftscan_cas_options_set_plugin_path != nil &&
288+
api.swiftscan_cas_options_set_option != nil &&
289+
api.swiftscan_cas_create_from_options != nil &&
285290
api.swiftscan_cas_dispose != nil &&
286291
api.swiftscan_compute_cache_key != nil &&
287292
api.swiftscan_cas_store != nil &&
@@ -382,20 +387,48 @@ internal extension swiftscan_diagnostic_severity_t {
382387
}
383388
}
384389

385-
func createCAS(casPath: String) throws {
386-
self.cas = api.swiftscan_cas_create(casPath.cString(using: String.Encoding.utf8))
387-
guard self.cas != nil else {
388-
throw DependencyScanningError.failedToInstantiateCAS
390+
private func handleCASError(_ closure: (inout swiftscan_string_ref_t) -> Bool) throws {
391+
var err_msg : swiftscan_string_ref_t = swiftscan_string_ref_t()
392+
guard !closure(&err_msg) else {
393+
let err_str = try toSwiftString(err_msg)
394+
api.swiftscan_string_dispose(err_msg)
395+
throw DependencyScanningError.casError(err_str)
396+
}
397+
}
398+
399+
func createCAS(pluginPath: String?, onDiskPath: String?, pluginOptions: [String: String]) throws {
400+
let casOpts = api.swiftscan_cas_options_create()
401+
defer {
402+
api.swiftscan_cas_options_dispose(casOpts)
403+
}
404+
if let path = pluginPath {
405+
api.swiftscan_cas_options_set_plugin_path(casOpts, path)
406+
}
407+
if let path = onDiskPath {
408+
api.swiftscan_cas_options_set_ondisk_path(casOpts, path)
409+
}
410+
for (name, value) in pluginOptions {
411+
try handleCASError { err_msg in
412+
return api.swiftscan_cas_options_set_option(casOpts, name, value, &err_msg)
413+
}
414+
}
415+
try handleCASError { err_msg in
416+
self.cas = api.swiftscan_cas_create_from_options(casOpts, &err_msg)
417+
return self.cas == nil
389418
}
390419
}
391420

392421
func store(data: Data) throws -> String {
393422
guard let scan_cas = self.cas else {
394-
throw DependencyScanningError.failedToInstantiateCAS
423+
throw DependencyScanningError.casError("cannot store into CAS because CAS is not yet created")
395424
}
396425
let bytes = UnsafeMutablePointer<UInt8>.allocate(capacity: data.count)
397426
data.copyBytes(to: bytes, count: data.count)
398-
let casid = api.swiftscan_cas_store(scan_cas, bytes, UInt32(data.count))
427+
var casid: swiftscan_string_ref_t = swiftscan_string_ref_t()
428+
try handleCASError { err_msg in
429+
casid = api.swiftscan_cas_store(scan_cas, bytes, UInt32(data.count), &err_msg)
430+
return casid.data == nil
431+
}
399432
return try toSwiftString(casid)
400433
}
401434

@@ -420,15 +453,19 @@ internal extension swiftscan_diagnostic_severity_t {
420453

421454
func computeCacheKeyForOutput(kind: FileType, commandLine: [String], input: String) throws -> String {
422455
guard let scan_cas = self.cas else {
423-
throw DependencyScanningError.failedToInstantiateCAS
456+
throw DependencyScanningError.casError("cannot compute CacheKey for compilation because CAS is not yet created")
424457
}
425-
var casid : swiftscan_string_ref_t = swiftscan_string_ref_t()
426-
withArrayOfCStrings(commandLine) { commandArray in
427-
casid = api.swiftscan_compute_cache_key(scan_cas,
428-
Int32(commandLine.count),
429-
commandArray,
430-
input.cString(using: String.Encoding.utf8),
431-
getSwiftScanOutputKind(kind: kind))
458+
var casid: swiftscan_string_ref_t = swiftscan_string_ref_t()
459+
try handleCASError { err_msg in
460+
withArrayOfCStrings(commandLine) { commandArray in
461+
casid = api.swiftscan_compute_cache_key(scan_cas,
462+
Int32(commandLine.count),
463+
commandArray,
464+
input.cString(using: String.Encoding.utf8),
465+
getSwiftScanOutputKind(kind: kind),
466+
&err_msg)
467+
}
468+
return casid.data == nil
432469
}
433470
return try toSwiftString(casid)
434471
}
@@ -513,7 +550,12 @@ private extension swiftscan_functions_t {
513550
self.swiftscan_clang_detail_get_module_cache_key =
514551
try loadOptional("swiftscan_clang_detail_get_module_cache_key")
515552

516-
self.swiftscan_cas_create = try loadOptional("swiftscan_cas_create")
553+
self.swiftscan_cas_options_create = try loadOptional("swiftscan_cas_options_create")
554+
self.swiftscan_cas_options_set_plugin_path = try loadOptional("swiftscan_cas_options_set_plugin_path")
555+
self.swiftscan_cas_options_set_ondisk_path = try loadOptional("swiftscan_cas_options_set_ondisk_path")
556+
self.swiftscan_cas_options_set_option = try loadOptional("swiftscan_cas_options_set_option")
557+
self.swiftscan_cas_options_dispose = try loadOptional("swiftscan_cas_options_dispose")
558+
self.swiftscan_cas_create_from_options = try loadOptional("swiftscan_cas_create_from_options")
517559
self.swiftscan_cas_dispose = try loadOptional("swiftscan_cas_dispose")
518560
self.swiftscan_compute_cache_key =
519561
try loadOptional("swiftscan_compute_cache_key")

‎Sources/SwiftDriver/Toolchains/Toolchain.swift‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,35 @@ extension Toolchain {
284284
#endif
285285
}
286286

287+
/// Looks for the executable in the `SWIFT_DRIVER_TOOLCHAIN_CASPLUGIN_LIB` environment variable, if found nothing,
288+
/// looks in the `lib` relative to the compiler executable.
289+
@_spi(Testing) public func lookupToolchainCASPluginLib() throws -> AbsolutePath? {
290+
if let overrideString = env["SWIFT_DRIVER_TOOLCHAIN_CASPLUGIN_LIB"],
291+
let path = try? AbsolutePath(validating: overrideString) {
292+
return path
293+
}
294+
#if os(Windows)
295+
return nil
296+
#else
297+
// Try to look for libToolchainCASPlugin in the developer dir, if found,
298+
// prefer using that. Otherwise, just return nil, and auto fallback to
299+
// builtin CAS.
300+
let libraryName = sharedLibraryName("libToolchainCASPlugin")
301+
let compilerPath = try getToolPath(.swiftCompiler)
302+
let developerPath = compilerPath.parentDirectory // bin
303+
.parentDirectory // toolchain root
304+
.parentDirectory // toolchains
305+
.parentDirectory // developer
306+
let libraryPath = developerPath.appending(component: "usr")
307+
.appending(component: "lib")
308+
.appending(component: libraryName)
309+
if fileSystem.isFile(libraryPath) {
310+
return libraryPath
311+
}
312+
return nil
313+
#endif
314+
}
315+
287316
private func xcrunFind(executable: String) throws -> AbsolutePath {
288317
let xcrun = "xcrun"
289318
guard lookupExecutablePath(filename: xcrun, searchPaths: searchPaths) != nil else {

0 commit comments

Comments
 (0)