From d8b5acfc32e46fbef864f78cc262a3167db64e42 Mon Sep 17 00:00:00 2001 From: Allan Shortlidge Date: Thu, 5 Oct 2023 16:04:55 -0700 Subject: [PATCH 1/2] Introduce -emit-api-descriptor-path option. This option and its argument, when specified on the driver command line, should be passed through to jobs that emit modules and have access to the whole module. Resolves rdar://116538520 --- Sources/SwiftDriver/Driver/Driver.swift | 12 +++++ Sources/SwiftDriver/Jobs/CompileJob.swift | 5 +-- Sources/SwiftDriver/Jobs/EmitModuleJob.swift | 1 + .../SwiftDriver/Jobs/FrontendJobHelpers.swift | 6 +++ Sources/SwiftDriver/Utilities/FileType.swift | 16 +++++-- Sources/SwiftOptions/Options.swift | 2 + Tests/SwiftDriverTests/SwiftDriverTests.swift | 44 +++++++++++++++++++ 7 files changed, 80 insertions(+), 6 deletions(-) diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift index 8f59e4b0e..0cfc0e212 100644 --- a/Sources/SwiftDriver/Driver/Driver.swift +++ b/Sources/SwiftDriver/Driver/Driver.swift @@ -347,6 +347,9 @@ public struct Driver { /// Path to the module's digester baseline file. let digesterBaselinePath: VirtualPath.Handle? + /// Path to the emitted API descriptor file. + let apiDescriptorFilePath: VirtualPath.Handle? + /// The mode the API digester should run in. let digesterMode: DigesterMode @@ -954,6 +957,15 @@ public struct Driver { outputFileMap: self.outputFileMap, moduleName: moduleOutputInfo.name) + self.apiDescriptorFilePath = try Self.computeSupplementaryOutputPath( + &parsedOptions, type: .jsonAPIDescriptor, isOutputOptions: [], + outputPath: .emitApiDescriptorPath, + compilerOutputType: compilerOutputType, + compilerMode: compilerMode, + emitModuleSeparately: emitModuleSeparately, + outputFileMap: self.outputFileMap, + moduleName: moduleOutputInfo.name) + Self.validateDigesterArgs(&parsedOptions, moduleOutputInfo: moduleOutputInfo, digesterMode: self.digesterMode, diff --git a/Sources/SwiftDriver/Jobs/CompileJob.swift b/Sources/SwiftDriver/Jobs/CompileJob.swift index 15bc0d980..cde5fdd2e 100644 --- a/Sources/SwiftDriver/Jobs/CompileJob.swift +++ b/Sources/SwiftDriver/Jobs/CompileJob.swift @@ -97,7 +97,7 @@ extension Driver { .moduleTrace, .yamlOptimizationRecord, .bitstreamOptimizationRecord, .pcm, .pch, .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts, .indexUnitOutputPath, .modDepCache, .jsonAPIBaseline, .jsonABIBaseline, - .swiftConstValues, nil: + .swiftConstValues, .jsonAPIDescriptor, nil: return false } } @@ -364,7 +364,6 @@ extension Driver { } let expirementalFeatures = parsedOptions.arguments(for: .enableExperimentalFeature) - let embeddedEnabled = expirementalFeatures.map(\.argument).map(\.asSingle).contains("Embedded") try commandLine.appendLast(.trackSystemDependencies, from: &parsedOptions) try commandLine.appendLast(.CrossModuleOptimization, from: &parsedOptions) @@ -467,7 +466,7 @@ extension FileType { .bitstreamOptimizationRecord, .swiftInterface, .privateSwiftInterface, .swiftSourceInfoFile, .clangModuleMap, .jsonSwiftArtifacts, .indexUnitOutputPath, .modDepCache, .jsonAPIBaseline, .jsonABIBaseline, - .swiftConstValues: + .swiftConstValues, .jsonAPIDescriptor: fatalError("Output type can never be a primary output") } } diff --git a/Sources/SwiftDriver/Jobs/EmitModuleJob.swift b/Sources/SwiftDriver/Jobs/EmitModuleJob.swift index 468764d2f..f3ce52dac 100644 --- a/Sources/SwiftDriver/Jobs/EmitModuleJob.swift +++ b/Sources/SwiftDriver/Jobs/EmitModuleJob.swift @@ -34,6 +34,7 @@ extension Driver { addSupplementalOutput(path: swiftPrivateInterfacePath, flag: "-emit-private-module-interface-path", type: .privateSwiftInterface) addSupplementalOutput(path: objcGeneratedHeaderPath, flag: "-emit-objc-header-path", type: .objcHeader) addSupplementalOutput(path: tbdPath, flag: "-emit-tbd-path", type: .tbd) + addSupplementalOutput(path: apiDescriptorFilePath, flag: "-emit-api-descriptor-path", type: .jsonAPIDescriptor) if isMergeModule { return diff --git a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift index 1c93859cb..9050e3f49 100644 --- a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift +++ b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift @@ -592,6 +592,12 @@ extension Driver { input: nil, flag: "-emit-abi-descriptor-path") } + + try addOutputOfType( + outputType: .jsonAPIDescriptor, + finalOutputPath: apiDescriptorFilePath, + input: nil, + flag: "-emit-api-descriptor-path") } } diff --git a/Sources/SwiftDriver/Utilities/FileType.swift b/Sources/SwiftDriver/Utilities/FileType.swift index 72076d9b9..8acbd145e 100644 --- a/Sources/SwiftDriver/Utilities/FileType.swift +++ b/Sources/SwiftDriver/Utilities/FileType.swift @@ -151,6 +151,9 @@ public enum FileType: String, Hashable, CaseIterable, Codable { /// ABI baseline JSON case jsonABIBaseline = "abi.json" + + /// API descriptor JSON + case jsonAPIDescriptor } extension FileType: CustomStringConvertible { @@ -235,6 +238,9 @@ extension FileType: CustomStringConvertible { case .swiftConstValues: return "const-values" + + case .jsonAPIDescriptor: + return "api-descriptor-json" } } } @@ -254,7 +260,7 @@ extension FileType { .swiftInterface, .privateSwiftInterface, .swiftSourceInfoFile, .jsonDependencies, .clangModuleMap, .jsonTargetInfo, .jsonCompilerFeatures, .jsonSwiftArtifacts, .indexUnitOutputPath, .modDepCache, .jsonAPIBaseline, - .jsonABIBaseline, .swiftConstValues: + .jsonABIBaseline, .swiftConstValues, .jsonAPIDescriptor: return false } } @@ -359,6 +365,8 @@ extension FileType { return "abi-baseline-json" case .swiftConstValues: return "const-values" + case .jsonAPIDescriptor: + return "api-descriptor-json" } } } @@ -370,7 +378,8 @@ extension FileType { .raw_sil, .llvmIR,.objcHeader, .autolink, .importedModules, .tbd, .moduleTrace, .yamlOptimizationRecord, .swiftInterface, .privateSwiftInterface, .jsonDependencies, .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, - .jsonSwiftArtifacts, .jsonAPIBaseline, .jsonABIBaseline, .swiftConstValues: + .jsonSwiftArtifacts, .jsonAPIBaseline, .jsonABIBaseline, .swiftConstValues, + .jsonAPIDescriptor: return true case .image, .object, .dSYM, .pch, .sib, .raw_sib, .swiftModule, .swiftDocumentation, .swiftSourceInfoFile, .llvmBitcode, .diagnostics, @@ -393,7 +402,8 @@ extension FileType { .importedModules, .tbd, .moduleTrace, .indexData, .yamlOptimizationRecord, .modDepCache, .bitstreamOptimizationRecord, .pcm, .pch, .jsonDependencies, .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts, - .indexUnitOutputPath, .jsonAPIBaseline, .jsonABIBaseline, .swiftConstValues: + .indexUnitOutputPath, .jsonAPIBaseline, .jsonABIBaseline, .swiftConstValues, + .jsonAPIDescriptor: return false } } diff --git a/Sources/SwiftOptions/Options.swift b/Sources/SwiftOptions/Options.swift index 9d6068444..b7809a6ee 100644 --- a/Sources/SwiftOptions/Options.swift +++ b/Sources/SwiftOptions/Options.swift @@ -273,6 +273,7 @@ extension Option { public static let embedBitcode: Option = Option("-embed-bitcode", .flag, attributes: [.frontend, .noInteractive], helpText: "Embed LLVM IR bitcode as data") public static let embedTbdForModule: Option = Option("-embed-tbd-for-module", .separate, attributes: [.frontend], helpText: "Embed symbols from the module in the emitted tbd file") public static let emitAbiDescriptorPath: Option = Option("-emit-abi-descriptor-path", .separate, attributes: [.frontend, .noDriver, .cacheInvariant], metaVar: "", helpText: "Output the ABI descriptor of current module to ") + public static let emitApiDescriptorPath: Option = Option("-emit-api-descriptor-path", .separate, attributes: [.frontend, .noInteractive, .argumentIsPath, .supplementaryOutput, .cacheInvariant], metaVar: "", helpText: "Output the API descriptor of current module to ") public static let emitAssembly: Option = Option("-emit-assembly", .flag, attributes: [.frontend, .noInteractive, .doesNotAffectIncrementalBuild, .cacheInvariant], helpText: "Emit assembly file(s) (-S)", group: .modes) public static let emitAst: Option = Option("-emit-ast", .flag, alias: Option.dumpAst, attributes: [.frontend, .noInteractive, .doesNotAffectIncrementalBuild]) public static let emitBc: Option = Option("-emit-bc", .flag, attributes: [.frontend, .noInteractive, .doesNotAffectIncrementalBuild, .cacheInvariant], helpText: "Emit LLVM BC file(s)", group: .modes) @@ -1070,6 +1071,7 @@ extension Option { Option.embedBitcode, Option.embedTbdForModule, Option.emitAbiDescriptorPath, + Option.emitApiDescriptorPath, Option.emitAssembly, Option.emitAst, Option.emitBc, diff --git a/Tests/SwiftDriverTests/SwiftDriverTests.swift b/Tests/SwiftDriverTests/SwiftDriverTests.swift index 5bfd85eaf..d336a05e7 100644 --- a/Tests/SwiftDriverTests/SwiftDriverTests.swift +++ b/Tests/SwiftDriverTests/SwiftDriverTests.swift @@ -7325,6 +7325,35 @@ final class SwiftDriverTests: XCTestCase { let emitModuleJob = try XCTUnwrap(jobs.first(where: {$0.kind == .emitModule})) XCTAssertTrue(emitModuleJob.commandLine.contains(.flag("-experimental-lazy-typecheck"))) } + + func testEmitAPIDescriptorEmitModule() throws { + try withTemporaryDirectory { path in + let apiDescriptorPath = path.appending(component: "api.json").nativePathString(escaped: true) + var driver = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "baz.swift", + "-emit-module", "-module-name", "Test", + "-emit-api-descriptor-path", apiDescriptorPath]) + + let jobs = try driver.planBuild().removingAutolinkExtractJobs() + let emitModuleJob = try jobs.findJob(.emitModule) + XCTAssert(emitModuleJob.commandLine.contains(.flag("-emit-api-descriptor-path"))) + } + } + + func testEmitAPIDescriptorWholeModuleOptimization() throws { + try withTemporaryDirectory { path in + let apiDescriptorPath = path.appending(component: "api.json").nativePathString(escaped: true) + var driver = try Driver(args: ["swiftc", "-whole-module-optimization", + "-driver-filelist-threshold=0", + "foo.swift", "bar.swift", "baz.swift", + "-module-name", "Test", "-emit-module", + "-emit-api-descriptor-path", apiDescriptorPath]) + + let jobs = try driver.planBuild().removingAutolinkExtractJobs() + let compileJob = try jobs.findJob(.compile) + let supplementaryOutputs = try XCTUnwrap(compileJob.commandLine.supplementaryOutputFilemap) + XCTAssertNotNil(supplementaryOutputs.entries.values.first?[.jsonAPIDescriptor]) + } + } } func assertString( @@ -7394,4 +7423,19 @@ private extension Array where Element == Job.ArgTemplate { } } } + + var supplementaryOutputFilemap: OutputFileMap? { + get throws { + guard let argIdx = firstIndex(where: { $0 == .flag("-supplementary-output-file-map") }) else { + return nil + } + let supplementaryOutputs = self[argIdx + 1] + guard case let .path(path) = supplementaryOutputs, + case let .fileList(_, fileList) = path, + case let .outputFileMap(outputFileMap) = fileList else { + throw StringError("Unexpected argument for output file map") + } + return outputFileMap + } + } } From bab5b4ce47165c86ed762cbc72da47166569530a Mon Sep 17 00:00:00 2001 From: Allan Shortlidge Date: Mon, 9 Oct 2023 10:25:03 -0700 Subject: [PATCH 2/2] Simplify SwiftDriverTests by adopting supplementaryOutputFilemap utility. --- Tests/SwiftDriverTests/SwiftDriverTests.swift | 125 ++---------------- 1 file changed, 14 insertions(+), 111 deletions(-) diff --git a/Tests/SwiftDriverTests/SwiftDriverTests.swift b/Tests/SwiftDriverTests/SwiftDriverTests.swift index d336a05e7..2848293bd 100644 --- a/Tests/SwiftDriverTests/SwiftDriverTests.swift +++ b/Tests/SwiftDriverTests/SwiftDriverTests.swift @@ -2811,41 +2811,12 @@ final class SwiftDriverTests: XCTestCase { ]) let plannedJobs = try driver1.planBuild().removingAutolinkExtractJobs() XCTAssertEqual(plannedJobs.count, 1) - let suppleArg = "-supplementary-output-file-map" - // Make sure we are using supplementary file map - XCTAssert(plannedJobs[0].commandLine.contains(.flag(suppleArg))) - let args = plannedJobs[0].commandLine - var fileMapPath: VirtualPath? - for pair in args.enumerated() { - if pair.element == .flag(suppleArg) { - let filemap = args[pair.offset + 1] - switch filemap { - case .path(let p): - fileMapPath = p - default: - break - } - } - } - XCTAssert(fileMapPath != nil) - switch fileMapPath! { - case .fileList(_, let list): - switch list { - case .outputFileMap(let map): - // This is to match the legacy driver behavior - // Make sure the supplementary output map has an entry for the Swift file - // under indexing and its indexData entry is the primary output file - let entry = map.entries[VirtualPath.relative(try RelativePath(validating: "foo5.swift")).intern()]! - XCTAssert(VirtualPath.lookup(entry[.indexData]!) == .absolute(try .init(validating: "/tmp/t.o"))) - return - default: - break - } - break - default: - break - } - XCTAssert(false) + let map = try XCTUnwrap(plannedJobs[0].commandLine.supplementaryOutputFilemap) + // This is to match the legacy driver behavior + // Make sure the supplementary output map has an entry for the Swift file + // under indexing and its indexData entry is the primary output file + let entry = map.entries[VirtualPath.relative(try RelativePath(validating: "foo5.swift")).intern()]! + XCTAssert(VirtualPath.lookup(entry[.indexData]!) == .absolute(try .init(validating: "/tmp/t.o"))) } func testMultiThreadedWholeModuleOptimizationCompiles() throws { @@ -2917,14 +2888,7 @@ final class SwiftDriverTests: XCTestCase { XCTAssertEqual(plannedJobs.count, 2) let compileJob = plannedJobs[0] XCTAssertEqual(compileJob.kind, .compile) - XCTAssert(compileJob.commandLine.contains(.flag("-supplementary-output-file-map"))) - let argIdx = try XCTUnwrap(compileJob.commandLine.firstIndex(where: { $0 == .flag("-supplementary-output-file-map") })) - let supplOutputs = compileJob.commandLine[argIdx+1] - guard case let .path(path) = supplOutputs, - case let .fileList(_, fileList) = path, - case let .outputFileMap(outFileMap) = fileList else { - throw StringError("Unexpected argument for output file map") - } + let outFileMap = try XCTUnwrap(compileJob.commandLine.supplementaryOutputFilemap) let firstKey: String = try VirtualPath.lookup(XCTUnwrap(outFileMap.entries.keys.first)).description XCTAssertEqual(firstKey, "foo.swift") } @@ -3073,14 +3037,7 @@ final class SwiftDriverTests: XCTestCase { XCTAssertEqual(plannedJobs.count, 2) XCTAssertEqual(plannedJobs[0].kind, .compile) print(plannedJobs[0].commandLine.joinedUnresolvedArguments) - XCTAssert(plannedJobs[0].commandLine.contains(.flag("-supplementary-output-file-map"))) - let argIdx = try XCTUnwrap(plannedJobs[0].commandLine.firstIndex(where: { $0 == .flag("-supplementary-output-file-map") })) - let supplOutputs = plannedJobs[0].commandLine[argIdx+1] - guard case let .path(path) = supplOutputs, - case let .fileList(_, fileList) = path, - case let .outputFileMap(outFileMap) = fileList else { - throw StringError("Unexpected argument for output file map") - } + let outFileMap = try XCTUnwrap(plannedJobs[0].commandLine.supplementaryOutputFilemap) XCTAssertEqual(outFileMap.entries.values.first?.keys.first, fileType) } @@ -6636,48 +6593,21 @@ final class SwiftDriverTests: XCTestCase { let plannedJobs = try driver.planBuild() let jobA = plannedJobs[0] - let flagA = jobA.commandLine.firstIndex(of: .flag("-supplementary-output-file-map"))! - let fileListArgumentA = jobA.commandLine[jobA.commandLine.index(after: flagA)] - guard case let .path(.fileList(_, fileListA)) = fileListArgumentA else { - XCTFail("Argument wasn't a filelist") - return - } - guard case let .outputFileMap(mapA) = fileListA else { - XCTFail("FileList wasn't OutputFileMap") - return - } + let mapA = try XCTUnwrap(jobA.commandLine.supplementaryOutputFilemap) let filesA = try XCTUnwrap(mapA.entries[VirtualPath.relative(try RelativePath(validating: "a.swift")).intern()]) XCTAssertTrue(filesA.keys.contains(.swiftModule)) XCTAssertTrue(filesA.keys.contains(.swiftDocumentation)) XCTAssertTrue(filesA.keys.contains(.swiftSourceInfoFile)) let jobB = plannedJobs[1] - let flagB = jobB.commandLine.firstIndex(of: .flag("-supplementary-output-file-map"))! - let fileListArgumentB = jobB.commandLine[jobB.commandLine.index(after: flagB)] - guard case let .path(.fileList(_, fileListB)) = fileListArgumentB else { - XCTFail("Argument wasn't a filelist") - return - } - guard case let .outputFileMap(mapB) = fileListB else { - XCTFail("FileList wasn't OutputFileMap") - return - } + let mapB = try XCTUnwrap(jobB.commandLine.supplementaryOutputFilemap) let filesB = try XCTUnwrap(mapB.entries[VirtualPath.relative(try RelativePath(validating: "b.swift")).intern()]) XCTAssertTrue(filesB.keys.contains(.swiftModule)) XCTAssertTrue(filesB.keys.contains(.swiftDocumentation)) XCTAssertTrue(filesB.keys.contains(.swiftSourceInfoFile)) let jobC = plannedJobs[2] - let flagC = jobC.commandLine.firstIndex(of: .flag("-supplementary-output-file-map"))! - let fileListArgumentC = jobC.commandLine[jobC.commandLine.index(after: flagC)] - guard case let .path(.fileList(_, fileListC)) = fileListArgumentC else { - XCTFail("Argument wasn't a filelist") - return - } - guard case let .outputFileMap(mapC) = fileListC else { - XCTFail("FileList wasn't OutputFileMap") - return - } + let mapC = try XCTUnwrap(jobC.commandLine.supplementaryOutputFilemap) let filesC = try XCTUnwrap(mapC.entries[VirtualPath.relative(try RelativePath(validating: "c.swift")).intern()]) XCTAssertTrue(filesC.keys.contains(.swiftModule)) XCTAssertTrue(filesC.keys.contains(.swiftDocumentation)) @@ -6800,29 +6730,11 @@ final class SwiftDriverTests: XCTestCase { let plannedJobs = try driver.planBuild() let jobA = plannedJobs[0] - let flagA = jobA.commandLine.firstIndex(of: .flag("-supplementary-output-file-map"))! - let fileListArgumentA = jobA.commandLine[jobA.commandLine.index(after: flagA)] - guard case let .path(.fileList(_, fileListA)) = fileListArgumentA else { - XCTFail("Argument wasn't a filelist") - return - } - guard case let .outputFileMap(mapA) = fileListA else { - XCTFail("FileList wasn't OutputFileMap") - return - } + let mapA = try XCTUnwrap(jobA.commandLine.supplementaryOutputFilemap) XCTAssertEqual(mapA.entries, [VirtualPath.relative(try .init(validating: "a.swift")).intern(): [:]]) let jobB = plannedJobs[1] - let flagB = jobB.commandLine.firstIndex(of: .flag("-supplementary-output-file-map"))! - let fileListArgumentB = jobB.commandLine[jobB.commandLine.index(after: flagB)] - guard case let .path(.fileList(_, fileListB)) = fileListArgumentB else { - XCTFail("Argument wasn't a filelist") - return - } - guard case let .outputFileMap(mapB) = fileListB else { - XCTFail("FileList wasn't OutputFileMap") - return - } + let mapB = try XCTUnwrap(jobB.commandLine.supplementaryOutputFilemap) XCTAssertEqual(mapB.entries, [VirtualPath.relative(try .init(validating: "b.swift")).intern(): [:]]) } @@ -6831,16 +6743,7 @@ final class SwiftDriverTests: XCTestCase { let plannedJobs = try driver.planBuild() let jobA = plannedJobs[0] - let flagA = jobA.commandLine.firstIndex(of: .flag("-supplementary-output-file-map"))! - let fileListArgumentA = jobA.commandLine[jobA.commandLine.index(after: flagA)] - guard case let .path(.fileList(_, fileListA)) = fileListArgumentA else { - XCTFail("Argument wasn't a filelist") - return - } - guard case let .outputFileMap(mapA) = fileListA else { - XCTFail("FileList wasn't OutputFileMap") - return - } + let mapA = try XCTUnwrap(jobA.commandLine.supplementaryOutputFilemap) XCTAssertEqual(mapA.entries, [VirtualPath.relative(try .init(validating: "a.swift")).intern(): [:]]) } }