From adce1337b5dba9610401e932d1dcf28a0bf114e6 Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Thu, 6 Jun 2024 15:17:22 -0700 Subject: [PATCH 1/3] [Explicit Module Builds][Incremental Builds] Re-compile module dependnecies whose dependencies are up-to-date themselves but are themselves newer For example consider the following module graph: test \ J \ G Where on an incremental build we detect that although G binary module product is *newer* than its textual source, said binary module product is also *newer* than a prior binary module product of J. Which means that although each of the modules is up-to-date with respect to its own textual source inputs, J's binary dependnecy input has been updated elsewhere and J needs to be re-built. Resolves rdar://129225956 --- .../FirstWaveComputer.swift | 30 +++++++--- .../IncrementalCompilationTests.swift | 60 ++++++++++++++++++- 2 files changed, 80 insertions(+), 10 deletions(-) diff --git a/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift b/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift index aa94cfdac..153f709e1 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift @@ -149,12 +149,12 @@ extension IncrementalCompilationState.FirstWaveComputer { throws -> Set { let mainModuleInfo = moduleDependencyGraph.mainModule var modulesRequiringRebuild: Set = [] - var visitedModules: Set = [] + var visited: Set = [] // Scan from the main module's dependencies to avoid reporting // the main module itself in the results. for dependencyId in mainModuleInfo.directDependencies ?? [] { - try outOfDateModuleScan(on: moduleDependencyGraph, from: dependencyId, visited: &visitedModules, - modulesRequiringRebuild: &modulesRequiringRebuild) + try outOfDateModuleScan(on: moduleDependencyGraph, from: dependencyId, + visited: &visited, modulesRequiringRebuild: &modulesRequiringRebuild) } reporter?.reportExplicitDependencyReBuildSet(Array(modulesRequiringRebuild)) @@ -171,24 +171,36 @@ extension IncrementalCompilationState.FirstWaveComputer { let moduleInfo = try moduleDependencyGraph.moduleInfo(of: moduleId) // Visit the module's dependencies var hasOutOfDateModuleDependency = false + var mostRecentlyUpdatedDependencyOutput: TimePoint = .zero for dependencyId in moduleInfo.directDependencies ?? [] { // If we have not already visited this module, recurse. if !visited.contains(dependencyId) { try outOfDateModuleScan(on: moduleDependencyGraph, from: dependencyId, - visited: &visited, - modulesRequiringRebuild: &modulesRequiringRebuild) + visited: &visited, modulesRequiringRebuild: &modulesRequiringRebuild) } // Even if we're not revisiting a dependency, we must check if it's already known to be out of date. hasOutOfDateModuleDependency = hasOutOfDateModuleDependency || modulesRequiringRebuild.contains(dependencyId) + + // Keep track of dependencies' output file time stamp to determine if it is newer than the current module. + if let depOutputTimeStamp = try? fileSystem.lastModificationTime(for: VirtualPath.lookup(moduleDependencyGraph.moduleInfo(of: dependencyId).modulePath.path)), + depOutputTimeStamp > mostRecentlyUpdatedDependencyOutput { + mostRecentlyUpdatedDependencyOutput = depOutputTimeStamp + } } if hasOutOfDateModuleDependency { - reporter?.reportExplicitDependencyWillBeReBuilt(moduleId.moduleNameForDiagnostic, reason: "Invalidated by downstream dependency") - modulesRequiringRebuild.insert(moduleId) + reporter?.reportExplicitDependencyWillBeReBuilt(moduleId.moduleNameForDiagnostic, reason: "Invalidated by downstream dependency") + modulesRequiringRebuild.insert(moduleId) } else if try !IncrementalCompilationState.IncrementalDependencyAndInputSetup.verifyModuleDependencyUpToDate(moduleID: moduleId, moduleInfo: moduleInfo, fileSystem: fileSystem, reporter: reporter) { - reporter?.reportExplicitDependencyWillBeReBuilt(moduleId.moduleNameForDiagnostic, reason: "Out-of-date") - modulesRequiringRebuild.insert(moduleId) + reporter?.reportExplicitDependencyWillBeReBuilt(moduleId.moduleNameForDiagnostic, reason: "Out-of-date") + modulesRequiringRebuild.insert(moduleId) + } else if let outputModTime = try? fileSystem.lastModificationTime(for: VirtualPath.lookup(moduleInfo.modulePath.path)), + outputModTime < mostRecentlyUpdatedDependencyOutput { + // If a prior variant of this module dependnecy exists, and is older than any of its direct or transitive + // module dependency outputs, it must also be re-built. + reporter?.reportExplicitDependencyWillBeReBuilt(moduleId.moduleNameForDiagnostic, reason: "Has newer module dependency inputs") + modulesRequiringRebuild.insert(moduleId) } // Now that we've determined if this module must be rebuilt, mark it as visited. diff --git a/Tests/SwiftDriverTests/IncrementalCompilationTests.swift b/Tests/SwiftDriverTests/IncrementalCompilationTests.swift index c9a33249e..d68ce18c6 100644 --- a/Tests/SwiftDriverTests/IncrementalCompilationTests.swift +++ b/Tests/SwiftDriverTests/IncrementalCompilationTests.swift @@ -382,7 +382,6 @@ extension IncrementalCompilationTests { // On this graph, inputs of 'G' are updated, causing it to be re-built // as well as all modules on paths from root to it: 'Y', 'H', 'T','J' func testExplicitIncrementalBuildChangedDependencyInvalidatesUpstreamDependencies() throws { - // Add an import of 'B', 'C' to make sure followup changes has consistent inputs replace(contentsOf: "other", with: "import Y;import T") try buildInitialState(checkDiagnostics: false, explicitModuleBuild: true) @@ -431,6 +430,62 @@ extension IncrementalCompilationTests { linking } } + + // A dependency has been re-built to be newer than its dependents + // so we must ensure the dependents get re-built even though all the + // modules are up-to-date with respect to their textual source inputs. + // + // test + // \ + // J + // \ + // G + // + // On this graph, after the initial build, if G module binary file is newer + // than that of J, even if each of the modules is up-to-date w.r.t. their source inputs + // we still expect that J gets re-built + func testExplicitIncrementalBuildChangedDependencyBinaryInvalidatesUpstreamDependencies() throws { + replace(contentsOf: "other", with: "import J;") + try buildInitialState(checkDiagnostics: false, explicitModuleBuild: true) + + let modCacheEntries = try localFileSystem.getDirectoryContents(explicitModuleCacheDir) + let nameOfGModule = try XCTUnwrap(modCacheEntries.first { $0.hasPrefix("G") && $0.hasSuffix(".swiftmodule")}) + let pathToGModule = explicitModuleCacheDir.appending(component: nameOfGModule) + // Just update the time-stamp of one of the module dependencies' outputs. + // Also add a dependency to cause a re-scan. + touch(pathToGModule) + replace(contentsOf: "other", with: "import J;import R") + + // Changing a dependency will mean that we both re-run the dependency scan, + // and also ensure that all source-files are re-built with a non-cascading build + // since the source files themselves have not changed. + try doABuild( + "update dependency (G) result timestamp", + checkDiagnostics: true, + extraArguments: explicitBuildArgs, + whenAutolinking: autolinkLifecycleExpectedDiags + ) { + readGraph + enablingCrossModule + readInterModuleGraph + explicitMustReScanDueToChangedImports + maySkip("main") + schedulingChangedInitialQueuing("other") + skipping("main") + findingBatchingCompiling("other") + reading(deps: "other") + fingerprintsChanged("other") + moduleOutputNotFound("R") + moduleWillBeRebuiltOutOfDate("R") + compilingExplicitSwiftDependency("R") + explicitModulesWillBeRebuilt(["J", "R"]) + explicitDependencyNewerModuleInputs("J") + compilingExplicitSwiftDependency("J") + skipped("main") + schedulingPostCompileJobs + linking + } + } } extension IncrementalCompilationTests { @@ -1645,6 +1700,9 @@ extension DiagVerifiable { @DiagsBuilder func explicitDependencyInvalidatedDownstream(_ moduleName: String) -> [Diagnostic.Message] { "Incremental compilation: Dependency module '\(moduleName)' will be re-built: Invalidated by downstream dependency" } + @DiagsBuilder func explicitDependencyNewerModuleInputs(_ moduleName: String) -> [Diagnostic.Message] { + "Incremental compilation: Dependency module '\(moduleName)' will be re-built: Has newer module dependency inputs" + } // MARK: - misc @DiagsBuilder var enablingCrossModule: [Diagnostic.Message] { From cc65d79454c4edb0acff35ecacbde0e2ff4025d4 Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Fri, 7 Jun 2024 10:04:13 -0700 Subject: [PATCH 2/3] [NFC] Move 'InterModuleDependencyGraph' out-of-date analysis into an extension on the graph itself --- .../CommonDependencyOperations.swift | 77 +++++++++++++++++++ .../BuildRecordInfo.swift | 2 +- .../FirstWaveComputer.swift | 71 +---------------- .../IncrementalDependencyAndInputSetup.swift | 2 +- 4 files changed, 80 insertions(+), 72 deletions(-) diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift index c9145ea51..fb4ea6e77 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// import func TSCBasic.topologicalSort +import protocol TSCBasic.FileSystem @_spi(Testing) public extension InterModuleDependencyGraph { /// For targets that are built alongside the driver's current module, the scanning action will report them as @@ -255,6 +256,82 @@ extension InterModuleDependencyGraph { } } +/// Incremental Build Machinery +internal extension InterModuleDependencyGraph { + /// We must determine if any of the module dependencies require re-compilation + /// Since we know that a prior dependency graph was not completely up-to-date, + /// there must be at least *some* dependencies that require being re-built. + /// + /// If a dependency is deemed as requiring a re-build, then every module + /// between it and the root (source module being built by this driver + /// instance) must also be re-built. + func computeInvalidatedModuleDependencies(fileSystem: FileSystem, + reporter: IncrementalCompilationState.Reporter? = nil) + throws -> Set { + let mainModuleInfo = mainModule + var modulesRequiringRebuild: Set = [] + var visited: Set = [] + // Scan from the main module's dependencies to avoid reporting + // the main module itself in the results. + for dependencyId in mainModuleInfo.directDependencies ?? [] { + try outOfDateModuleScan(from: dependencyId, visited: &visited, + modulesRequiringRebuild: &modulesRequiringRebuild, + fileSystem: fileSystem, reporter: reporter) + } + + reporter?.reportExplicitDependencyReBuildSet(Array(modulesRequiringRebuild)) + return modulesRequiringRebuild + } + + /// Perform a postorder DFS to locate modules which are out-of-date with respect + /// to their inputs. Upon encountering such a module, add it to the set of invalidated + /// modules, along with the path from the root to this module. + func outOfDateModuleScan(from sourceModuleId: ModuleDependencyId, + visited: inout Set, + modulesRequiringRebuild: inout Set, + fileSystem: FileSystem, + reporter: IncrementalCompilationState.Reporter? = nil) throws { + let sourceModuleInfo = try moduleInfo(of: sourceModuleId) + // Visit the module's dependencies + var hasOutOfDateModuleDependency = false + var mostRecentlyUpdatedDependencyOutput: TimePoint = .zero + for dependencyId in sourceModuleInfo.directDependencies ?? [] { + // If we have not already visited this module, recurse. + if !visited.contains(dependencyId) { + try outOfDateModuleScan(from: dependencyId, visited: &visited, + modulesRequiringRebuild: &modulesRequiringRebuild, + fileSystem: fileSystem, reporter: reporter) + } + // Even if we're not revisiting a dependency, we must check if it's already known to be out of date. + hasOutOfDateModuleDependency = hasOutOfDateModuleDependency || modulesRequiringRebuild.contains(dependencyId) + + // Keep track of dependencies' output file time stamp to determine if it is newer than the current module. + if let depOutputTimeStamp = try? fileSystem.lastModificationTime(for: VirtualPath.lookup(moduleInfo(of: dependencyId).modulePath.path)), + depOutputTimeStamp > mostRecentlyUpdatedDependencyOutput { + mostRecentlyUpdatedDependencyOutput = depOutputTimeStamp + } + } + + if hasOutOfDateModuleDependency { + reporter?.reportExplicitDependencyWillBeReBuilt(sourceModuleId.moduleNameForDiagnostic, reason: "Invalidated by downstream dependency") + modulesRequiringRebuild.insert(sourceModuleId) + } else if try !IncrementalCompilationState.IncrementalDependencyAndInputSetup.verifyModuleDependencyUpToDate(moduleID: sourceModuleId, moduleInfo: sourceModuleInfo, + fileSystem: fileSystem, reporter: reporter) { + reporter?.reportExplicitDependencyWillBeReBuilt(sourceModuleId.moduleNameForDiagnostic, reason: "Out-of-date") + modulesRequiringRebuild.insert(sourceModuleId) + } else if let outputModTime = try? fileSystem.lastModificationTime(for: VirtualPath.lookup(sourceModuleInfo.modulePath.path)), + outputModTime < mostRecentlyUpdatedDependencyOutput { + // If a prior variant of this module dependnecy exists, and is older than any of its direct or transitive + // module dependency outputs, it must also be re-built. + reporter?.reportExplicitDependencyWillBeReBuilt(sourceModuleId.moduleNameForDiagnostic, reason: "Has newer module dependency inputs") + modulesRequiringRebuild.insert(sourceModuleId) + } + + // Now that we've determined if this module must be rebuilt, mark it as visited. + visited.insert(sourceModuleId) + } +} + internal extension InterModuleDependencyGraph { func explainDependency(dependencyModuleName: String) throws -> [[ModuleDependencyId]]? { guard modules.contains(where: { $0.key.moduleName == dependencyModuleName }) else { return nil } diff --git a/Sources/SwiftDriver/IncrementalCompilation/BuildRecordInfo.swift b/Sources/SwiftDriver/IncrementalCompilation/BuildRecordInfo.swift index f327faa78..59afb612e 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/BuildRecordInfo.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/BuildRecordInfo.swift @@ -184,7 +184,7 @@ import class Dispatch.DispatchQueue try? fileSystem.removeFileTree(absPath) } - func readOutOfDateInterModuleDependencyGraph( + func readPriorInterModuleDependencyGraph( reporter: IncrementalCompilationState.Reporter? ) -> InterModuleDependencyGraph? { let decodedGraph: InterModuleDependencyGraph diff --git a/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift b/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift index 153f709e1..5e18b5b64 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift @@ -138,75 +138,6 @@ extension IncrementalCompilationState.FirstWaveComputer { mandatoryJobsInOrder: mandatoryJobsInOrder) } - /// We must determine if any of the module dependencies require re-compilation - /// Since we know that a prior dependency graph was not completely up-to-date, - /// there must be at least *some* dependencies that require being re-built. - /// - /// If a dependency is deemed as requiring a re-build, then every module - /// between it and the root (source module being built by this driver - /// instance) must also be re-built. - private func computeInvalidatedModuleDependencies(on moduleDependencyGraph: InterModuleDependencyGraph) - throws -> Set { - let mainModuleInfo = moduleDependencyGraph.mainModule - var modulesRequiringRebuild: Set = [] - var visited: Set = [] - // Scan from the main module's dependencies to avoid reporting - // the main module itself in the results. - for dependencyId in mainModuleInfo.directDependencies ?? [] { - try outOfDateModuleScan(on: moduleDependencyGraph, from: dependencyId, - visited: &visited, modulesRequiringRebuild: &modulesRequiringRebuild) - } - - reporter?.reportExplicitDependencyReBuildSet(Array(modulesRequiringRebuild)) - return modulesRequiringRebuild - } - - /// Perform a postorder DFS to locate modules which are out-of-date with respect - /// to their inputs. Upon encountering such a module, add it to the set of invalidated - /// modules, along with the path from the root to this module. - private func outOfDateModuleScan(on moduleDependencyGraph: InterModuleDependencyGraph, - from moduleId: ModuleDependencyId, - visited: inout Set, - modulesRequiringRebuild: inout Set) throws { - let moduleInfo = try moduleDependencyGraph.moduleInfo(of: moduleId) - // Visit the module's dependencies - var hasOutOfDateModuleDependency = false - var mostRecentlyUpdatedDependencyOutput: TimePoint = .zero - for dependencyId in moduleInfo.directDependencies ?? [] { - // If we have not already visited this module, recurse. - if !visited.contains(dependencyId) { - try outOfDateModuleScan(on: moduleDependencyGraph, from: dependencyId, - visited: &visited, modulesRequiringRebuild: &modulesRequiringRebuild) - } - // Even if we're not revisiting a dependency, we must check if it's already known to be out of date. - hasOutOfDateModuleDependency = hasOutOfDateModuleDependency || modulesRequiringRebuild.contains(dependencyId) - - // Keep track of dependencies' output file time stamp to determine if it is newer than the current module. - if let depOutputTimeStamp = try? fileSystem.lastModificationTime(for: VirtualPath.lookup(moduleDependencyGraph.moduleInfo(of: dependencyId).modulePath.path)), - depOutputTimeStamp > mostRecentlyUpdatedDependencyOutput { - mostRecentlyUpdatedDependencyOutput = depOutputTimeStamp - } - } - - if hasOutOfDateModuleDependency { - reporter?.reportExplicitDependencyWillBeReBuilt(moduleId.moduleNameForDiagnostic, reason: "Invalidated by downstream dependency") - modulesRequiringRebuild.insert(moduleId) - } else if try !IncrementalCompilationState.IncrementalDependencyAndInputSetup.verifyModuleDependencyUpToDate(moduleID: moduleId, moduleInfo: moduleInfo, - fileSystem: fileSystem, reporter: reporter) { - reporter?.reportExplicitDependencyWillBeReBuilt(moduleId.moduleNameForDiagnostic, reason: "Out-of-date") - modulesRequiringRebuild.insert(moduleId) - } else if let outputModTime = try? fileSystem.lastModificationTime(for: VirtualPath.lookup(moduleInfo.modulePath.path)), - outputModTime < mostRecentlyUpdatedDependencyOutput { - // If a prior variant of this module dependnecy exists, and is older than any of its direct or transitive - // module dependency outputs, it must also be re-built. - reporter?.reportExplicitDependencyWillBeReBuilt(moduleId.moduleNameForDiagnostic, reason: "Has newer module dependency inputs") - modulesRequiringRebuild.insert(moduleId) - } - - // Now that we've determined if this module must be rebuilt, mark it as visited. - visited.insert(moduleId) - } - /// In an explicit module build, filter out dependency module pre-compilation tasks /// for modules up-to-date from a prior compile. private func computeMandatoryBeforeCompilesJobs() throws -> [Job] { @@ -224,7 +155,7 @@ extension IncrementalCompilationState.FirstWaveComputer { // Determine which module pre-build jobs must be re-run let modulesRequiringReBuild = - try computeInvalidatedModuleDependencies(on: moduleDependencyGraph) + try moduleDependencyGraph.computeInvalidatedModuleDependencies(fileSystem: fileSystem, reporter: reporter) // Filter the `.generatePCM` and `.compileModuleFromInterface` jobs for // modules which do *not* need re-building. diff --git a/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift b/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift index 89297c777..56188569e 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift @@ -106,7 +106,7 @@ extension IncrementalCompilationState.IncrementalDependencyAndInputSetup { ) throws -> InterModuleDependencyGraph? { // Attempt to read a serialized inter-module dependency graph from a prior build guard let priorInterModuleDependencyGraph = - buildRecordInfo.readOutOfDateInterModuleDependencyGraph(reporter: reporter), + buildRecordInfo.readPriorInterModuleDependencyGraph(reporter: reporter), let priorImports = priorInterModuleDependencyGraph.mainModule.directDependencies?.map({ $0.moduleName }) else { reporter?.reportExplicitBuildMustReScan("Could not read inter-module dependency graph at \(buildRecordInfo.interModuleDependencyGraphPath)") return nil From 3be43964b9440b08754b68771469c6a4e4e15567 Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Fri, 7 Jun 2024 10:43:10 -0700 Subject: [PATCH 3/3] [Explicit Module Builds][Incremental Builds] Unify logic used to check if a prior inter-module dep graph is up-to-date with a check to decide which modules to re-build Using the same 'computeInvalidatedModuleDependencies' routine, which is more thorough and checks module inputs to each dependency as well. --- .../CommonDependencyOperations.swift | 122 +++++++++++++++--- .../FirstWaveComputer.swift | 4 +- ...crementalCompilationState+Extensions.swift | 5 + .../IncrementalDependencyAndInputSetup.swift | 99 +------------- .../IncrementalCompilationTests.swift | 48 ++++--- 5 files changed, 141 insertions(+), 137 deletions(-) diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift index fb4ea6e77..dc650a222 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift @@ -266,6 +266,7 @@ internal extension InterModuleDependencyGraph { /// between it and the root (source module being built by this driver /// instance) must also be re-built. func computeInvalidatedModuleDependencies(fileSystem: FileSystem, + forRebuild: Bool, reporter: IncrementalCompilationState.Reporter? = nil) throws -> Set { let mainModuleInfo = mainModule @@ -276,10 +277,13 @@ internal extension InterModuleDependencyGraph { for dependencyId in mainModuleInfo.directDependencies ?? [] { try outOfDateModuleScan(from: dependencyId, visited: &visited, modulesRequiringRebuild: &modulesRequiringRebuild, - fileSystem: fileSystem, reporter: reporter) + fileSystem: fileSystem, forRebuild: forRebuild, + reporter: reporter) } - reporter?.reportExplicitDependencyReBuildSet(Array(modulesRequiringRebuild)) + if forRebuild { + reporter?.reportExplicitDependencyReBuildSet(Array(modulesRequiringRebuild)) + } return modulesRequiringRebuild } @@ -290,46 +294,124 @@ internal extension InterModuleDependencyGraph { visited: inout Set, modulesRequiringRebuild: inout Set, fileSystem: FileSystem, + forRebuild: Bool, reporter: IncrementalCompilationState.Reporter? = nil) throws { + let reportOutOfDate = { (name: String, reason: String) in + if forRebuild { + reporter?.reportExplicitDependencyWillBeReBuilt(sourceModuleId.moduleNameForDiagnostic, reason: reason) + } else { + reporter?.reportPriorExplicitDependencyStale(sourceModuleId.moduleNameForDiagnostic, reason: reason) + } + } + let sourceModuleInfo = try moduleInfo(of: sourceModuleId) // Visit the module's dependencies var hasOutOfDateModuleDependency = false - var mostRecentlyUpdatedDependencyOutput: TimePoint = .zero for dependencyId in sourceModuleInfo.directDependencies ?? [] { // If we have not already visited this module, recurse. if !visited.contains(dependencyId) { try outOfDateModuleScan(from: dependencyId, visited: &visited, modulesRequiringRebuild: &modulesRequiringRebuild, - fileSystem: fileSystem, reporter: reporter) + fileSystem: fileSystem, forRebuild: forRebuild, + reporter: reporter) } // Even if we're not revisiting a dependency, we must check if it's already known to be out of date. hasOutOfDateModuleDependency = hasOutOfDateModuleDependency || modulesRequiringRebuild.contains(dependencyId) - - // Keep track of dependencies' output file time stamp to determine if it is newer than the current module. - if let depOutputTimeStamp = try? fileSystem.lastModificationTime(for: VirtualPath.lookup(moduleInfo(of: dependencyId).modulePath.path)), - depOutputTimeStamp > mostRecentlyUpdatedDependencyOutput { - mostRecentlyUpdatedDependencyOutput = depOutputTimeStamp - } } if hasOutOfDateModuleDependency { - reporter?.reportExplicitDependencyWillBeReBuilt(sourceModuleId.moduleNameForDiagnostic, reason: "Invalidated by downstream dependency") - modulesRequiringRebuild.insert(sourceModuleId) - } else if try !IncrementalCompilationState.IncrementalDependencyAndInputSetup.verifyModuleDependencyUpToDate(moduleID: sourceModuleId, moduleInfo: sourceModuleInfo, - fileSystem: fileSystem, reporter: reporter) { - reporter?.reportExplicitDependencyWillBeReBuilt(sourceModuleId.moduleNameForDiagnostic, reason: "Out-of-date") + reportOutOfDate(sourceModuleId.moduleNameForDiagnostic, "Invalidated by downstream dependency") modulesRequiringRebuild.insert(sourceModuleId) - } else if let outputModTime = try? fileSystem.lastModificationTime(for: VirtualPath.lookup(sourceModuleInfo.modulePath.path)), - outputModTime < mostRecentlyUpdatedDependencyOutput { - // If a prior variant of this module dependnecy exists, and is older than any of its direct or transitive - // module dependency outputs, it must also be re-built. - reporter?.reportExplicitDependencyWillBeReBuilt(sourceModuleId.moduleNameForDiagnostic, reason: "Has newer module dependency inputs") + } else if try !verifyModuleDependencyUpToDate(moduleID: sourceModuleId, fileSystem: fileSystem, reporter: reporter) { + reportOutOfDate(sourceModuleId.moduleNameForDiagnostic, "Out-of-date") modulesRequiringRebuild.insert(sourceModuleId) } // Now that we've determined if this module must be rebuilt, mark it as visited. visited.insert(sourceModuleId) } + + func verifyModuleDependencyUpToDate(moduleID: ModuleDependencyId, + fileSystem: FileSystem, + reporter: IncrementalCompilationState.Reporter?) throws -> Bool { + let checkedModuleInfo = try moduleInfo(of: moduleID) + // Verify that the specified input exists and is older than the specified output + let verifyInputOlderThanOutputModTime: (String, VirtualPath, TimePoint) -> Bool = + { moduleName, inputPath, outputModTime in + guard let inputModTime = + try? fileSystem.lastModificationTime(for: inputPath) else { + reporter?.report("Unable to 'stat' \(inputPath.description)") + return false + } + if inputModTime > outputModTime { + reporter?.reportExplicitDependencyOutOfDate(moduleName, + inputPath: inputPath.description) + return false + } + return true + } + + // Check if the output file exists + guard let outputModTime = try? fileSystem.lastModificationTime(for: VirtualPath.lookup(checkedModuleInfo.modulePath.path)) else { + reporter?.report("Module output not found: '\(moduleID.moduleNameForDiagnostic)'") + return false + } + + // Check if a dependency of this module has a newer output than this module + for dependencyId in checkedModuleInfo.directDependencies ?? [] { + let dependencyInfo = try moduleInfo(of: dependencyId) + if !verifyInputOlderThanOutputModTime(moduleID.moduleName, + VirtualPath.lookup(dependencyInfo.modulePath.path), + outputModTime) { + return false + } + } + + // Check if any of the textual sources of this module are newer than this module + switch checkedModuleInfo.details { + case .swift(let swiftDetails): + if let moduleInterfacePath = swiftDetails.moduleInterfacePath { + if !verifyInputOlderThanOutputModTime(moduleID.moduleName, + VirtualPath.lookup(moduleInterfacePath.path), + outputModTime) { + return false + } + } + if let bridgingHeaderPath = swiftDetails.bridgingHeaderPath { + if !verifyInputOlderThanOutputModTime(moduleID.moduleName, + VirtualPath.lookup(bridgingHeaderPath.path), + outputModTime) { + return false + } + } + for bridgingSourceFile in swiftDetails.bridgingSourceFiles ?? [] { + if !verifyInputOlderThanOutputModTime(moduleID.moduleName, + VirtualPath.lookup(bridgingSourceFile.path), + outputModTime) { + return false + } + } + case .clang(_): + for inputSourceFile in checkedModuleInfo.sourceFiles ?? [] { + if !verifyInputOlderThanOutputModTime(moduleID.moduleName, + try VirtualPath(path: inputSourceFile), + outputModTime) { + return false + } + } + case .swiftPrebuiltExternal(_): + // TODO: We have to give-up here until we have a way to verify the timestamp of the binary module. + // We can do better here by knowing if this module hasn't changed - which would allows us to not + // invalidate any of the dependencies that depend on it. + reporter?.report("Unable to verify binary module dependency up-to-date: \(moduleID.moduleNameForDiagnostic)") + return false; + case .swiftPlaceholder(_): + // TODO: This should never ever happen. Hard error? + return false; + } + + return true + } } internal extension InterModuleDependencyGraph { diff --git a/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift b/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift index 5e18b5b64..9b26b547f 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift @@ -155,7 +155,9 @@ extension IncrementalCompilationState.FirstWaveComputer { // Determine which module pre-build jobs must be re-run let modulesRequiringReBuild = - try moduleDependencyGraph.computeInvalidatedModuleDependencies(fileSystem: fileSystem, reporter: reporter) + try moduleDependencyGraph.computeInvalidatedModuleDependencies(fileSystem: fileSystem, + forRebuild: true, + reporter: reporter) // Filter the `.generatePCM` and `.compileModuleFromInterface` jobs for // modules which do *not* need re-building. diff --git a/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift index a8731eda9..63a9b30a6 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift @@ -308,6 +308,11 @@ extension IncrementalCompilationState { report("Dependency module '\(moduleOutputPath)' will be re-built: \(reason)") } + func reportPriorExplicitDependencyStale(_ moduleOutputPath: String, + reason: String) { + report("Dependency module '\(moduleOutputPath)' info is stale: \(reason)") + } + func reportExplicitDependencyReBuildSet(_ modules: [ModuleDependencyId]) { report("Following explicit module dependencies will be re-built: [\(modules.map { $0.moduleNameForDiagnostic }.sorted().joined(separator: ", "))]") } diff --git a/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift b/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift index 56188569e..be08fd947 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift @@ -120,9 +120,9 @@ extension IncrementalCompilationState.IncrementalDependencyAndInputSetup { } // Verify that each dependnecy is up-to-date with respect to its inputs - guard try verifyInterModuleDependenciesUpToDate(in: priorInterModuleDependencyGraph, - buildRecordInfo: buildRecordInfo, - reporter: reporter) else { + guard try priorInterModuleDependencyGraph.computeInvalidatedModuleDependencies(fileSystem: buildRecordInfo.fileSystem, + forRebuild: false, + reporter: reporter).isEmpty else { reporter?.reportExplicitBuildMustReScan("Not all dependencies are up-to-date.") return nil } @@ -130,99 +130,6 @@ extension IncrementalCompilationState.IncrementalDependencyAndInputSetup { reporter?.report("Confirmed prior inter-module dependency graph is up-to-date at: \(buildRecordInfo.interModuleDependencyGraphPath)") return priorInterModuleDependencyGraph } - - static func verifyModuleDependencyUpToDate(moduleID: ModuleDependencyId, moduleInfo: ModuleInfo, - fileSystem: FileSystem, - reporter: IncrementalCompilationState.Reporter?) throws -> Bool { - // Verify that the specified input exists and is older than the specified output - let verifyInputOlderThanOutputModTime: (String, VirtualPath, VirtualPath, TimePoint) -> Bool = - { moduleName, inputPath, outputPath, outputModTime in - guard let inputModTime = - try? fileSystem.lastModificationTime(for: inputPath) else { - reporter?.report("Unable to 'stat' \(inputPath.description)") - return false - } - if inputModTime > outputModTime { - reporter?.reportExplicitDependencyOutOfDate(moduleName, - inputPath: inputPath.description) - return false - } - return true - } - - switch moduleInfo.details { - case .swift(let swiftDetails): - guard let outputModTime = try? fileSystem.lastModificationTime(for: VirtualPath.lookup(moduleInfo.modulePath.path)) else { - reporter?.report("Module output not found: '\(moduleID.moduleNameForDiagnostic)'") - return false - } - if let moduleInterfacePath = swiftDetails.moduleInterfacePath { - if !verifyInputOlderThanOutputModTime(moduleID.moduleName, - VirtualPath.lookup(moduleInterfacePath.path), - VirtualPath.lookup(moduleInfo.modulePath.path), - outputModTime) { - return false - } - } - if let bridgingHeaderPath = swiftDetails.bridgingHeaderPath { - if !verifyInputOlderThanOutputModTime(moduleID.moduleName, - VirtualPath.lookup(bridgingHeaderPath.path), - VirtualPath.lookup(moduleInfo.modulePath.path), - outputModTime) { - return false - } - } - for bridgingSourceFile in swiftDetails.bridgingSourceFiles ?? [] { - if !verifyInputOlderThanOutputModTime(moduleID.moduleName, - VirtualPath.lookup(bridgingSourceFile.path), - VirtualPath.lookup(moduleInfo.modulePath.path), - outputModTime) { - return false - } - } - case .clang(_): - guard let outputModTime = try? fileSystem.lastModificationTime(for: VirtualPath.lookup(moduleInfo.modulePath.path)) else { - reporter?.report("Module output not found: '\(moduleID.moduleNameForDiagnostic)'") - return false - } - for inputSourceFile in moduleInfo.sourceFiles ?? [] { - if !verifyInputOlderThanOutputModTime(moduleID.moduleName, - try VirtualPath(path: inputSourceFile), - VirtualPath.lookup(moduleInfo.modulePath.path), - outputModTime) { - return false - } - } - case .swiftPrebuiltExternal(_): - // TODO: We have to give-up here until we have a way to verify the timestamp of the binary module. - // We can do better here by knowing if this module hasn't change - which would allows us to not - // invalidate any of the dependencies that depend on it. - reporter?.report("Unable to verify binary module dependency up-to-date: \(moduleID.moduleNameForDiagnostic)") - return false; - case .swiftPlaceholder(_): - // TODO: This should never ever happen. Hard error? - return false; - } - return true - } - - /// For each direct and transitive module dependency, check if any of the inputs are newer than the output - static func verifyInterModuleDependenciesUpToDate(in graph: InterModuleDependencyGraph, - buildRecordInfo: BuildRecordInfo, - reporter: IncrementalCompilationState.Reporter?) throws -> Bool { - for module in graph.modules { - if module.key == .swift(graph.mainModuleName) { - continue - } - if try !verifyModuleDependencyUpToDate(moduleID: module.key, - moduleInfo: module.value, - fileSystem: buildRecordInfo.fileSystem, - reporter: reporter) { - return false - } - } - return true - } } /// Builds the `InitialState` diff --git a/Tests/SwiftDriverTests/IncrementalCompilationTests.swift b/Tests/SwiftDriverTests/IncrementalCompilationTests.swift index d68ce18c6..4f51d25c2 100644 --- a/Tests/SwiftDriverTests/IncrementalCompilationTests.swift +++ b/Tests/SwiftDriverTests/IncrementalCompilationTests.swift @@ -351,6 +351,7 @@ extension IncrementalCompilationTests { readInterModuleGraph // Ensure the above 'touch' is detected and causes a re-scan explicitDependencyModuleOlderThanInput("E") + moduleInfoStaleOutOfDate("E") explicitMustReScanDueToChangedDependencyInput noFingerprintInSwiftModule("E.swiftinterface") dependencyNewerThanNode("E.swiftinterface") @@ -402,6 +403,11 @@ extension IncrementalCompilationTests { readInterModuleGraph // Ensure the above 'touch' is detected and causes a re-scan explicitDependencyModuleOlderThanInput("G") + moduleInfoStaleOutOfDate("G") + moduleInfoStaleInvalidatedDownstream("J") + moduleInfoStaleInvalidatedDownstream("T") + moduleInfoStaleInvalidatedDownstream("Y") + moduleInfoStaleInvalidatedDownstream("H") explicitMustReScanDueToChangedDependencyInput noFingerprintInSwiftModule("G.swiftinterface") dependencyNewerThanNode("G.swiftinterface") @@ -415,10 +421,10 @@ extension IncrementalCompilationTests { queuingInitial("main", "other") findingBatchingCompiling("main", "other") explicitDependencyModuleOlderThanInput("G") - explicitDependencyInvalidatedDownstream("J") - explicitDependencyInvalidatedDownstream("T") - explicitDependencyInvalidatedDownstream("Y") - explicitDependencyInvalidatedDownstream("H") + moduleWillBeRebuiltInvalidatedDownstream("J") + moduleWillBeRebuiltInvalidatedDownstream("T") + moduleWillBeRebuiltInvalidatedDownstream("Y") + moduleWillBeRebuiltInvalidatedDownstream("H") explicitModulesWillBeRebuilt(["G", "H", "J", "T", "Y"]) moduleWillBeRebuiltOutOfDate("G") compilingExplicitSwiftDependency("G") @@ -452,9 +458,9 @@ extension IncrementalCompilationTests { let nameOfGModule = try XCTUnwrap(modCacheEntries.first { $0.hasPrefix("G") && $0.hasSuffix(".swiftmodule")}) let pathToGModule = explicitModuleCacheDir.appending(component: nameOfGModule) // Just update the time-stamp of one of the module dependencies' outputs. - // Also add a dependency to cause a re-scan. touch(pathToGModule) - replace(contentsOf: "other", with: "import J;import R") + // Touch one of the inputs to actually trigger the incremental build + touch(inputPath(basename: "other")) // Changing a dependency will mean that we both re-run the dependency scan, // and also ensure that all source-files are re-built with a non-cascading build @@ -468,19 +474,18 @@ extension IncrementalCompilationTests { readGraph enablingCrossModule readInterModuleGraph - explicitMustReScanDueToChangedImports + explicitDependencyModuleOlderThanInput("J") + moduleInfoStaleOutOfDate("J") + explicitMustReScanDueToChangedDependencyInput maySkip("main") schedulingChangedInitialQueuing("other") skipping("main") + explicitDependencyModuleOlderThanInput("J") + moduleWillBeRebuiltOutOfDate("J") + explicitModulesWillBeRebuilt(["J"]) + compilingExplicitSwiftDependency("J") findingBatchingCompiling("other") reading(deps: "other") - fingerprintsChanged("other") - moduleOutputNotFound("R") - moduleWillBeRebuiltOutOfDate("R") - compilingExplicitSwiftDependency("R") - explicitModulesWillBeRebuilt(["J", "R"]) - explicitDependencyNewerModuleInputs("J") - compilingExplicitSwiftDependency("J") skipped("main") schedulingPostCompileJobs linking @@ -1694,14 +1699,17 @@ extension DiagVerifiable { @DiagsBuilder func moduleWillBeRebuiltOutOfDate(_ moduleName: String) -> [Diagnostic.Message] { "Incremental compilation: Dependency module '\(moduleName)' will be re-built: Out-of-date" } - @DiagsBuilder func explicitModulesWillBeRebuilt(_ moduleNames: [String]) -> [Diagnostic.Message] { - "Incremental compilation: Following explicit module dependencies will be re-built: [\(moduleNames.joined(separator: ", "))]" - } - @DiagsBuilder func explicitDependencyInvalidatedDownstream(_ moduleName: String) -> [Diagnostic.Message] { + @DiagsBuilder func moduleWillBeRebuiltInvalidatedDownstream(_ moduleName: String) -> [Diagnostic.Message] { "Incremental compilation: Dependency module '\(moduleName)' will be re-built: Invalidated by downstream dependency" } - @DiagsBuilder func explicitDependencyNewerModuleInputs(_ moduleName: String) -> [Diagnostic.Message] { - "Incremental compilation: Dependency module '\(moduleName)' will be re-built: Has newer module dependency inputs" + @DiagsBuilder func moduleInfoStaleOutOfDate(_ moduleName: String) -> [Diagnostic.Message] { + "Incremental compilation: Dependency module '\(moduleName)' info is stale: Out-of-date" + } + @DiagsBuilder func moduleInfoStaleInvalidatedDownstream(_ moduleName: String) -> [Diagnostic.Message] { + "Incremental compilation: Dependency module '\(moduleName)' info is stale: Invalidated by downstream dependency" + } + @DiagsBuilder func explicitModulesWillBeRebuilt(_ moduleNames: [String]) -> [Diagnostic.Message] { + "Incremental compilation: Following explicit module dependencies will be re-built: [\(moduleNames.joined(separator: ", "))]" } // MARK: - misc