[Explicit Module Builds][Incremental Builds] Re-compile module dependnecies whose dependencies are up-to-date themselves but are themselves newer

For example consider the following module graph:
             test
                 \
                  J
                   \
                    G

Where on an incremental build we detect that although G binary module product is *newer* than its textual source, said binary module product is also *newer* than a prior binary module product of J. Which means that although each of the modules is up-to-date with respect to its own textual source inputs, J's binary dependnecy input has been updated elsewhere and J needs to be re-built.

Resolves rdar://129225956

Author: artemcm

Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift

@@ -148,12 +148,14 @@ extension IncrementalCompilationState.FirstWaveComputer {
   private func computeInvalidatedModuleDependencies(on moduleDependencyGraph: InterModuleDependencyGraph)
   throws -> Set<ModuleDependencyId> {
     let mainModuleInfo = moduleDependencyGraph.mainModule
+    let reachabilityMap = try moduleDependencyGraph.computeTransitiveClosure()
     var modulesRequiringRebuild: Set<ModuleDependencyId> = []
-    var visitedModules: Set<ModuleDependencyId> = []
+    var visited: Set<ModuleDependencyId> = []
     // Scan from the main module's dependencies to avoid reporting
     // the main module itself in the results.
     for dependencyId in mainModuleInfo.directDependencies ?? [] {
-      try outOfDateModuleScan(on: moduleDependencyGraph, from: dependencyId, visited: &visitedModules,
+      try outOfDateModuleScan(on: moduleDependencyGraph, from: dependencyId, 
+                              reachabilityMap: reachabilityMap, visited: &visited,
                               modulesRequiringRebuild: &modulesRequiringRebuild)
     }
 
@@ -164,8 +166,12 @@ extension IncrementalCompilationState.FirstWaveComputer {
   /// Perform a postorder DFS to locate modules which are out-of-date with respect
   /// to their inputs. Upon encountering such a module, add it to the set of invalidated
   /// modules, along with the path from the root to this module.
+  ///
+  /// Returns the last-modification-time of the newest module dependency output file,
+  /// direct or transitive.
   private func outOfDateModuleScan(on moduleDependencyGraph: InterModuleDependencyGraph,
                                    from moduleId: ModuleDependencyId,
+                                   reachabilityMap: [ModuleDependencyId : Set<ModuleDependencyId>],
                                    visited: inout Set<ModuleDependencyId>,
                                    modulesRequiringRebuild: inout Set<ModuleDependencyId>) throws {
     let moduleInfo = try moduleDependencyGraph.moduleInfo(of: moduleId)
@@ -175,20 +181,36 @@ extension IncrementalCompilationState.FirstWaveComputer {
       // If we have not already visited this module, recurse.
       if !visited.contains(dependencyId) {
         try outOfDateModuleScan(on: moduleDependencyGraph, from: dependencyId,
-                                visited: &visited,
+                                reachabilityMap: reachabilityMap, visited: &visited,
                                 modulesRequiringRebuild: &modulesRequiringRebuild)
       }
       // Even if we're not revisiting a dependency, we must check if it's already known to be out of date.
       hasOutOfDateModuleDependency = hasOutOfDateModuleDependency || modulesRequiringRebuild.contains(dependencyId)
     }
 
     if hasOutOfDateModuleDependency {
-        reporter?.reportExplicitDependencyWillBeReBuilt(moduleId.moduleNameForDiagnostic, reason: "Invalidated by downstream dependency")
-        modulesRequiringRebuild.insert(moduleId)
+      reporter?.reportExplicitDependencyWillBeReBuilt(moduleId.moduleNameForDiagnostic, reason: "Invalidated by downstream dependency")
+      modulesRequiringRebuild.insert(moduleId)
     } else if try !IncrementalCompilationState.IncrementalDependencyAndInputSetup.verifyModuleDependencyUpToDate(moduleID: moduleId, moduleInfo: moduleInfo,
                                                                                                                  fileSystem: fileSystem, reporter: reporter) {
-        reporter?.reportExplicitDependencyWillBeReBuilt(moduleId.moduleNameForDiagnostic, reason: "Out-of-date")
-        modulesRequiringRebuild.insert(moduleId)
+      reporter?.reportExplicitDependencyWillBeReBuilt(moduleId.moduleNameForDiagnostic, reason: "Out-of-date")
+      modulesRequiringRebuild.insert(moduleId)
+    }
+
+    // If a prior variant of this module dependnecy exists, and is older than any of its direct or transitive
+    // module dependency outputs, it must also be re-built
+    if let outputModTime = try? fileSystem.lastModificationTime(for: VirtualPath.lookup(moduleInfo.modulePath.path)) {
+      // FIXME: This needs caching for mod times, otherwise there's too much repeated `stat` traffic
+      for depId in reachabilityMap[moduleId] ?? [] {
+        guard let depOutputTimeStamp = try? fileSystem.lastModificationTime(for: VirtualPath.lookup(moduleDependencyGraph.moduleInfo(of: depId).modulePath.path)) else {
+          continue
+        }
+        if depOutputTimeStamp > outputModTime {
+          reporter?.reportExplicitDependencyWillBeReBuilt(moduleId.moduleNameForDiagnostic, reason: "Has newer module dependency inputs")
+          modulesRequiringRebuild.insert(moduleId)
+          break
+        }
+      }
     }
 
     // Now that we've determined if this module must be rebuilt, mark it as visited.

Tests/SwiftDriverTests/IncrementalCompilationTests.swift

@@ -382,7 +382,6 @@ extension IncrementalCompilationTests {
   // On this graph, inputs of 'G' are updated, causing it to be re-built
   // as well as all modules on paths from root to it: 'Y', 'H', 'T','J'
   func testExplicitIncrementalBuildChangedDependencyInvalidatesUpstreamDependencies() throws {
-    // Add an import of 'B', 'C' to make sure followup changes has consistent inputs
     replace(contentsOf: "other", with: "import Y;import T")
     try buildInitialState(checkDiagnostics: false, explicitModuleBuild: true)
 
@@ -431,6 +430,62 @@ extension IncrementalCompilationTests {
       linking
     }
   }
+
+  // A dependency has been re-built to be newer than its dependents
+  // so we must ensure the dependents get re-built even though all the
+  // modules are up-to-date with respect to their textual source inputs.
+  //
+  //             test
+  //                 \
+  //                  J
+  //                   \
+  //                    G
+  //
+  // On this graph, after the initial build, if G module binary file is newer
+  // than that of J, even if each of the modules is up-to-date w.r.t. their source inputs
+  // we still expect that J gets re-built
+  func testExplicitIncrementalBuildChangedDependencyBinaryInvalidatesUpstreamDependencies() throws {
+    replace(contentsOf: "other", with: "import J;")
+    try buildInitialState(checkDiagnostics: false, explicitModuleBuild: true)
+
+    let modCacheEntries = try localFileSystem.getDirectoryContents(explicitModuleCacheDir)
+    let nameOfGModule = try XCTUnwrap(modCacheEntries.first { $0.hasPrefix("G") && $0.hasSuffix(".swiftmodule")})
+    let pathToGModule = explicitModuleCacheDir.appending(component: nameOfGModule)
+    // Just update the time-stamp of one of the module dependencies' outputs.
+    // Also add a dependency to cause a re-scan.
+    touch(pathToGModule)
+    replace(contentsOf: "other", with: "import J;import R")
+
+    // Changing a dependency will mean that we both re-run the dependency scan,
+    // and also ensure that all source-files are re-built with a non-cascading build
+    // since the source files themselves have not changed.
+    try doABuild(
+      "update dependency (G) result timestamp",
+      checkDiagnostics: true,
+      extraArguments: explicitBuildArgs,
+      whenAutolinking: autolinkLifecycleExpectedDiags
+    ) {
+      readGraph
+      enablingCrossModule
+      readInterModuleGraph
+      explicitMustReScanDueToChangedImports
+      maySkip("main")
+      schedulingChangedInitialQueuing("other")
+      skipping("main")
+      findingBatchingCompiling("other")
+      reading(deps: "other")
+      fingerprintsChanged("other")
+      moduleOutputNotFound("R")
+      moduleWillBeRebuiltOutOfDate("R")
+      compilingExplicitSwiftDependency("R")
+      explicitModulesWillBeRebuilt(["J", "R"])
+      explicitDependencyNewerModuleInputs("J")
+      compilingExplicitSwiftDependency("J")
+      skipped("main")
+      schedulingPostCompileJobs
+      linking
+    }
+  }
 }
 
 extension IncrementalCompilationTests {
@@ -1645,6 +1700,9 @@ extension DiagVerifiable {
   @DiagsBuilder func explicitDependencyInvalidatedDownstream(_ moduleName: String) -> [Diagnostic.Message] {
     "Incremental compilation: Dependency module '\(moduleName)' will be re-built: Invalidated by downstream dependency"
   }
+  @DiagsBuilder func explicitDependencyNewerModuleInputs(_ moduleName: String) -> [Diagnostic.Message] {
+    "Incremental compilation: Dependency module '\(moduleName)' will be re-built: Has newer module dependency inputs"
+  }
 
   // MARK: - misc
   @DiagsBuilder var enablingCrossModule: [Diagnostic.Message] {