Use wrapper types to ensure direct/transitive handled correctly.

Author: David Ungar

Sources/SwiftDriver/IncrementalCompilation/DirectAndTransitiveCollections.swift

@@ -13,34 +13,74 @@
 // Use the type system to ensure that dependencies are transitively closed
 // without doing too much work at the leaves of the call tree
 
-public struct DirectlyInvalidatedNodes {
-  typealias Node = ModuleDependencyGraph.Node
-  var contents = Set<Node>()
+public struct Transitively {}
+public struct Directly {}
 
-  init(_ s: Set<Node> = Set()) {
+public struct InvalidatedSet<ClosureLevel, Element: Hashable>: Sequence {
+  var contents: Set<Element>
+
+  init(_ s: Set<Element> = Set()) {
     self.contents = s
   }
-
-  init<Nodes: Sequence>(_ nodes: Nodes)
-  where Nodes.Element == Node
+  init<Elements: Sequence>(_ elements: Elements)
+  where Elements.Element == Element
   {
-    self.init(Set(nodes))
+    self.init(Set(elements))
   }
-
-  mutating func insert(_ e: Node) {
+  mutating func insert(_ e: Element) {
     contents.insert(e)
   }
-  mutating func formUnion(_ nodes: DirectlyInvalidatedNodes) {
-    contents.formUnion(nodes.contents)
+  mutating func formUnion<Elements: Sequence>(_ elements: Elements)
+  where Elements.Element == Element{
+    contents.formUnion(elements)
+  }
+  public func makeIterator() -> Set<Element>.Iterator {
+    contents.makeIterator()
+  }
+  public func map<R>(_ transform: (Element) -> R) -> InvalidatedArray<ClosureLevel, R> {
+    InvalidatedArray(contents.map(transform))
+  }
+  public func compactMap<R>(_ transform: (Element) -> R? ) -> InvalidatedArray<ClosureLevel, R> {
+    InvalidatedArray(contents.compactMap(transform))
+  }
+}
+
+extension InvalidatedSet where Element: Comparable {
+  func sorted() -> InvalidatedArray<ClosureLevel, Element> {
+    InvalidatedArray(contents.sorted())
+  }
+}
+
+public struct InvalidatedArray<ClosureLevel, Element>: Sequence {
+  var contents: [Element]
+
+  init(_ s: [Element] = []) {
+    self.contents = s
+  }
+  init<Elements: Sequence>(_ elements: Elements)
+  where Elements.Element == Element
+  {
+    self.init(Array(elements))
+  }
+  public func makeIterator() -> Array<Element>.Iterator {
+    contents.makeIterator()
+  }
+  public mutating func append(_ e: Element) {
+    contents.append(e)
+  }
+  public func reduce<R: Hashable>(
+    into initialResult: InvalidatedSet<ClosureLevel, R>,
+    _ updateAccumulatingResult: (inout Set<R>, Element) -> ()
+  ) -> InvalidatedSet<ClosureLevel, R> {
+    InvalidatedSet(
+      contents.reduce(into: initialResult.contents, updateAccumulatingResult))
   }
+  public var count: Int { contents.count }
 }
 
-//extension Sequence {
-//  func reduce(into initialResult: DirectlyInvalidatedNodes,
-//              _ updateAccumulatingResult: (inout: DirectlyInvalidatedNodes, Element) -> Void) {
-//    var r = initialResult
-//    reduce(into: r.contents, updateAccumulatingResult)
-//    return r
-//    }
-//  }
-//}
+public typealias TransitivelyInvalidatedNodeArray = InvalidatedArray<Transitively, ModuleDependencyGraph.Node>
+public typealias TransitivelyInvalidatedSourceSet = InvalidatedSet<Transitively, DependencySource>
+public typealias TransitivelyInvalidatedInputArray = InvalidatedArray<Transitively, TypedVirtualPath>
+public typealias TransitivelyInvalidatedInputSet = InvalidatedSet<Transitively, TypedVirtualPath>
+public typealias DirectlyInvalidatedNodeArray = InvalidatedArray<Directly, ModuleDependencyGraph.Node>
+public typealias DirectlyInvalidatedNodeSet = InvalidatedSet<Directly, ModuleDependencyGraph.Node>

Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift

@@ -249,13 +249,13 @@ extension IncrementalCompilationState {
   }
 
   private func collectInputsInvalidated(byCompiling input: TypedVirtualPath)
-  -> Set<TypedVirtualPath> {
+  -> TransitivelyInvalidatedInputSet {
     if let found = moduleDependencyGraph.collectInputsRequiringCompilation(byCompiling: input) {
       return found
     }
     self.reporter?.report(
       "Failed to read some dependencies source; compiling everything", input)
-    return Set<TypedVirtualPath>(skippedCompileGroups.keys)
+    return TransitivelyInvalidatedInputSet(skippedCompileGroups.keys)
   }
 
   /// Find the jobs that now must be run that were not originally known to be needed.

Sources/SwiftDriver/IncrementalCompilation/InitialStateComputer.swift

@@ -116,7 +116,7 @@ extension IncrementalCompilationState.InitialStateComputer {
   /// For inputs with swiftDeps in OFM, but no readable file, puts input in graph map, but no nodes in graph:
   ///   caller must ensure scheduling of those
   private func computeGraphAndInputsInvalidatedByExternals()
-  -> (ModuleDependencyGraph, Set<TypedVirtualPath>)? {
+  -> (ModuleDependencyGraph, TransitivelyInvalidatedInputSet)? {
     precondition(sourceFiles.disappeared.isEmpty,
                  "Would have to remove nodes from the graph if reading prior")
     if readPriorsFromModuleDependencyGraph {
@@ -128,7 +128,7 @@ extension IncrementalCompilationState.InitialStateComputer {
   }
 
   private func readPriorGraphAndCollectInputsInvalidatedByChangedOrAddedExternals(
-  ) -> (ModuleDependencyGraph, Set<TypedVirtualPath>)?
+  ) -> (ModuleDependencyGraph, TransitivelyInvalidatedInputSet)?
   {
     let dependencyGraphPath = buildRecordInfo.dependencyGraphPath
     let graphIfPresent: ModuleDependencyGraph?
@@ -154,9 +154,9 @@ extension IncrementalCompilationState.InitialStateComputer {
     // recompilation. Thus, `ChangedOrAdded`.
     let nodesDirectlyInvalidatedByExternals = graph.collectNodesInvalidatedByChangedOrAddedExternals()
     // Wait till the last minute to do the transitive closure as an optimization.
-    let inputsTransitivelyInvalidatedByExternals = graph.collectInputsUsingTransitivelyInvalidated(
+    let inputsInvalidatedByExternals = graph.collectInputsUsingInvalidated(
       nodes: nodesDirectlyInvalidatedByExternals)
-    return (graph, inputsTransitivelyInvalidatedByExternals)
+    return (graph, inputsInvalidatedByExternals)
   }
 
   /// Builds a graph
@@ -167,15 +167,15 @@ extension IncrementalCompilationState.InitialStateComputer {
   /// For externalDependencies, puts then in graph.fingerprintedExternalDependencies, but otherwise
   /// does nothing special.
   private func buildInitialGraphFromSwiftDepsAndCollectInputsInvalidatedByChangedExternals(
-  ) -> (ModuleDependencyGraph, Set<TypedVirtualPath>)?
+  ) -> (ModuleDependencyGraph, TransitivelyInvalidatedInputSet)?
   {
     let graph = ModuleDependencyGraph(self, .buildingWithoutAPrior)
     assert(outputFileMap.onlySourceFilesHaveSwiftDeps())
     guard graph.populateInputDependencySourceMap() else {
       return nil
     }
 
-    var inputsInvalidatedByChangedExternals = Set<TypedVirtualPath>()
+    var inputsInvalidatedByChangedExternals = TransitivelyInvalidatedInputSet()
     for input in sourceFiles.currentInOrder {
        guard let invalidatedInputs = graph.collectInputsRequiringCompilationFromExternalsFoundByCompiling(input: input)
       else {
@@ -195,7 +195,7 @@ extension IncrementalCompilationState.InitialStateComputer {
   /// listed in fingerprintExternalDependencies.
   private func computeInputsAndGroups(
     _ moduleDependencyGraph: ModuleDependencyGraph,
-    _ inputsInvalidatedByExternals: Set<TypedVirtualPath>,
+    _ inputsInvalidatedByExternals: TransitivelyInvalidatedInputSet,
     batchJobFormer: inout Driver
   ) throws -> (skippedCompileGroups: [TypedVirtualPath: CompileJobGroup],
                mandatoryJobsInOrder: [Job])
@@ -249,7 +249,7 @@ extension IncrementalCompilationState.InitialStateComputer {
 
   /// Figure out which compilation inputs are *not* mandatory
   private func computeSkippedCompilationInputs(
-    inputsInvalidatedByExternals: Set<TypedVirtualPath>,
+    inputsInvalidatedByExternals: TransitivelyInvalidatedInputSet,
     _ moduleDependencyGraph: ModuleDependencyGraph,
     _ buildRecord: BuildRecord
   ) -> Set<TypedVirtualPath> {
@@ -285,7 +285,7 @@ extension IncrementalCompilationState.InitialStateComputer {
     // as each first wave job finished.
     let speculativeInputs = collectInputsToBeSpeculativelyRecompiled(
       changedInputs: changedInputs,
-      externalDependents: Array(inputsInvalidatedByExternals),
+      externalDependents: inputsInvalidatedByExternals,
       inputsMissingOutputs: Set(inputsMissingOutputs),
       moduleDependencyGraph)
       .subtracting(definitelyRequiredInputs)
@@ -371,28 +371,28 @@ extension IncrementalCompilationState.InitialStateComputer {
   /// before the whole frontend job finished.
   private func collectInputsToBeSpeculativelyRecompiled(
     changedInputs: [ChangedInput],
-    externalDependents: [TypedVirtualPath],
+    externalDependents: TransitivelyInvalidatedInputSet,
     inputsMissingOutputs: Set<TypedVirtualPath>,
     _ moduleDependencyGraph: ModuleDependencyGraph
   ) -> Set<TypedVirtualPath> {
     let cascadingChangedInputs = computeCascadingChangedInputs(
       from: changedInputs,
       inputsMissingOutputs: inputsMissingOutputs)
-    let cascadingExternalDependents = alwaysRebuildDependents ? externalDependents : []
-    // Collect the dependent files to speculatively schedule
-    var dependentFiles = Set<TypedVirtualPath>()
-    let cascadingFileSet = Set(cascadingChangedInputs).union(cascadingExternalDependents)
-    for cascadingFile in cascadingFileSet {
-      let dependentsOfOneFile = moduleDependencyGraph
-        .collectInputsTransitivelyInvalidatedBy(input: cascadingFile)
-      for dep in dependentsOfOneFile where !cascadingFileSet.contains(dep) {
-        if dependentFiles.insert(dep).0 {
+
+    var inputsToBeCertainlyRecompiled = alwaysRebuildDependents ? externalDependents : TransitivelyInvalidatedInputSet()
+    inputsToBeCertainlyRecompiled.formUnion(cascadingChangedInputs)
+
+    return inputsToBeCertainlyRecompiled.reduce(into: Set()) {
+      speculativelyRecompiledInputs, certainlyRecompiledInput in
+      let speculativeDependents = moduleDependencyGraph.collectInputsInvalidatedBy(input: certainlyRecompiledInput)
+      for speculativeDependent in speculativeDependents
+      where !inputsToBeCertainlyRecompiled.contains(speculativeDependent) {
+        if speculativelyRecompiledInputs.insert(speculativeDependent).inserted {
           reporter?.report(
-            "Immediately scheduling dependent on \(cascadingFile.file.basename)", dep)
+            "Immediately scheduling dependent on \(certainlyRecompiledInput.file.basename)", speculativeDependent)
         }
       }
     }
-    return dependentFiles
   }
 
   // Collect the files that will be compiled whose dependents should be schedule