step1-use type for direct

Author: David Ungar

Sources/SwiftDriver/CMakeLists.txt

@@ -44,6 +44,7 @@ add_library(SwiftDriver
   "IncrementalCompilation/DependencyGraphDotFileWriter.swift"
   "IncrementalCompilation/DependencyKey.swift"
   "IncrementalCompilation/DictionaryOfDictionaries.swift"
+  "IncrementalCompilation/DirectAndTransitiveCollections.swift"
   "IncrementalCompilation/ExternalDependencyAndFingerprintEnforcer.swift"
   "IncrementalCompilation/IncrementalCompilationState.swift"
   "IncrementalCompilation/InitialStateComputer.swift"

Sources/SwiftDriver/IncrementalCompilation/DirectAndTransitiveCollections.swift

@@ -0,0 +1,46 @@
+//===------------------ DirectAndTransitiveCollections.swift --------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+// Use the type system to ensure that dependencies are transitively closed
+// without doing too much work at the leaves of the call tree
+
+public struct DirectlyInvalidatedNodes {
+  typealias Node = ModuleDependencyGraph.Node
+  var contents = Set<Node>()
+
+  init(_ s: Set<Node> = Set()) {
+    self.contents = s
+  }
+
+  init<Nodes: Sequence>(_ nodes: Nodes)
+  where Nodes.Element == Node
+  {
+    self.init(Set(nodes))
+  }
+
+  mutating func insert(_ e: Node) {
+    contents.insert(e)
+  }
+  mutating func formUnion(_ nodes: DirectlyInvalidatedNodes) {
+    contents.formUnion(nodes.contents)
+  }
+}
+
+//extension Sequence {
+//  func reduce(into initialResult: DirectlyInvalidatedNodes,
+//              _ updateAccumulatingResult: (inout: DirectlyInvalidatedNodes, Element) -> Void) {
+//    var r = initialResult
+//    reduce(into: r.contents, updateAccumulatingResult)
+//    return r
+//    }
+//  }
+//}

Sources/SwiftDriver/IncrementalCompilation/InitialStateComputer.swift

@@ -152,7 +152,7 @@ extension IncrementalCompilationState.InitialStateComputer {
 
     // Any externals not already in graph must be additions which should trigger
     // recompilation. Thus, `ChangedOrAdded`.
-    let nodesDirectlyInvalidatedByExternals = graph.collectNodesDirectlyInvalidatedByChangedOrAddedExternals()
+    let nodesDirectlyInvalidatedByExternals = graph.collectNodesInvalidatedByChangedOrAddedExternals()
     // Wait till the last minute to do the transitive closure as an optimization.
     let inputsTransitivelyInvalidatedByExternals = graph.collectInputsUsingTransitivelyInvalidated(
       nodes: nodesDirectlyInvalidatedByExternals)

Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift

@@ -106,10 +106,11 @@ extension ModuleDependencyGraph {
 
 // MARK: - Getting a graph read from priors ready to use
 extension ModuleDependencyGraph {
-  func collectNodesDirectlyInvalidatedByChangedOrAddedExternals() -> Set<Node> {
-    fingerprintedExternalDependencies.reduce(into: Set()) { invalidatedNodes, fed in
+  func collectNodesInvalidatedByChangedOrAddedExternals() -> DirectlyInvalidatedNodes {
+    fingerprintedExternalDependencies.reduce(into: DirectlyInvalidatedNodes()) {
+      invalidatedNodes, fed in
       invalidatedNodes.formUnion (
-        self.collectNodesDirectlyInvalidatedByProcessing(fingerprintedExternalDependency: fed))
+        self.collectNodesInvalidatedByProcessing(fingerprintedExternalDependency: fed))
     }
   }
 }
@@ -140,7 +141,7 @@ extension ModuleDependencyGraph {
   ) -> Set<DependencySource> {
     let nodes = nodeFinder.findNodes(for: dependencySource) ?? [:]
     /// Tests expect this to be reflexive
-    return collectSwiftDepsUsingTransitivelyInvalidated(nodes: nodes.values)
+    return collectSwiftDepsUsingTransitivelyInvalidated(nodes: DirectlyInvalidatedNodes(nodes.values))
   }
 
   /// Does the graph contain any dependency nodes for a given source-code file?
@@ -188,10 +189,9 @@ extension ModuleDependencyGraph {
   /// Given a set of invalidated nodes, find all swiftDeps dependency sources containing defs that transitively use
   /// any of the invalidated nodes.
   /*@_spi(Testing)*/
-  public func collectSwiftDepsUsingTransitivelyInvalidated<Nodes: Sequence>(
-    nodes: Nodes
+  public func collectSwiftDepsUsingTransitivelyInvalidated(
+    nodes: DirectlyInvalidatedNodes
   ) -> Set<DependencySource>
-  where Nodes.Element == Node
   {
     // Is this correct for the 1st wave after having read a prior?
     // Yes, because
@@ -214,9 +214,9 @@ extension ModuleDependencyGraph {
   /// Given an external dependency & its fingerprint, find any nodes directly using that dependency.
   /// As an optimization, only return the nodes that have not been already traced, because the traced nodes
   /// will have already been used to schedule jobs to run.
-  /*@_spi(Testing)*/ public func collectUntracedNodesDirectlyUsing(
+  /*@_spi(Testing)*/ public func collectUntracedNodesUsing(
     _ fingerprintedExternalDependency: FingerprintedExternalDependency
-  ) -> Set<Node> {
+  ) -> DirectlyInvalidatedNodes {
     // These nodes will depend on the *interface* of the external Decl.
     let key = DependencyKey(
       aspect: .interface,
@@ -227,9 +227,10 @@ extension ModuleDependencyGraph {
     let node = Node(key: key,
                     fingerprint: fingerprintedExternalDependency.fingerprint,
                     dependencySource: nil)
-    return nodeFinder
-      .uses(of: node)
-      .filter({ use in use.isUntraced })
+    return DirectlyInvalidatedNodes(
+      nodeFinder
+        .uses(of: node)
+        .filter({ use in use.isUntraced }))
   }
 
   /// Find all the inputs known to need recompilation as a consequence of reading a swiftdeps or swiftmodule
@@ -252,16 +253,16 @@ extension ModuleDependencyGraph {
     /// If reading for the first time, the driver is compiling all outdated source files anyway, so only
     /// nodes invalidated by external dependencies matter.
     /// But when updating, all invalidations matter.
-    let directlyInvalidatedNodes = phase.isUpdating
-      ? results.allDirectlyInvalidatedNodes
-      : results.nodesDirectlyInvalidatedByUsingSomeExternal
+    let invalidatedNodes = phase.isUpdating
+      ? results.all
+      : results.usesOfSomeExternal
 
-    return collectInputsUsingTransitivelyInvalidated(nodes: directlyInvalidatedNodes)
+    return collectInputsUsingTransitivelyInvalidated(nodes: invalidatedNodes)
   }
 
   /// Given nodes that are invalidated, find all the affected inputs that must be recompiled.
   func collectInputsUsingTransitivelyInvalidated(
-    nodes directlyInvalidatedNodes: Set<Node>
+    nodes directlyInvalidatedNodes: DirectlyInvalidatedNodes
   ) -> Set<TypedVirtualPath> {
     collectSwiftDepsUsingTransitivelyInvalidated(nodes: directlyInvalidatedNodes)
       .reduce(into: Set()) { invalidatedInputs, invalidatedSwiftDeps in
@@ -277,9 +278,9 @@ extension ModuleDependencyGraph {
   /// Return the nodes thus invalidated.
   /// But always integrate, in order to detect future changes.
   /// This function does not to the transitive closure; that is left to the callers
-  func collectNodesDirectlyInvalidatedByProcessing(
+  func collectNodesInvalidatedByProcessing(
     fingerprintedExternalDependency fed: FingerprintedExternalDependency)
-  -> Set<Node> {
+  -> DirectlyInvalidatedNodes {
 
     let isNewToTheGraph = fingerprintedExternalDependencies.insert(fed).inserted
 
@@ -293,13 +294,13 @@ extension ModuleDependencyGraph {
       (isNewToTheGraph || lazyModTimer.hasExternalFileChanged)
 
     // Do this no matter what in order to integrate any incremental external dependencies.
-    let directlyInvalidatedNodesFromIncrementalExternal = shouldTryToProcess
-      ? collectNodesDirectlyInvalidatedByAttemptingToProcess(fed, info)
+    let invalidatedNodesFromIncrementalExternal = shouldTryToProcess
+      ? collectNodesInvalidatedByAttemptingToProcess(fed, info)
       : nil
 
     if phase == .buildingAfterEachCompilation {
       // going to compile every input anyway, less work for callers
-      return Set()
+      return DirectlyInvalidatedNodes()
     }
 
     /// When building a graph from scratch, an unchanged but new-to-the-graph external dependendcy should be ignored.
@@ -308,12 +309,12 @@ extension ModuleDependencyGraph {
       lazyModTimer.hasExternalFileChanged
 
     guard callerWantsTheseChanges else {
-      return Set()
+      return DirectlyInvalidatedNodes()
     }
 
     // If there was an error integrating the external dependency, or if it was not an incremental one,
     // return anything that uses that dependency.
-    return directlyInvalidatedNodesFromIncrementalExternal ?? collectUntracedNodesDirectlyUsing(fed)
+    return invalidatedNodesFromIncrementalExternal ?? collectUntracedNodesUsing(fed)
   }
 
   private struct LazyModTimer {
@@ -326,15 +327,15 @@ extension ModuleDependencyGraph {
 
   /// Try to read and integrate an external dependency.
   /// Return nil if it's not incremental, or if an error occurs.
-  private func collectNodesDirectlyInvalidatedByAttemptingToProcess(
+  private func collectNodesInvalidatedByAttemptingToProcess(
     _ fed: FingerprintedExternalDependency,
-    _ info: IncrementalCompilationState.InitialStateComputer) -> Set<Node>? {
+    _ info: IncrementalCompilationState.InitialStateComputer) -> DirectlyInvalidatedNodes? {
     fed.incrementalDependencySource?
       .read(in: info.fileSystem, reporter: info.reporter)
       .map { unserializedDepGraph in
         info.reporter?.report("Integrating changes from: \(fed.externalDependency)")
         return Integrator.integrate(from: unserializedDepGraph, into: self)
-          .allDirectlyInvalidatedNodes
+          .all
       }
   }
 
@@ -364,10 +365,8 @@ extension OutputFileMap {
 // MARK: - tracking traced nodes
 extension ModuleDependencyGraph {
 
- func ensureGraphWillRetrace<Nodes: Sequence>(_ nodes: Nodes)
-  where Nodes.Element == Node
-  {
-    for node in nodes {
+ func ensureGraphWillRetrace(_ nodes: DirectlyInvalidatedNodes) {
+   for node in nodes.contents {
       node.setUntraced()
     }
   }

Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Integrator.swift

@@ -76,7 +76,7 @@ extension ModuleDependencyGraph.Integrator {
     integrateEachSourceNode()
     handleDisappearedNodes()
     // Ensure transitive closure will get started.
-    destination.ensureGraphWillRetrace(results.allDirectlyInvalidatedNodes)
+    destination.ensureGraphWillRetrace(results.all)
   }
   private mutating func integrateEachSourceNode() {
     sourceGraph.forEachNode { integrate(oneNode: $0) }
@@ -197,9 +197,9 @@ extension ModuleDependencyGraph.Integrator {
     externalDependency fingerprintedExternalDependency: FingerprintedExternalDependency,
     moduleFileGraphUseNode moduleUseNode: Graph.Node) {
 
-    let invalidated = destination.collectNodesDirectlyInvalidatedByProcessing(
+    let invalidated = destination.collectNodesInvalidatedByProcessing(
       fingerprintedExternalDependency: fingerprintedExternalDependency)
-    results.addNodesDirectlyInvalidatedByUsingSomeExternal(invalidated)
+    results.addUsesOfSomeExternal(invalidated)
   }
 }
 
@@ -209,25 +209,25 @@ extension ModuleDependencyGraph.Integrator {
   public struct Results {
     typealias Node = ModuleDependencyGraph.Node
 
-    private(set) var allDirectlyInvalidatedNodes = Set<Node>()
-    private(set) var nodesDirectlyInvalidatedByUsingSomeExternal = Set<Node>()
+    private(set) var all = DirectlyInvalidatedNodes()
+    private(set) var usesOfSomeExternal = DirectlyInvalidatedNodes()
 
-    mutating func addNodesDirectlyInvalidatedByUsingSomeExternal(_ invalidated: Set<Node>)
+    mutating func addUsesOfSomeExternal(_ invalidated: DirectlyInvalidatedNodes)
     {
-      allDirectlyInvalidatedNodes.formUnion(invalidated)
-      allDirectlyInvalidatedNodes.formUnion(invalidated)
+      all.formUnion(invalidated)
+      usesOfSomeExternal.formUnion(invalidated)
     }
     mutating func addDisappeared(_ node: Node) {
-      allDirectlyInvalidatedNodes.insert(node)
+      all.insert(node)
     }
     mutating func addChanged(_ node: Node) {
-      allDirectlyInvalidatedNodes.insert(node)
+      all.insert(node)
     }
     mutating func addPatriated(_ node: Node) {
-      allDirectlyInvalidatedNodes.insert(node)
+      all.insert(node)
     }
     mutating func addNew(_ node: Node) {
-      allDirectlyInvalidatedNodes.insert(node)
+      all.insert(node)
     }
   }
 }

Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Tracer.swift

@@ -37,28 +37,24 @@ extension ModuleDependencyGraph.Tracer {
 
   /// Find all uses of `defs` that have not already been traced.
   /// (If already traced, jobs have already been scheduled.)
-  static func collectPreviouslyUntracedNodesUsing<Nodes: Sequence> (
-    defNodes: Nodes,
+  static func collectPreviouslyUntracedNodesUsing(
+    defNodes: DirectlyInvalidatedNodes,
     in graph: ModuleDependencyGraph,
     diagnosticEngine: DiagnosticsEngine
-  ) -> Self
-  where Nodes.Element == ModuleDependencyGraph.Node
-  {
+  ) -> Self {
     var tracer = Self(collectingUsesOf: defNodes,
                       in: graph,
                       diagnosticEngine: diagnosticEngine)
     tracer.collectPreviouslyUntracedDependents()
     return tracer
   }
 
-  private init<Nodes: Sequence>(collectingUsesOf defs: Nodes,
+  private init(collectingUsesOf defs: DirectlyInvalidatedNodes,
                in graph: ModuleDependencyGraph,
-               diagnosticEngine: DiagnosticsEngine)
-  where Nodes.Element == ModuleDependencyGraph.Node
-  {
+               diagnosticEngine: DiagnosticsEngine) {
     self.graph = graph
     // Sort so "Tracing" diagnostics are deterministically ordered
-    self.startingPoints = defs.sorted()
+    self.startingPoints = defs.contents.sorted()
     self.currentPathIfTracing = graph.info.reporter != nil ? [] : nil
     self.diagnosticEngine = diagnosticEngine
   }

Tests/SwiftDriverTests/ModuleDependencyGraphTests.swift

@@ -984,7 +984,7 @@ extension ModuleDependencyGraph {
     _ interfaceHashIfPresent: String? = nil,
     includePrivateDeps: Bool = true,
     hadCompilationError: Bool = false
-  ) -> Set<Node> {
+  ) -> DirectlyInvalidatedNodes {
     let dependencySource = DependencySource(mock: swiftDepsIndex)
     // Only needed for serialization testing:
     mockMapEntry(TypedVirtualPath.init(mockInput: swiftDepsIndex),
@@ -1001,7 +1001,7 @@ extension ModuleDependencyGraph {
 
     let results = Integrator.integrate(from: sfdg, into: self)
 
-    return results.allDirectlyInvalidatedNodes
+    return results.all
   }
 
   func findUntracedSwiftDepsDependent(onExternal s: String) -> [Int] {
@@ -1015,7 +1015,7 @@ extension ModuleDependencyGraph {
     on fingerprintedExternalDependency: FingerprintedExternalDependency
   ) -> [DependencySource] {
     var foundSources = [DependencySource]()
-    for dependent in self.collectUntracedNodesDirectlyUsing(fingerprintedExternalDependency) {
+    for dependent in collectUntracedNodesUsing(fingerprintedExternalDependency).contents {
       let dependencySource = dependent.dependencySource!
       foundSources.append(dependencySource)
       // findSwiftDepsToRecompileWhenWholeSwiftDepChanges is reflexive