Merge pull request #508 from davidungar/retrench2

[Incremental] Simplify integration logic using a phase concept & Use wrapper types

Author: David Ungar

Sources/SwiftDriver/CMakeLists.txt

@@ -44,6 +44,7 @@ add_library(SwiftDriver
   "IncrementalCompilation/DependencyGraphDotFileWriter.swift"
   "IncrementalCompilation/DependencyKey.swift"
   "IncrementalCompilation/DictionaryOfDictionaries.swift"
+  "IncrementalCompilation/DirectAndTransitiveCollections.swift"
   "IncrementalCompilation/ExternalDependencyAndFingerprintEnforcer.swift"
   "IncrementalCompilation/IncrementalCompilationState.swift"
   "IncrementalCompilation/InitialStateComputer.swift"

Sources/SwiftDriver/IncrementalCompilation/DirectAndTransitiveCollections.swift

@@ -0,0 +1,86 @@
+//===------------------ DirectAndTransitiveCollections.swift --------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+// Use the type system to ensure that dependencies are transitively closed
+// without doing too much work at the leaves of the call tree
+
+public struct Transitively {}
+public struct Directly {}
+
+public struct InvalidatedSet<ClosureLevel, Element: Hashable>: Sequence {
+  var contents: Set<Element>
+
+  init(_ s: Set<Element> = Set()) {
+    self.contents = s
+  }
+  init<Elements: Sequence>(_ elements: Elements)
+  where Elements.Element == Element
+  {
+    self.init(Set(elements))
+  }
+  mutating func insert(_ e: Element) {
+    contents.insert(e)
+  }
+  mutating func formUnion<Elements: Sequence>(_ elements: Elements)
+  where Elements.Element == Element{
+    contents.formUnion(elements)
+  }
+  public func makeIterator() -> Set<Element>.Iterator {
+    contents.makeIterator()
+  }
+  public func map<R>(_ transform: (Element) -> R) -> InvalidatedArray<ClosureLevel, R> {
+    InvalidatedArray(contents.map(transform))
+  }
+  public func compactMap<R>(_ transform: (Element) -> R? ) -> InvalidatedArray<ClosureLevel, R> {
+    InvalidatedArray(contents.compactMap(transform))
+  }
+}
+
+extension InvalidatedSet where Element: Comparable {
+  func sorted() -> InvalidatedArray<ClosureLevel, Element> {
+    InvalidatedArray(contents.sorted())
+  }
+}
+
+public struct InvalidatedArray<ClosureLevel, Element>: Sequence {
+  var contents: [Element]
+
+  init(_ s: [Element] = []) {
+    self.contents = s
+  }
+  init<Elements: Sequence>(_ elements: Elements)
+  where Elements.Element == Element
+  {
+    self.init(Array(elements))
+  }
+  public func makeIterator() -> Array<Element>.Iterator {
+    contents.makeIterator()
+  }
+  public mutating func append(_ e: Element) {
+    contents.append(e)
+  }
+  public func reduce<R: Hashable>(
+    into initialResult: InvalidatedSet<ClosureLevel, R>,
+    _ updateAccumulatingResult: (inout Set<R>, Element) -> ()
+  ) -> InvalidatedSet<ClosureLevel, R> {
+    InvalidatedSet(
+      contents.reduce(into: initialResult.contents, updateAccumulatingResult))
+  }
+  public var count: Int { contents.count }
+}
+
+public typealias TransitivelyInvalidatedNodeArray = InvalidatedArray<Transitively, ModuleDependencyGraph.Node>
+public typealias TransitivelyInvalidatedSourceSet = InvalidatedSet<Transitively, DependencySource>
+public typealias TransitivelyInvalidatedInputArray = InvalidatedArray<Transitively, TypedVirtualPath>
+public typealias TransitivelyInvalidatedInputSet = InvalidatedSet<Transitively, TypedVirtualPath>
+public typealias DirectlyInvalidatedNodeArray = InvalidatedArray<Directly, ModuleDependencyGraph.Node>
+public typealias DirectlyInvalidatedNodeSet = InvalidatedSet<Directly, ModuleDependencyGraph.Node>

Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift

@@ -182,9 +182,6 @@ extension Diagnostic.Message {
         "output file map has no master dependencies entry (\"\(FileType.swiftDeps)\" under \"\")"
     )
   }
-  fileprivate static func remark_disabling_incremental_build(because why: String) -> Diagnostic.Message {
-    return .remark("Disabling incremental build: \(why)")
-  }
 
   static let remarkDisabled = Diagnostic.Message.remark_incremental_compilation_has_been_disabled
 
@@ -202,13 +199,6 @@ extension Diagnostic.Message {
 }
 
 
-// MARK: - Scheduling the first wave, i.e. the mandatory pre- and compile jobs
-
-extension IncrementalCompilationState {
-
-
-}
-
 // MARK: - Scheduling the 2nd wave
 extension IncrementalCompilationState {
   /// Remember a job (group) that is before a compile or a compile itself.
@@ -259,13 +249,13 @@ extension IncrementalCompilationState {
   }
 
   private func collectInputsInvalidated(byCompiling input: TypedVirtualPath)
-  -> Set<TypedVirtualPath> {
+  -> TransitivelyInvalidatedInputSet {
     if let found = moduleDependencyGraph.collectInputsRequiringCompilation(byCompiling: input) {
       return found
     }
     self.reporter?.report(
       "Failed to read some dependencies source; compiling everything", input)
-    return Set<TypedVirtualPath>(skippedCompileGroups.keys)
+    return TransitivelyInvalidatedInputSet(skippedCompileGroups.keys)
   }
 
   /// Find the jobs that now must be run that were not originally known to be needed.

Sources/SwiftDriver/IncrementalCompilation/InitialStateComputer.swift

@@ -116,7 +116,7 @@ extension IncrementalCompilationState.InitialStateComputer {
   /// For inputs with swiftDeps in OFM, but no readable file, puts input in graph map, but no nodes in graph:
   ///   caller must ensure scheduling of those
   private func computeGraphAndInputsInvalidatedByExternals()
-  -> (ModuleDependencyGraph, Set<TypedVirtualPath>)? {
+  -> (ModuleDependencyGraph, TransitivelyInvalidatedInputSet)? {
     precondition(sourceFiles.disappeared.isEmpty,
                  "Would have to remove nodes from the graph if reading prior")
     if readPriorsFromModuleDependencyGraph {
@@ -128,7 +128,7 @@ extension IncrementalCompilationState.InitialStateComputer {
   }
 
   private func readPriorGraphAndCollectInputsInvalidatedByChangedOrAddedExternals(
-  ) -> (ModuleDependencyGraph, Set<TypedVirtualPath>)?
+  ) -> (ModuleDependencyGraph, TransitivelyInvalidatedInputSet)?
   {
     let dependencyGraphPath = buildRecordInfo.dependencyGraphPath
     let graphIfPresent: ModuleDependencyGraph?
@@ -152,8 +152,10 @@ extension IncrementalCompilationState.InitialStateComputer {
 
     // Any externals not already in graph must be additions which should trigger
     // recompilation. Thus, `ChangedOrAdded`.
-    let nodesInvalidatedByExternals = graph.collectNodesInvalidatedByChangedOrAddedExternals()
-    let inputsInvalidatedByExternals = graph.collectInputsUsingTransitivelyInvalidated(nodes: nodesInvalidatedByExternals)
+    let nodesDirectlyInvalidatedByExternals = graph.collectNodesInvalidatedByChangedOrAddedExternals()
+    // Wait till the last minute to do the transitive closure as an optimization.
+    let inputsInvalidatedByExternals = graph.collectInputsUsingInvalidated(
+      nodes: nodesDirectlyInvalidatedByExternals)
     return (graph, inputsInvalidatedByExternals)
   }
 
@@ -165,17 +167,15 @@ extension IncrementalCompilationState.InitialStateComputer {
   /// For externalDependencies, puts then in graph.fingerprintedExternalDependencies, but otherwise
   /// does nothing special.
   private func buildInitialGraphFromSwiftDepsAndCollectInputsInvalidatedByChangedExternals(
-  ) -> (ModuleDependencyGraph, Set<TypedVirtualPath>)?
+  ) -> (ModuleDependencyGraph, TransitivelyInvalidatedInputSet)?
   {
-    let graph = ModuleDependencyGraph(self)
+    let graph = ModuleDependencyGraph(self, .buildingWithoutAPrior)
     assert(outputFileMap.onlySourceFilesHaveSwiftDeps())
     guard graph.populateInputDependencySourceMap() else {
       return nil
     }
 
-    // Every external will be an addition to the graph, but may not cause
-    // a recompile, so includeAddedExternals is false.
-    var inputsInvalidatedByChangedExternals = Set<TypedVirtualPath>()
+    var inputsInvalidatedByChangedExternals = TransitivelyInvalidatedInputSet()
     for input in sourceFiles.currentInOrder {
        guard let invalidatedInputs = graph.collectInputsRequiringCompilationFromExternalsFoundByCompiling(input: input)
       else {
@@ -195,7 +195,7 @@ extension IncrementalCompilationState.InitialStateComputer {
   /// listed in fingerprintExternalDependencies.
   private func computeInputsAndGroups(
     _ moduleDependencyGraph: ModuleDependencyGraph,
-    _ inputsInvalidatedByExternals: Set<TypedVirtualPath>,
+    _ inputsInvalidatedByExternals: TransitivelyInvalidatedInputSet,
     batchJobFormer: inout Driver
   ) throws -> (skippedCompileGroups: [TypedVirtualPath: CompileJobGroup],
                mandatoryJobsInOrder: [Job])
@@ -215,9 +215,13 @@ extension IncrementalCompilationState.InitialStateComputer {
         batchJobFormer.formBatchedJobs(
           mandatoryCompileGroupsInOrder.flatMap {$0.allJobs()},
           showJobLifecycle: showJobLifecycle)
+
+      moduleDependencyGraph.phase = .buildingAfterEachCompilation
       return (skippedCompileGroups: [:],
               mandatoryJobsInOrder: mandatoryJobsInOrder)
     }
+    moduleDependencyGraph.phase = .updatingAfterCompilation
+
 
     let skippedInputs = computeSkippedCompilationInputs(
       inputsInvalidatedByExternals: inputsInvalidatedByExternals,
@@ -245,7 +249,7 @@ extension IncrementalCompilationState.InitialStateComputer {
 
   /// Figure out which compilation inputs are *not* mandatory
   private func computeSkippedCompilationInputs(
-    inputsInvalidatedByExternals: Set<TypedVirtualPath>,
+    inputsInvalidatedByExternals: TransitivelyInvalidatedInputSet,
     _ moduleDependencyGraph: ModuleDependencyGraph,
     _ buildRecord: BuildRecord
   ) -> Set<TypedVirtualPath> {
@@ -281,7 +285,7 @@ extension IncrementalCompilationState.InitialStateComputer {
     // as each first wave job finished.
     let speculativeInputs = collectInputsToBeSpeculativelyRecompiled(
       changedInputs: changedInputs,
-      externalDependents: Array(inputsInvalidatedByExternals),
+      externalDependents: inputsInvalidatedByExternals,
       inputsMissingOutputs: Set(inputsMissingOutputs),
       moduleDependencyGraph)
       .subtracting(definitelyRequiredInputs)
@@ -367,28 +371,28 @@ extension IncrementalCompilationState.InitialStateComputer {
   /// before the whole frontend job finished.
   private func collectInputsToBeSpeculativelyRecompiled(
     changedInputs: [ChangedInput],
-    externalDependents: [TypedVirtualPath],
+    externalDependents: TransitivelyInvalidatedInputSet,
     inputsMissingOutputs: Set<TypedVirtualPath>,
     _ moduleDependencyGraph: ModuleDependencyGraph
   ) -> Set<TypedVirtualPath> {
     let cascadingChangedInputs = computeCascadingChangedInputs(
       from: changedInputs,
       inputsMissingOutputs: inputsMissingOutputs)
-    let cascadingExternalDependents = alwaysRebuildDependents ? externalDependents : []
-    // Collect the dependent files to speculatively schedule
-    var dependentFiles = Set<TypedVirtualPath>()
-    let cascadingFileSet = Set(cascadingChangedInputs).union(cascadingExternalDependents)
-    for cascadingFile in cascadingFileSet {
-      let dependentsOfOneFile = moduleDependencyGraph
-        .collectInputsTransitivelyInvalidatedBy(input: cascadingFile)
-      for dep in dependentsOfOneFile where !cascadingFileSet.contains(dep) {
-        if dependentFiles.insert(dep).0 {
+
+    var inputsToBeCertainlyRecompiled = alwaysRebuildDependents ? externalDependents : TransitivelyInvalidatedInputSet()
+    inputsToBeCertainlyRecompiled.formUnion(cascadingChangedInputs)
+
+    return inputsToBeCertainlyRecompiled.reduce(into: Set()) {
+      speculativelyRecompiledInputs, certainlyRecompiledInput in
+      let speculativeDependents = moduleDependencyGraph.collectInputsInvalidatedBy(input: certainlyRecompiledInput)
+      for speculativeDependent in speculativeDependents
+      where !inputsToBeCertainlyRecompiled.contains(speculativeDependent) {
+        if speculativelyRecompiledInputs.insert(speculativeDependent).inserted {
           reporter?.report(
-            "Immediately scheduling dependent on \(cascadingFile.file.basename)", dep)
+            "Immediately scheduling dependent on \(certainlyRecompiledInput.file.basename)", speculativeDependent)
         }
       }
     }
-    return dependentFiles
   }
 
   // Collect the files that will be compiled whose dependents should be schedule