Merge pull request #395 from davidungar/missing-unreadable-swiftdeps

[Incremental] Handle missing or unreadable swiftdeps like the legacy driver.

Author: David Ungar

Sources/SwiftDriver/Incremental Compilation/IncrementalCompilationState.swift

@@ -91,7 +91,7 @@ public class IncrementalCompilationState {
       ? { Self.reportIncrementalDecisionFn($0, $1, outputFileMap, diagnosticEngine) }
       : nil
 
-    guard let moduleDependencyGraph =
+    guard let (moduleDependencyGraph, inputsWithUnreadableSwiftDeps) =
             ModuleDependencyGraph.buildInitialGraph(
               diagnosticEngine: diagnosticEngine,
               inputs: buildRecordInfo.compilationInputModificationDates.keys,
@@ -102,13 +102,23 @@ public class IncrementalCompilationState {
     else {
       return nil
     }
+    // preserve legacy behavior
+    if let badSwiftDeps = inputsWithUnreadableSwiftDeps.first?.1 {
+      diagnosticEngine.emit(
+        .remark_incremental_compilation_disabled(
+          because: "malformed swift dependencies file '\(badSwiftDeps)'")
+      )
+      return nil
+    }
 
+    // But someday, just ensure inputsWithUnreadableSwiftDeps are compiled
     self.skippedCompilationInputs = Self.computeSkippedCompilationInputs(
-        inputFiles: inputFiles,
-        buildRecordInfo: buildRecordInfo,
-        moduleDependencyGraph: moduleDependencyGraph,
-        outOfDateBuildRecord: outOfDateBuildRecord,
-        reportIncrementalDecision: reportIncrementalDecision)
+      inputFiles: inputFiles,
+      inputsWithUnreadableSwiftDeps: inputsWithUnreadableSwiftDeps.map {$0.0},
+      buildRecordInfo: buildRecordInfo,
+      moduleDependencyGraph: moduleDependencyGraph,
+      outOfDateBuildRecord: outOfDateBuildRecord,
+      reportIncrementalDecision: reportIncrementalDecision)
 
     self.moduleDependencyGraph = moduleDependencyGraph
     self.reportIncrementalDecision = reportIncrementalDecision
@@ -194,6 +204,7 @@ extension IncrementalCompilationState {
   /// Figure out which compilation inputs are *not* mandatory
   private static func computeSkippedCompilationInputs(
     inputFiles: [TypedVirtualPath],
+    inputsWithUnreadableSwiftDeps: [TypedVirtualPath],
     buildRecordInfo: BuildRecordInfo,
     moduleDependencyGraph: ModuleDependencyGraph,
     outOfDateBuildRecord: BuildRecord,
@@ -213,7 +224,7 @@ extension IncrementalCompilationState {
       reportIncrementalDecision: reportIncrementalDecision)
 
     // Combine to obtain the inputs that definitely must be recompiled.
-    let definitelyRequiredInputs = Set(changedInputs.map {$0.0} + externalDependents)
+    let definitelyRequiredInputs = Set(changedInputs.map {$0.0} + externalDependents + inputsWithUnreadableSwiftDeps)
     if let report = reportIncrementalDecision {
       for scheduledInput in definitelyRequiredInputs.sorted(by: {$0.file.name < $1.file.name}) {
         report("Queuing (initial):", scheduledInput)
@@ -471,8 +482,12 @@ extension IncrementalCompilationState {
     Array(
       Set(
         job.primaryInputs.flatMap {
-          moduleDependencyGraph.findSourcesToCompileAfterCompiling($0)
-            ?? Array(skippedCompilationInputs)
+          input -> [TypedVirtualPath] in
+          if let found = moduleDependencyGraph.findSourcesToCompileAfterCompiling(input) {
+            return found
+          }
+          reportIncrementalDecision?("Failed to read some swiftdeps; compiling everything", input)
+          return Array(skippedCompilationInputs)
         }
       )
     )

Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph Parts/Integrator.swift

@@ -58,11 +58,14 @@ extension ModuleDependencyGraph.Integrator {
   static func integrate(
     swiftDeps: Graph.SwiftDeps,
     into destination: Graph,
+    input: TypedVirtualPath,
+    reportIncrementalDecision: ((String, TypedVirtualPath) -> Void)?,
     diagnosticEngine: DiagnosticsEngine
   ) -> Changes? {
     guard let sfdg = try? SourceFileDependencyGraph.read(
             from: swiftDeps)
     else {
+      reportIncrementalDecision?("Could not read \(swiftDeps)", input)
       return nil
     }
     return integrate(from: sfdg,

Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph.swift

@@ -49,40 +49,54 @@ final class ModuleDependencyGraph {
 }
 // MARK: - initial build only
 extension ModuleDependencyGraph {
+  /// Builds a graph
+  /// Returns nil if some input has no place to put a swiftdeps file
+  /// Returns a list of inputs whose swiftdeps files could not be read
   static func buildInitialGraph<Inputs: Sequence>(
     diagnosticEngine: DiagnosticsEngine,
     inputs: Inputs,
     outputFileMap: OutputFileMap?,
     parsedOptions: inout ParsedOptions,
     remarkDisabled: (String) -> Diagnostic.Message,
     reportIncrementalDecision: ((String, TypedVirtualPath?) -> Void)?
-  ) -> Self?
+  ) -> (ModuleDependencyGraph, [(TypedVirtualPath, VirtualPath)])?
   where Inputs.Element == TypedVirtualPath
   {
     let emitOpt = Option.driverEmitFineGrainedDependencyDotFileAfterEveryImport
     let veriOpt = Option.driverVerifyFineGrainedDependencyGraphAfterEveryImport
-    let r = Self (
+    let graph = Self (
       diagnosticEngine: diagnosticEngine,
       reportIncrementalDecision: reportIncrementalDecision,
       emitDependencyDotFileAfterEveryImport: parsedOptions.contains(emitOpt),
       verifyDependencyGraphAfterEveryImport: parsedOptions.contains(veriOpt))
-    for input in inputs {
-      guard let swiftDepsFile = outputFileMap?.existingOutput(
-              inputFile: input.file,
-              outputType: .swiftDeps)
+
+    let inputsAndSwiftdeps = inputs.map {input in
+      (input, outputFileMap?.existingOutput( inputFile: input.file,
+                                             outputType: .swiftDeps)
+      )
+    }
+    for isd in inputsAndSwiftdeps where isd.1 == nil {
+      diagnosticEngine.emit(
+        remarkDisabled("\(isd.0.file.basename) has no swiftDeps file")
+      )
+      return nil
+    }
+    let inputsWithUnreadableSwiftDeps = inputsAndSwiftdeps.compactMap {
+      input, swiftDepsFile -> (TypedVirtualPath, VirtualPath)? in
+      guard let swiftDepsFile = swiftDepsFile
       else {
-        diagnosticEngine.emit(
-          remarkDisabled("\(input.file.basename) has no swiftDeps file")
-        )
         return nil
       }
       let swiftDeps = SwiftDeps(swiftDepsFile)
-      r.sourceSwiftDepsMap[input] = swiftDeps
-      _ = Integrator.integrate(swiftDeps: swiftDeps,
-                               into: r,
-                               diagnosticEngine: diagnosticEngine)
+      graph.sourceSwiftDepsMap[input] = swiftDeps
+      let changes = Integrator.integrate(swiftDeps: swiftDeps,
+                                         into: graph,
+                                         input: input,
+                                         reportIncrementalDecision: reportIncrementalDecision,
+                                         diagnosticEngine: diagnosticEngine)
+      return changes == nil ? (input, swiftDepsFile) : nil
     }
-    return r
+    return (graph, inputsWithUnreadableSwiftDeps)
   }
 }
 // MARK: - Scheduling the first wave
@@ -129,17 +143,25 @@ extension ModuleDependencyGraph {
   func findSourcesToCompileAfterCompiling(
     _ source: TypedVirtualPath
   ) -> [TypedVirtualPath]? {
-    findSourcesToCompileAfterIntegrating( sourceSwiftDepsMap[source] )
+    findSourcesToCompileAfterIntegrating(
+      input: source,
+      swiftDeps: sourceSwiftDepsMap[source],
+      reportIncrementalDecision: reportIncrementalDecision
+    )
   }
 
   /// After a compile job has finished, read its swiftDeps file and return the source files needing
   /// recompilation.
   /// Return nil in case of an error.
   private func findSourcesToCompileAfterIntegrating(
-    _ swiftDeps: SwiftDeps
+    input: TypedVirtualPath,
+    swiftDeps: SwiftDeps,
+    reportIncrementalDecision: ((String, TypedVirtualPath) -> Void)?
   ) -> [TypedVirtualPath]? {
     Integrator.integrate(swiftDeps: swiftDeps,
                          into: self,
+                         input: input,
+                         reportIncrementalDecision: reportIncrementalDecision,
                          diagnosticEngine: diagnosticEngine)
       .map {
         findSwiftDepsToRecompileWhenNodesChange($0)