Merge pull request #494 from davidungar/def-use-serialization

[Incremental] Add `inputDependencySourceMap` to `ModuleDependencyGraph` serialization

Author: David Ungar

Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift

@@ -387,6 +387,7 @@ extension ModuleDependencyGraph {
     case useIDNode          = 4
     case externalDepNode    = 5
     case identifierNode     = 6
+    case mapNode            = 7
 
     /// The human-readable name of this record.
     ///
@@ -407,6 +408,8 @@ extension ModuleDependencyGraph {
         return "EXTERNAL_DEP_NODE"
       case .identifierNode:
         return "IDENTIFIER_NODE"
+      case .mapNode:
+        return "MAP_NODE"
       }
     }
   }
@@ -420,6 +423,7 @@ extension ModuleDependencyGraph {
     case malformedIdentifierRecord
     case malformedModuleDepGraphNodeRecord
     case malformedDependsOnRecord
+    case malformedMapRecord
     case malformedExternalDepNodeRecord
     case unknownRecord
     case unexpectedSubblock
@@ -436,14 +440,17 @@ extension ModuleDependencyGraph {
   ///   - diagnosticEngine: The diagnostics engine.
   ///   - reporter: An optional reporter used to log information about
   /// - Throws: An error describing any failures to read the graph from the given file.
-  /// - Returns: A fully deserialized ModuleDependencyGraph.
+  /// - Returns: A fully deserialized ModuleDependencyGraph, or nil if nothing is there
   static func read(
     from path: VirtualPath,
     on fileSystem: FileSystem,
     diagnosticEngine: DiagnosticsEngine,
     reporter: IncrementalCompilationState.Reporter?,
     options: IncrementalCompilationState.Options
-  ) throws -> ModuleDependencyGraph {
+  ) throws -> ModuleDependencyGraph? {
+    guard try fileSystem.exists(path) else {
+      return nil
+    }
     let data = try fileSystem.readFileContents(path)
 
     struct Visitor: BitstreamVisitor {
@@ -456,6 +463,7 @@ extension ModuleDependencyGraph {
       private var identifiers: [String] = [""]
       private var currentDefKey: DependencyKey? = nil
       private var nodeUses: [DependencyKey: [Int]] = [:]
+      private var inputDependencySourceMap: [(TypedVirtualPath, DependencySource)] = []
       private var allNodes: [Node] = []
 
       init(
@@ -467,7 +475,7 @@ extension ModuleDependencyGraph {
         self.graph = ModuleDependencyGraph(
           diagnosticEngine: diagnosticEngine,
           reporter: reporter,
-		  fileSystem: fileSystem,
+          fileSystem: fileSystem,
           options: options)
       }
 
@@ -479,6 +487,10 @@ extension ModuleDependencyGraph {
             assert(isNewUse, "Duplicate use def-use arc in graph?")
           }
         }
+        for (input, source) in inputDependencySourceMap {
+          graph.inputDependencySourceMap[input] = source
+        }
+
         return self.graph
       }
 
@@ -568,6 +580,25 @@ extension ModuleDependencyGraph {
             throw ReadError.malformedDependsOnRecord
           }
           self.nodeUses[key, default: []].append(Int(record.fields[0]))
+        case .mapNode:
+          guard record.fields.count == 2,
+                record.fields[0] < identifiers.count,
+                record.fields[1] < identifiers.count
+          else {
+            throw ReadError.malformedModuleDepGraphNodeRecord
+          }
+          let inputPathString = identifiers[Int(record.fields[0])]
+          let dependencySourcePathString = identifiers[Int(record.fields[1])]
+          let inputPath = try VirtualPath(path: inputPathString)
+          let dependencySourcePath = try VirtualPath(path: dependencySourcePathString)
+          guard inputPath.extension == FileType.swift.rawValue,
+                dependencySourcePath.extension == FileType.swiftDeps.rawValue,
+                let dependencySource = DependencySource(dependencySourcePath)
+          else {
+            throw ReadError.malformedMapRecord
+          }
+          let input = TypedVirtualPath(file: inputPath, type: .swift)
+          inputDependencySourceMap.append((input, dependencySource))
         case .externalDepNode:
           guard record.fields.count == 2,
                 record.fields[0] < identifiers.count,
@@ -605,7 +636,9 @@ extension ModuleDependencyGraph {
     else {
       throw ReadError.malformedMetadataRecord
     }
-    return visitor.finalizeGraph()
+    let graph = visitor.finalizeGraph()
+    graph.reporter?.report("Read dependency graph", path)
+    return graph
   }
 }
 
@@ -690,6 +723,7 @@ extension ModuleDependencyGraph {
         self.emitRecordID(.useIDNode)
         self.emitRecordID(.externalDepNode)
         self.emitRecordID(.identifierNode)
+        self.emitRecordID(.mapNode)
       }
     }
 
@@ -751,6 +785,11 @@ extension ModuleDependencyGraph {
         }
       }
 
+      for (input, dependencySource) in graph.inputDependencySourceMap {
+        self.addIdentifier(input.file.name)
+        self.addIdentifier(dependencySource.file.name)
+      }
+
       for edF in graph.fingerprintedExternalDependencies {
         self.addIdentifier(edF.externalDependency.file.name)
       }
@@ -822,7 +861,14 @@ extension ModuleDependencyGraph {
         // identifier data
         .blob
       ])
-    }
+      self.abbreviate(.mapNode, [
+        .literal(RecordID.mapNode.rawValue),
+        // input name
+        .vbr(chunkBitWidth: 13),
+        // dependencySource name
+        .vbr(chunkBitWidth: 13),
+      ])
+   }
 
     private func abbreviate(
       _ record: RecordID,
@@ -884,6 +930,13 @@ extension ModuleDependencyGraph {
             }
           }
         }
+        for (input, dependencySource) in graph.inputDependencySourceMap {
+          serializer.stream.writeRecord(serializer.abbreviations[.mapNode]!) {
+            $0.append(RecordID.mapNode)
+            $0.append(serializer.lookupIdentifierCode(for: input.file.name))
+            $0.append(serializer.lookupIdentifierCode(for: dependencySource.file.name))
+          }
+        }
 
         for fingerprintedExternalDependency in graph.fingerprintedExternalDependencies {
           serializer.stream.writeRecord(serializer.abbreviations[.externalDepNode]!, {
@@ -1053,3 +1106,11 @@ extension Set where Element == FingerprintedExternalDependency {
     self == other
   }
 }
+
+// MARK: - Testing
+extension ModuleDependencyGraph {
+  /// Must be here so that the setter can be private
+  func recordMapping(mockInput input: TypedVirtualPath, mockDependencySource dependencySource: DependencySource) {
+    inputDependencySourceMap[input] = dependencySource
+  }
+}

Tests/SwiftDriverTests/DependencyGraphSerializationTests.swift

@@ -21,11 +21,12 @@ class DependencyGraphSerializationTests: XCTestCase {
     let fs = InMemoryFileSystem()
     graph.write(to: mockPath, on: fs, compilerVersion: "Swift 99")
 
-    let deserializedGraph = try ModuleDependencyGraph.read(from: mockPath,
-                                                           on: fs,
-                                                           diagnosticEngine: de,
-                                                           reporter: nil,
-                                                           options: [])
+    let deserializedGraph = try XCTUnwrap(
+      try ModuleDependencyGraph.read(from: mockPath,
+                                     on: fs,
+                                     diagnosticEngine: de,
+                                     reporter: nil,
+                                     options: []))
     var originalNodes = Set<ModuleDependencyGraph.Node>()
     graph.nodeFinder.forEachNode {
       originalNodes.insert($0)

Tests/SwiftDriverTests/Helpers/MockingIncrementalCompilation.swift

@@ -30,6 +30,13 @@ extension ModuleDependencyGraph {
 }
 
 // MARK: - mocking
+
+extension TypedVirtualPath {
+  init(mockInput i: Int) {
+    self.init(file: try! VirtualPath(path: "\(i).swift"), type: .swift)
+  }
+}
+
 extension DependencySource {
   init(mock i: Int) {
     self.init(try! VirtualPath(path: String(i) + "." + FileType.swiftDeps.rawValue))!

Tests/SwiftDriverTests/ModuleDependencyGraphTests.swift

@@ -927,6 +927,9 @@ extension ModuleDependencyGraph {
     hadCompilationError: Bool = false
   ) -> Set<ModuleDependencyGraph.Node> {
     let dependencySource = DependencySource(mock: swiftDepsIndex)
+    // Only needed for serialization testing:
+    recordMapping(mockInput: TypedVirtualPath.init(mockInput: swiftDepsIndex),
+                  mockDependencySource: dependencySource)
     let interfaceHash =
       interfaceHashIfPresent ?? dependencySource.interfaceHashForMockDependencySource