Merge pull request #236 from artemcm/BatchClangScanAgain

artemcm · web-flow · commit a065fa940209 · 2020-09-08T09:30:10.000-07:00
[Explicit Module Builds] Use the new batch scanning mode for versioned PCM clang modele re-scan
diff --git a/Sources/SwiftDriver/CMakeLists.txt b/Sources/SwiftDriver/CMakeLists.txt
@@ -13,7 +13,7 @@ add_library(SwiftDriver
   "Explicit Module Builds/ClangVersionedDependencyResolution.swift"
   "Explicit Module Builds/InterModuleDependencyGraph.swift"
   "Explicit Module Builds/ModuleDependencyScanning.swift"
-  "Explicit Module Builds/ModuleArtifacts.swift"
+  "Explicit Module Builds/SerializableModuleArtifacts.swift"
 
   Driver/CompilerMode.swift
   Driver/DebugInfo.swift
diff --git a/Sources/SwiftDriver/Execution/DriverExecutor.swift b/Sources/SwiftDriver/Execution/DriverExecutor.swift
@@ -51,15 +51,7 @@ extension DriverExecutor {
                              recordedInputModificationDates: recordedInputModificationDates)
     
     if (result.exitStatus != .terminated(code: EXIT_SUCCESS)) {
-      let returnCode: Int
-      switch result.exitStatus {
-      case .terminated(let code):
-        returnCode = Int(code)
-#if !os(Windows)
-      case .signalled(let signal):
-        returnCode = Int(signal)
-#endif
-      }
+      let returnCode = Self.computeReturnCode(exitStatus: result.exitStatus)
       throw JobExecutionError.jobFailedWithNonzeroExitCode(returnCode, try result.utf8stderrOutput())
     }
     guard let outputData = try? Data(result.utf8Output().utf8) else {
@@ -68,6 +60,19 @@ extension DriverExecutor {
     
     return try JSONDecoder().decode(outputType, from: outputData)
   }
+
+  static func computeReturnCode(exitStatus: ProcessResult.ExitStatus) -> Int {
+    var returnCode: Int
+    switch exitStatus {
+      case .terminated(let code):
+        returnCode = Int(code)
+      #if !os(Windows)
+      case .signalled(let signal):
+        returnCode = Int(signal)
+      #endif
+    }
+    return returnCode
+  }
 }
 
 public protocol JobExecutionDelegate {
diff --git a/Sources/SwiftDriver/Explicit Module Builds/ClangVersionedDependencyResolution.swift b/Sources/SwiftDriver/Explicit Module Builds/ClangVersionedDependencyResolution.swift
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 import Foundation
+import TSCBasic
 
 /// A map from a module identifier to a set of module dependency graphs
 /// Used to compute distinct graphs corresponding to different target versions for a given clang module
@@ -30,19 +31,31 @@ internal extension Driver {
     // to all Clang modules, and compute a set of distinct PCMArgs across all paths to a
     // given Clang module in the graph.
     let modulePCMArgsSetMap = try dependencyGraph.computePCMArgSetsForClangModules()
-    var moduleVersionedGraphMap: [ModuleDependencyId: [InterModuleDependencyGraph]] = [:]
-    for (moduleId, pcmArgSet) in modulePCMArgsSetMap {
-      for pcmArgs in pcmArgSet {
-        let pcmSpecificDepGraph = try scanClangModule(moduleId: moduleId,
-                                                      pcmArgs: pcmArgs)
-        if moduleVersionedGraphMap[moduleId] != nil {
-          moduleVersionedGraphMap[moduleId]!.append(pcmSpecificDepGraph)
-        } else {
-          moduleVersionedGraphMap[moduleId] = [pcmSpecificDepGraph]
+    let temporaryDirectory = try determineTempDirectory()
+    let batchScanInputList =
+      try modulePCMArgsSetMap.compactMap { (moduleId, pcmArgsSet) throws -> [BatchScanModuleInfo] in
+        var moduleInfos: [BatchScanModuleInfo] = []
+        for pcmArgs in pcmArgsSet {
+          var hasher = Hasher()
+          pcmArgs.forEach { hasher.combine($0) }
+          // Generate a filepath for the output dependency graph
+          let moduleDependencyGraphPath =
+            temporaryDirectory.appending(component: moduleId.moduleName +
+                                          String(hasher.finalize()) +
+                                          "-dependencies.json")
+          let moduleBatchInfo =
+            BatchScanModuleInfo.clang(
+              BatchScanClangModuleInfo(moduleName: moduleId.moduleName,
+                                       pcmArgs: pcmArgs.joined(separator: " "),
+                                       outputPath: moduleDependencyGraphPath.description))
+          moduleInfos.append(moduleBatchInfo)
         }
-      }
-    }
+        return moduleInfos
+      }.reduce([], +)
 
+    // Batch scan all clang modules for each discovered unique set of PCMArgs, per module
+    let moduleVersionedGraphMap: [ModuleDependencyId: [InterModuleDependencyGraph]] =
+      try performBatchDependencyScan(moduleInfos: batchScanInputList)
     try dependencyGraph.resolveVersionedClangModules(using: moduleVersionedGraphMap)
   }
 }
diff --git a/Sources/SwiftDriver/Explicit Module Builds/ModuleDependencyScanning.swift b/Sources/SwiftDriver/Explicit Module Builds/ModuleDependencyScanning.swift
@@ -24,9 +24,6 @@ extension Driver {
     var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
     commandLine.appendFlag("-frontend")
     commandLine.appendFlag("-scan-dependencies")
-    if parsedOptions.hasArgument(.parseStdlib) {
-       commandLine.appendFlag(.disableObjcAttrRequiresFoundationModule)
-    }
     try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs,
                                  bridgingHeaderHandling: .precompiled,
                                  moduleDependencyGraphUse: .dependencyScan)
@@ -75,65 +72,105 @@ extension Driver {
     return placeholderMapFilePath
   }
 
-  /// Compute the dependencies for a given Clang module, by invoking the Clang dependency scanning action
-  /// with the given module's name and a set of arguments (including the target version)
-  mutating func clangDependencyScanningJob(moduleId: ModuleDependencyId,
-                                           pcmArgs: [String]) throws -> Job {
+  mutating func performBatchDependencyScan(moduleInfos: [BatchScanModuleInfo])
+  throws -> [ModuleDependencyId: [InterModuleDependencyGraph]] {
+    let batchScanningJob = try batchDependencyScanningJob(for: moduleInfos)
+    let forceResponseFiles = parsedOptions.hasArgument(.driverForceResponseFiles)
+    let batchScanResult =
+      try self.executor.execute(job: batchScanningJob,
+                                forceResponseFiles: forceResponseFiles,
+                                recordedInputModificationDates: recordedInputModificationDates)
+    let success = batchScanResult.exitStatus == .terminated(code: EXIT_SUCCESS)
+    guard success else {
+      throw JobExecutionError.jobFailedWithNonzeroExitCode(
+        SwiftDriverExecutor.computeReturnCode(exitStatus: batchScanResult.exitStatus),
+        try batchScanResult.utf8stderrOutput())
+    }
+
+    // Decode the resulting dependency graphs and build a dictionary from a moduleId to
+    // a set of dependency graphs that were built for it
+    let moduleVersionedGraphMap =
+      try moduleInfos.reduce(into: [ModuleDependencyId: [InterModuleDependencyGraph]]()) {
+      let moduleId: ModuleDependencyId
+      let dependencyGraphPath: VirtualPath
+      switch $1 {
+        case .swift(let swiftModuleBatchScanInfo):
+          moduleId = .swift(swiftModuleBatchScanInfo.swiftModuleName)
+          dependencyGraphPath = try VirtualPath(path: swiftModuleBatchScanInfo.output)
+        case .clang(let clangModuleBatchScanInfo):
+          moduleId = .clang(clangModuleBatchScanInfo.clangModuleName)
+          dependencyGraphPath = try VirtualPath(path: clangModuleBatchScanInfo.output)
+      }
+      let contents = try fileSystem.readFileContents(dependencyGraphPath)
+      let decodedGraph = try JSONDecoder().decode(InterModuleDependencyGraph.self,
+                                            from: Data(contents.contents))
+      if $0[moduleId] != nil {
+        $0[moduleId]!.append(decodedGraph)
+      } else {
+        $0[moduleId] = [decodedGraph]
+      }
+    }
+    return moduleVersionedGraphMap
+  }
+
+  /// Precompute the dependencies for a given collection of modules using swift frontend's batch scanning mode
+  mutating func batchDependencyScanningJob(for moduleInfos: [BatchScanModuleInfo]) throws -> Job {
     var inputs: [TypedVirtualPath] = []
 
     // Aggregate the fast dependency scanner arguments
     var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
     commandLine.appendFlag("-frontend")
-    commandLine.appendFlag("-scan-clang-dependencies")
-
+    // The dependency scanner automatically operates in batch mode if -batch-scan-input-file
+    // is present.
+    commandLine.appendFlag("-scan-dependencies")
     try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs,
                                  bridgingHeaderHandling: .precompiled,
                                  moduleDependencyGraphUse: .dependencyScan)
 
-    // Ensure the `-target` option is inherited from the dependent Swift module's PCM args
-    if let targetOptionIndex = pcmArgs.firstIndex(of: Option.target.spelling) {
-      // PCM args are formulated as Clang command line options specified with:
-      // -Xcc <option> -Xcc <option_value>
-      assert(pcmArgs.count > targetOptionIndex + 1 && pcmArgs[targetOptionIndex + 1] == "-Xcc")
-      let pcmArgTriple = Triple(pcmArgs[targetOptionIndex + 2])
-      // Override the invocation's default target argument by appending the one extracted from
-      // the pcmArgs
-      commandLine.appendFlag(.target)
-      commandLine.appendFlag(pcmArgTriple.triple)
-    }
-
-    // Add the PCM args specific to this scan
-    pcmArgs.forEach { commandLine.appendFlags($0) }
+    let batchScanInputFilePath = try serializeBatchScanningModuleArtifacts(moduleInfos: moduleInfos)
+    commandLine.appendFlag("-batch-scan-input-file")
+    commandLine.appendPath(batchScanInputFilePath)
 
     // This action does not require any input files, but all frontend actions require
     // at least one input so pick any input of the current compilation.
     let inputFile = inputFiles.first { $0.type == .swift }
     commandLine.appendPath(inputFile!.file)
     inputs.append(inputFile!)
 
-    commandLine.appendFlags("-module-name", moduleId.moduleName)
+    // This job's outputs are defined as a set of dependency graph json files
+    let outputs: [TypedVirtualPath] = try moduleInfos.map {
+      switch $0 {
+        case .swift(let swiftModuleBatchScanInfo):
+          return TypedVirtualPath(file: try VirtualPath(path: swiftModuleBatchScanInfo.output),
+                                  type: .jsonDependencies)
+        case .clang(let clangModuleBatchScanInfo):
+          return TypedVirtualPath(file: try VirtualPath(path: clangModuleBatchScanInfo.output),
+                                  type: .jsonDependencies)
+      }
+    }
+
     // Construct the scanning job.
     return Job(moduleName: moduleOutputInfo.name,
-               kind: .scanClangDependencies,
+               kind: .scanDependencies,
                tool: VirtualPath.absolute(try toolchain.getToolPath(.swiftCompiler)),
                commandLine: commandLine,
                displayInputs: inputs,
                inputs: inputs,
-               outputs: [TypedVirtualPath(file: .standardOutput, type: .jsonDependencies)],
+               outputs: outputs,
                supportsResponseFiles: true)
   }
 
-  mutating func scanClangModule(moduleId: ModuleDependencyId, pcmArgs: [String])
-  throws -> InterModuleDependencyGraph {
-    let clangDependencyScannerJob = try clangDependencyScanningJob(moduleId: moduleId,
-                                                                   pcmArgs: pcmArgs)
-    let forceResponseFiles = parsedOptions.hasArgument(.driverForceResponseFiles)
+  /// Serialize a collection of modules into an input format expected by the batch module dependency scanner.
+  func serializeBatchScanningModuleArtifacts(moduleInfos: [BatchScanModuleInfo])
+  throws -> AbsolutePath {
+    let temporaryDirectory = try determineTempDirectory()
+    let batchScanInputFilePath =
+      temporaryDirectory.appending(component: "\(moduleOutputInfo.name)-batch-module-scan.json")
 
-    let dependencyGraph =
-      try self.executor.execute(job: clangDependencyScannerJob,
-                                capturingJSONOutputAs: InterModuleDependencyGraph.self,
-                                forceResponseFiles: forceResponseFiles,
-                                recordedInputModificationDates: recordedInputModificationDates)
-    return dependencyGraph
+    let encoder = JSONEncoder()
+    encoder.outputFormatting = [.prettyPrinted]
+    let contents = try encoder.encode(moduleInfos)
+    try fileSystem.writeFileContents(batchScanInputFilePath, bytes: ByteString(contents))
+    return batchScanInputFilePath
   }
 }
diff --git a/Sources/SwiftDriver/Explicit Module Builds/SerializableModuleArtifacts.swift b/Sources/SwiftDriver/Explicit Module Builds/SerializableModuleArtifacts.swift
@@ -16,7 +16,7 @@ import Foundation
 /// - Swift Module Path
 /// - Swift Doc Path
 /// - Swift Source Info Path
-public struct SwiftModuleArtifactInfo: Codable {
+@_spi(Testing) public struct SwiftModuleArtifactInfo: Codable {
   /// The module's name
   public let moduleName: String
   /// The path for the module's .swiftmodule file
@@ -42,7 +42,7 @@ public struct SwiftModuleArtifactInfo: Codable {
 /// - Clang Module (name)
 /// - Clang Module (PCM) Path
 /// - Clang Module Map Path
-public struct ClangModuleArtifactInfo {
+@_spi(Testing) public struct ClangModuleArtifactInfo: Codable {
   /// The module's name
   public let moduleName: String
   /// The path for the module's .pcm file
@@ -56,3 +56,46 @@ public struct ClangModuleArtifactInfo {
     self.moduleMapPath = moduleMapPath
   }
 }
+
+/// Describes a given module's batch dependency scanning input info
+/// - Module Name
+/// - Extra PCM build arguments (for Clang modules only)
+/// - Dependency graph output path
+internal enum BatchScanModuleInfo: Encodable {
+  case swift(BatchScanSwiftModuleInfo)
+  case clang(BatchScanClangModuleInfo)
+}
+
+internal struct BatchScanSwiftModuleInfo: Encodable {
+  var swiftModuleName: String
+  var output: String
+
+  init(moduleName: String, outputPath: String) {
+    self.swiftModuleName = moduleName
+    self.output = outputPath
+  }
+}
+
+internal struct BatchScanClangModuleInfo: Encodable {
+  var clangModuleName: String
+  var arguments: String
+  var output: String
+
+  init(moduleName: String, pcmArgs: String, outputPath: String) {
+    self.clangModuleName = moduleName
+    self.arguments = pcmArgs
+    self.output = outputPath
+  }
+}
+
+internal extension BatchScanModuleInfo {
+  func encode(to encoder: Encoder) throws {
+    var container = encoder.singleValueContainer()
+    switch self {
+      case .swift(let swiftInfo):
+        try container.encode(swiftInfo)
+      case .clang(let clangInfo):
+        try container.encode(clangInfo)
+    }
+  }
+}
diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift
@@ -133,6 +133,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
   func testModuleDependencyBuildCommandGeneration() throws {
     do {
       var driver = try Driver(args: ["swiftc", "-driver-print-module-dependencies-jobs",
+                                     "-module-name", "testModuleDependencyBuildCommandGeneration",
                                      "test.swift"])
       let pcmArgs = ["-Xcc","-target","-Xcc","x86_64-apple-macosx10.15"]
       let moduleDependencyGraph =
@@ -250,7 +251,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
   /// is invoked with -experimental-explicit-module-build
   func testExplicitModuleBuildJobs() throws {
     try withTemporaryDirectory { path in
-      let main = path.appending(component: "main.swift")
+      let main = path.appending(component: "testExplicitModuleBuildJobs.swift")
       try localFileSystem.writeFileContents(main) {
         $0 <<< "import C;"
         $0 <<< "import E;"
@@ -328,7 +329,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
           case .relative(try pcmArgsEncodedRelativeModulePath(for: "SwiftShims", with: pcmArgsCurrent)):
             try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgsCurrent, moduleId: .clang("SwiftShims"),
                                             moduleDependencyGraph: dependencyGraph)
-          case .temporary(RelativePath("main.o")):
+          case .temporary(RelativePath("testExplicitModuleBuildJobs.o")):
             XCTAssertTrue(driver.isExplicitMainModuleJob(job: job))
             guard case .swift(let mainModuleSwiftDetails) = dependencyGraph.mainModule.details else {
               XCTFail("Main module does not have Swift details field")
@@ -338,11 +339,11 @@ final class ExplicitModuleBuildTests: XCTestCase {
             try checkExplicitModuleBuildJobDependencies(job: job, pcmArgs: pcmArgs,
                                                         moduleInfo: dependencyGraph.mainModule,
                                                         moduleDependencyGraph: dependencyGraph)
-          case .relative(RelativePath("main")):
+          case .relative(RelativePath("testExplicitModuleBuildJobs")):
             XCTAssertTrue(driver.isExplicitMainModuleJob(job: job))
             XCTAssertEqual(job.kind, .link)
 
-          case .temporary(RelativePath("main.autolink")):
+          case .temporary(RelativePath("testExplicitModuleBuildJobs.autolink")):
             XCTAssertTrue(driver.isExplicitMainModuleJob(job: job))
             XCTAssertEqual(job.kind, .autolinkExtract)
 
@@ -359,7 +360,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
     #if os(macOS)
     try withTemporaryDirectory { path in
       try localFileSystem.changeCurrentWorkingDirectory(to: path)
-      let main = path.appending(component: "main.swift")
+      let main = path.appending(component: "testExplicitModuleBuildEndToEnd.swift")
       try localFileSystem.writeFileContents(main) {
         $0 <<< "import C;"
         $0 <<< "import E;"
