Merge pull request #181 from artemcm/ExplicitPackageBuilds

[Explicit Module Builds] Support explicit module builds with external dependencies.

Author: artemcm

Sources/SwiftDriver/CMakeLists.txt

@@ -8,7 +8,8 @@
 
 add_library(SwiftDriver
   "Explicit Module Builds/ClangModuleBuildJobCache.swift"
-  "Explicit Module Builds/ExplicitModuleBuildHandler.swift"   
+  "Explicit Module Builds/ExplicitModuleBuildHandler.swift"
+  "Explicit Module Builds/PlaceholderDependencyResolution.swift"
   "Explicit Module Builds/InterModuleDependencyGraph.swift"
   "Explicit Module Builds/ModuleDependencyScanning.swift"
   "Explicit Module Builds/ModuleArtifacts.swift"

Sources/SwiftDriver/Driver/Driver.swift

@@ -30,6 +30,7 @@ public struct Driver {
     case missingPCMArguments(String)
     case missingModuleDependency(String)
     case dependencyScanningFailure(Int, String)
+    case missingExternalDependency(String)
 
     public var description: String {
       switch self {
@@ -59,6 +60,8 @@ public struct Driver {
         return "Module Dependency Scanner returned with non-zero exit status: \(code), \(error)"
       case .unableToLoadOutputFileMap(let path):
         return "unable to load output file map '\(path)': no such file or directory"
+      case .missingExternalDependency(let moduleName):
+        return "Missing External dependency info for module: \(moduleName)"
       }
     }
   }
@@ -193,6 +196,11 @@ public struct Driver {
   /// as explicit by the various compilation jobs.
   @_spi(Testing) public var explicitModuleBuildHandler: ExplicitModuleBuildHandler? = nil
 
+  /// A collection describing external dependencies for the current main module that may be invisible to
+  /// the driver itself, but visible to its clients (e.g. build systems like SwiftPM). Along with the external dependencies'
+  /// module dependency graphs.
+  internal var externalDependencyArtifactMap: ExternalDependencyArtifactMap? = nil
+
   /// Handler for emitting diagnostics to stderr.
   public static let stderrDiagnosticsHandler: DiagnosticsEngine.DiagnosticsHandler = { diagnostic in
     let stream = stderrStream
@@ -226,24 +234,30 @@ public struct Driver {
   /// - Parameter diagnosticsHandler: A callback executed when a diagnostic is
   ///   emitted. The default argument prints diagnostics to stderr.
   /// - Parameter executor: Used by the driver to execute jobs. The default argument
-  /// is present to streamline testing, it shouldn't be used in production.
+  ///   is present to streamline testing, it shouldn't be used in production.
+  /// - Parameter externalModuleDependencies: A collection of external modules that the main module
+  ///   of the current compilation depends on. Explicit Module Build use only.
   public init(
     args: [String],
     env: [String: String] = ProcessEnv.vars,
     diagnosticsEngine: DiagnosticsEngine = DiagnosticsEngine(handlers: [Driver.stderrDiagnosticsHandler]),
     fileSystem: FileSystem = localFileSystem,
-    executor: DriverExecutor
+    executor: DriverExecutor,
+    externalModuleDependencies: ExternalDependencyArtifactMap? = nil
   ) throws {
     self.env = env
     self.fileSystem = fileSystem
 
     self.diagnosticEngine = diagnosticsEngine
     self.executor = executor
 
+    self.externalDependencyArtifactMap = externalModuleDependencies
+
     if case .subcommand = try Self.invocationRunMode(forArgs: args).mode {
       throw Error.subcommandPassedToDriver
     }
 
+
     var args = try Self.expandResponseFiles(args, fileSystem: fileSystem, diagnosticsEngine: self.diagnosticEngine)
 
     self.driverKind = try Self.determineDriverKind(args: &args)

Sources/SwiftDriver/Explicit Module Builds/ExplicitModuleBuildHandler.swift

@@ -13,6 +13,10 @@ import TSCBasic
 import TSCUtility
 import Foundation
 
+/// A map from a module identifier to a pair consisting of a path to its .swiftmodule file and its module dependency graph.
+public typealias ExternalDependencyArtifactMap =
+    [ModuleDependencyId: (AbsolutePath, InterModuleDependencyGraph)]
+
 /// In Explicit Module Build mode, this handler is responsible for generating and providing
 /// build jobs for all module dependencies and providing compile command options
 /// that specify said explicit module dependencies.
@@ -30,6 +34,9 @@ import Foundation
   /// The toolchain to be used for frontend job generation.
   private let toolchain: Toolchain
 
+  /// A collection of external dependency modules, and their binary module file paths and dependency graph.
+  internal let externalDependencyArtifactMap: ExternalDependencyArtifactMap
+
   /// The file system which we should interact with.
   /// FIXME: Our end goal is to not have any direct filesystem manipulation in here, but  that's dependent on getting the
   /// dependency scanner/dependency job generation  moved into a Job.
@@ -42,9 +49,11 @@ import Foundation
   private let temporaryDirectory: AbsolutePath
 
   public init(dependencyGraph: InterModuleDependencyGraph, toolchain: Toolchain,
-              fileSystem: FileSystem) throws {
+              fileSystem: FileSystem,
+              externalDependencyArtifactMap: ExternalDependencyArtifactMap) throws {
     self.dependencyGraph = dependencyGraph
     self.toolchain = toolchain
+    self.externalDependencyArtifactMap = externalDependencyArtifactMap
     self.fileSystem = fileSystem
     self.temporaryDirectory = try determineTempDirectory()
   }
@@ -110,10 +119,17 @@ import Foundation
   /// - Generate Job: S1
   ///
   mutating public func generateExplicitModuleDependenciesBuildJobs() throws -> [Job] {
+    // Resolve placeholder dependencies in the dependency graph, if any.
+    if (!externalDependencyArtifactMap.isEmpty) {
+      try resolvePlaceholderDependencies()
+    }
+
+    // Compute jobs for all main module dependencies
     var mainModuleInputs: [TypedVirtualPath] = []
     var mainModuleCommandLine: [Job.ArgTemplate] = []
     try resolveMainModuleDependencies(inputs: &mainModuleInputs,
                                       commandLine: &mainModuleCommandLine)
+    
     return Array(swiftModuleBuildCache.values) + clangTargetModuleBuildCache.allJobs
   }
 
@@ -297,7 +313,7 @@ import Foundation
                                               clangDependencyArtifacts: inout [ClangModuleArtifactInfo],
                                               swiftDependencyArtifacts: inout [SwiftModuleArtifactInfo]
   ) throws {
-    for dependencyId in try dependencyGraph.moduleInfo(of: moduleId).directDependencies {
+    for dependencyId in try dependencyGraph.moduleInfo(of: moduleId).directDependencies! {
       guard addedDependenciesSet.insert(dependencyId).inserted else {
         continue
       }
@@ -314,6 +330,8 @@ import Foundation
                                        addedDependenciesSet: &addedDependenciesSet,
                                        clangDependencyArtifacts: &clangDependencyArtifacts,
                                        swiftDependencyArtifacts: &swiftDependencyArtifacts)
+        case .swiftPlaceholder:
+          fatalError("Unresolved placeholder dependencies at planning stage: \(dependencyId) of \(moduleId)")
       }
     }
   }
@@ -334,7 +352,7 @@ import Foundation
 
     let swiftModulePath: TypedVirtualPath
     if case .swift(let details) = dependencyInfo.details,
-       let compiledModulePath = details.compiledModulePath {
+       let compiledModulePath = details.explicitCompiledModulePath {
       // If an already-compiled module is available, use it.
       swiftModulePath = .init(file: try VirtualPath(path: compiledModulePath),
                               type: .swiftModule)
@@ -457,4 +475,10 @@ private extension InterModuleDependencyGraph {
   }
 }
 
-
+// To keep the ExplicitModuleBuildHandler an implementation detail, provide an API
+// to access the dependency graph
+extension Driver {
+  public var interModuleDependencyGraph: InterModuleDependencyGraph? {
+    return explicitModuleBuildHandler?.dependencyGraph
+  }
+}

Sources/SwiftDriver/Explicit Module Builds/InterModuleDependencyGraph.swift

@@ -12,21 +12,24 @@
 import Foundation
 
 
-@_spi(Testing) public enum ModuleDependencyId: Hashable {
+public enum ModuleDependencyId: Hashable {
   case swift(String)
+  case swiftPlaceholder(String)
   case clang(String)
 
   public var moduleName: String {
     switch self {
-      case .swift(let name): return name
-      case .clang(let name): return name
+    case .swift(let name): return name
+    case .swiftPlaceholder(let name): return name
+    case .clang(let name): return name
     }
   }
 }
 
 extension ModuleDependencyId: Codable {
   enum CodingKeys: CodingKey {
     case swift
+    case swiftPlaceholder
     case clang
   }
 
@@ -36,8 +39,13 @@ extension ModuleDependencyId: Codable {
       let moduleName =  try container.decode(String.self, forKey: .swift)
       self = .swift(moduleName)
     } catch {
-      let moduleName =  try container.decode(String.self, forKey: .clang)
-      self = .clang(moduleName)
+      do {
+        let moduleName =  try container.decode(String.self, forKey: .swiftPlaceholder)
+        self = .swiftPlaceholder(moduleName)
+      } catch {
+        let moduleName =  try container.decode(String.self, forKey: .clang)
+        self = .clang(moduleName)
+      }
     }
   }
 
@@ -46,59 +54,81 @@ extension ModuleDependencyId: Codable {
     switch self {
       case .swift(let moduleName):
         try container.encode(moduleName, forKey: .swift)
+      case .swiftPlaceholder(let moduleName):
+        try container.encode(moduleName, forKey: .swift)
       case .clang(let moduleName):
         try container.encode(moduleName, forKey: .clang)
     }
   }
 }
 
+/// Bridging header
+public struct BridgingHeader: Codable {
+  var path: String
+  var sourceFiles: [String]
+  var moduleDependencies: [String]
+}
+
 /// Details specific to Swift modules.
-@_spi(Testing) public struct SwiftModuleDetails: Codable {
+public struct SwiftModuleDetails: Codable {
   /// The module interface from which this module was built, if any.
-  public var moduleInterfacePath: String?
+  @_spi(Testing) public var moduleInterfacePath: String?
 
   /// The paths of potentially ready-to-use compiled modules for the interface.
-  public var compiledModuleCandidates: [String]?
+  @_spi(Testing) public var compiledModuleCandidates: [String]?
 
-  /// The path to the already-compiled module.
-  public var compiledModulePath: String?
+  /// The path to the already-compiled module that must be used instead of
+  /// generating a job to build this module. In standard compilation, the dependency scanner
+  /// may discover compiled module candidates to be used instead of re-compiling from interface.
+  /// In contrast, this explicitCompiledModulePath is only to be used for precompiled modules
+  /// external dependencies in Explicit Module Build mode
+  @_spi(Testing) public var explicitCompiledModulePath: String?
 
   /// The bridging header, if any.
-  public var bridgingHeaderPath: String?
+  var bridgingHeaderPath: String?
 
   /// The source files referenced by the bridging header.
-  public var bridgingSourceFiles: [String]? = []
+  var bridgingSourceFiles: [String]? = []
 
   /// Options to the compile command
-  public var commandLine: [String]? = []
+  var commandLine: [String]? = []
 
   /// To build a PCM to be used by this Swift module, we need to append these
   /// arguments to the generic PCM build arguments reported from the dependency
   /// graph.
-  public var extraPcmArgs: [String]? = []
+  @_spi(Testing) public var extraPcmArgs: [String]?
+}
+
+/// Details specific to Swift external modules.
+public struct swiftPlaceholderModuleDetails: Codable {
+  /// The path to the .swiftModuleDoc file.
+  var moduleDocPath: String?
+
+  /// The path to the .swiftSourceInfo file.
+  var moduleSourceInfoPath: String?
 }
 
 /// Details specific to Clang modules.
-@_spi(Testing) public struct ClangModuleDetails: Codable {
+public struct ClangModuleDetails: Codable {
   /// The path to the module map used to build this module.
-  public var moduleMapPath: String
+  @_spi(Testing) public var moduleMapPath: String
 
   /// clang-generated context hash
-  public var contextHash: String?
+  var contextHash: String?
 
   /// Options to the compile command
-  public var commandLine: [String]? = []
+  var commandLine: [String]? = []
 }
 
-@_spi(Testing) public struct ModuleInfo: Codable {
+public struct ModuleInfo: Codable {
   /// The path for the module.
   public var modulePath: String
 
   /// The source files used to build this module.
-  public var sourceFiles: [String] = []
+  public var sourceFiles: [String]? = []
 
   /// The set of direct module dependencies of this module.
-  public var directDependencies: [ModuleDependencyId] = []
+  public var directDependencies: [ModuleDependencyId]? = []
 
   /// Specific details of a particular kind of module.
   public var details: Details
@@ -109,6 +139,10 @@ extension ModuleDependencyId: Codable {
     /// a bridging header.
     case swift(SwiftModuleDetails)
 
+    /// Swift external modules carry additional details that specify their
+    /// module doc path and source info paths.
+    case swiftPlaceholder(swiftPlaceholderModuleDetails)
+
     /// Clang modules are built from a module map file.
     case clang(ClangModuleDetails)
   }
@@ -117,6 +151,7 @@ extension ModuleDependencyId: Codable {
 extension ModuleInfo.Details: Codable {
   enum CodingKeys: CodingKey {
     case swift
+    case swiftPlaceholder
     case clang
   }
 
@@ -126,8 +161,13 @@ extension ModuleInfo.Details: Codable {
       let details = try container.decode(SwiftModuleDetails.self, forKey: .swift)
       self = .swift(details)
     } catch {
-      let details = try container.decode(ClangModuleDetails.self, forKey: .clang)
-      self = .clang(details)
+      do {
+        let details = try container.decode(swiftPlaceholderModuleDetails.self, forKey: .swiftPlaceholder)
+        self = .swiftPlaceholder(details)
+      } catch {
+        let details = try container.decode(ClangModuleDetails.self, forKey: .clang)
+        self = .clang(details)
+      }
     }
   }
 
@@ -136,6 +176,8 @@ extension ModuleInfo.Details: Codable {
     switch self {
       case .swift(let details):
         try container.encode(details, forKey: .swift)
+      case .swiftPlaceholder(let details):
+        try container.encode(details, forKey: .swiftPlaceholder)
       case .clang(let details):
         try container.encode(details, forKey: .clang)
     }
@@ -144,7 +186,7 @@ extension ModuleInfo.Details: Codable {
 
 /// Describes the complete set of dependencies for a Swift module, including
 /// all of the Swift and C modules and source files it depends on.
-@_spi(Testing) public struct InterModuleDependencyGraph: Codable {
+public struct InterModuleDependencyGraph: Codable {
   /// The name of the main module.
   public var mainModuleName: String
 

Sources/SwiftDriver/Explicit Module Builds/ModuleDependencyScanning.swift

@@ -31,6 +31,15 @@ extension Driver {
                                  moduleDependencyGraphUse: .dependencyScan)
     // FIXME: MSVC runtime flags
 
+    // Pass in external dependencies to be treated as placeholder dependencies by the scanner
+    if let externalDependencyArtifactMap = externalDependencyArtifactMap {
+      let dependencyPlaceholderMapFile =
+        try serializeExternalDependencyArtifacts(externalDependencyArtifactMap:
+                                                  externalDependencyArtifactMap)
+      commandLine.appendFlag("-placeholder-dependency-module-map-file")
+      commandLine.appendPath(dependencyPlaceholderMapFile)
+    }
+
     // Pass on the input files
     commandLine.append(contentsOf: inputFiles.map { .path($0.file)})
 
@@ -44,4 +53,24 @@ extension Driver {
                outputs: [TypedVirtualPath(file: .standardOutput, type: .jsonDependencies)],
                supportsResponseFiles: true)
   }
+
+  /// Serialize a map of placeholder (external) dependencies for the dependency scanner.
+  func serializeExternalDependencyArtifacts(externalDependencyArtifactMap: ExternalDependencyArtifactMap)
+  throws -> AbsolutePath {
+    let temporaryDirectory = try determineTempDirectory()
+    let placeholderMapFilePath =
+      temporaryDirectory.appending(component: "\(moduleOutputInfo.name)-placeholder-modules.json")
+
+    var placeholderArtifacts: [SwiftModuleArtifactInfo] = []
+    for (moduleId, dependencyInfo) in externalDependencyArtifactMap {
+      placeholderArtifacts.append(
+          SwiftModuleArtifactInfo(name: moduleId.moduleName,
+                                  modulePath: dependencyInfo.0.description))
+    }
+    let encoder = JSONEncoder()
+    encoder.outputFormatting = [.prettyPrinted]
+    let contents = try encoder.encode(placeholderArtifacts)
+    try fileSystem.writeFileContents(placeholderMapFilePath, bytes: ByteString(contents))
+    return placeholderMapFilePath
+  }
 }