Skip to content

Commit b38c149

Browse files
artemcmartemcm
committed
[Explicit Module Builds] Re-use external build artifacts to avoid redundant Swift moduel re-scan
In contexts where SwiftDriver is used to plan a multi-module build (e.g. SwiftPM), use the `ModuleInfoMap` containing results of prior scans to avoid further redundant re-scanning. *All* previously-seen Swift modules are treated as placeholder dependencies during subsequent dependency-scanning actions. This causes the dependency scanner, upon encountering an `import` of such module, to not search for its interface, parse it, and build a full dependency tree yet again. The dependency scanner output, on subsequent re-scans is therefore a much shallower graph with many placeholders, which get resolved in the client (SwiftDriver) using the external `ModuleInfoMap` provided by the build system. This PR adds: - Serialize *all* previously-seen Swift modules, not just ones corresponding to build targets, as placeholder inputs to the dependency-scanning action. - Adds resolve logic for additional, non-target Swift modules found in the initial inter-module dependency graph. - Renames ExplicitModuleBuildHandler to ExplicitDependencyBuildPlanner.
1 parent 59b6357 commit b38c149

File tree

10 files changed

+93
-92
lines changed

10 files changed

+93
-92
lines changed

‎Sources/SwiftDriver/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ add_library(SwiftDriver
1010
"Explicit Module Builds/ClangModuleBuildJobCache.swift"
1111
"Explicit Module Builds/ClangVersionedDependencyResolution.swift"
1212
"Explicit Module Builds/CommonDependencyGraphOperations.swift"
13-
"Explicit Module Builds/ExplicitModuleBuildHandler.swift"
13+
"Explicit Module Builds/ExplicitDependencyBuildPlanner.swift"
1414
"Explicit Module Builds/InterModuleDependencyGraph.swift"
1515
"Explicit Module Builds/ModuleDependencyScanning.swift"
1616
"Explicit Module Builds/PlaceholderDependencyResolution.swift"

‎Sources/SwiftDriver/Driver/Driver.swift‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -252,10 +252,10 @@ public struct Driver {
252252
/// dependencies in parallel builds.
253253
var forceEmitModuleBeforeCompile: Bool = false
254254

255-
/// Handler for constructing module build jobs using Explicit Module Builds.
255+
/// Planner for constructing module build jobs using Explicit Module Builds.
256256
/// Constructed during the planning phase only when all modules will be prebuilt and treated
257257
/// as explicit by the various compilation jobs.
258-
@_spi(Testing) public var explicitModuleBuildHandler: ExplicitModuleBuildHandler? = nil
258+
@_spi(Testing) public var explicitDependencyBuildPlanner: ExplicitDependencyBuildPlanner? = nil
259259

260260
/// All external artifacts a build system (e.g. SwiftPM) may pass in as input to the explicit
261261
/// build of the current module. Consists of a map of externally-built targets, and a map of all previously

‎Sources/SwiftDriver/Explicit Module Builds/CommonDependencyGraphOperations.swift‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,8 +110,8 @@ internal extension InterModuleDependencyGraph {
110110
}
111111
if let originalModuleIndex = moduleInfo.directDependencies?.firstIndex(of: originalId) {
112112
moduleInfo.directDependencies![originalModuleIndex] = replacementId;
113+
moduleInfoMap[moduleId] = moduleInfo
113114
}
114-
moduleInfoMap[moduleId] = moduleInfo
115115
}
116116
}
117117
}

Sources/SwiftDriver/Explicit Module Builds/ExplicitModuleBuildHandler.swift renamed to Sources/SwiftDriver/Explicit Module Builds/ExplicitDependencyBuildPlanner.swift

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
//===--------------- ExplicitModuleBuildHandler.swift ---------------------===//
1+
//===--------------- ExplicitDependencyBuildPlanner.swift ---------------------===//
22
//
33
// This source file is part of the Swift.org open source project
44
//
@@ -13,16 +13,16 @@ import TSCBasic
1313
import TSCUtility
1414
import Foundation
1515

16-
/// A map from a module identifier to a pair consisting of a path to its .swiftmodule file.
16+
/// A map from a module identifier to a path to its .swiftmodule file.
1717
public typealias ExternalTargetModulePathMap = [ModuleDependencyId: AbsolutePath]
1818
/// A tuple all external artifacts a build system may pass in as input to the explicit build of the current module
1919
/// Consists of a map of externally-built targets, and a map of all previously discovered/scanned modules.
2020
public typealias ExternalBuildArtifacts = (ExternalTargetModulePathMap, ModuleInfoMap)
2121

22-
/// In Explicit Module Build mode, this handler is responsible for generating and providing
22+
/// In Explicit Module Build mode, this planner is responsible for generating and providing
2323
/// build jobs for all module dependencies and providing compile command options
2424
/// that specify said explicit module dependencies.
25-
@_spi(Testing) public struct ExplicitModuleBuildHandler {
25+
@_spi(Testing) public struct ExplicitDependencyBuildPlanner {
2626
/// The module dependency graph.
2727
public var dependencyGraph: InterModuleDependencyGraph
2828

@@ -201,7 +201,7 @@ public typealias ExternalBuildArtifacts = (ExternalTargetModulePathMap, ModuleIn
201201

202202
// Encode the target triple pcm args into the output `.pcm` filename
203203
let targetEncodedModulePath =
204-
try ExplicitModuleBuildHandler.targetEncodedClangModuleFilePath(for: moduleInfo,
204+
try ExplicitDependencyBuildPlanner.targetEncodedClangModuleFilePath(for: moduleInfo,
205205
pcmArgs: pcmArgs)
206206
outputs.append(TypedVirtualPath(file: targetEncodedModulePath, type: .pcm))
207207
commandLine.appendFlags("-emit-pcm", "-module-name", moduleId.moduleName,
@@ -384,7 +384,7 @@ public typealias ExternalBuildArtifacts = (ExternalTargetModulePathMap, ModuleIn
384384
let dependencyInfo = try dependencyGraph.moduleInfo(of: dependencyId)
385385
let dependencyClangModuleDetails = try dependencyGraph.clangModuleDetails(of: dependencyId)
386386
let clangModulePath =
387-
try ExplicitModuleBuildHandler.targetEncodedClangModuleFilePath(for: dependencyInfo,
387+
try ExplicitDependencyBuildPlanner.targetEncodedClangModuleFilePath(for: dependencyInfo,
388388
pcmArgs: pcmArgs)
389389

390390
// Collect the requried information about this module
@@ -430,7 +430,7 @@ public typealias ExternalBuildArtifacts = (ExternalTargetModulePathMap, ModuleIn
430430
}
431431

432432
/// Utility methods for encoding PCM's target triple into its name.
433-
extension ExplicitModuleBuildHandler {
433+
extension ExplicitDependencyBuildPlanner {
434434
/// Compute a full path to the resulting .pcm file for a given Clang module, with the
435435
/// target triple encoded in the name.
436436
public static func targetEncodedClangModuleFilePath(for moduleInfo: ModuleInfo,
@@ -455,7 +455,7 @@ extension ExplicitModuleBuildHandler {
455455
}
456456
}
457457

458-
/// Encapsulates some of the common queries of the ExplicitModuleBuildeHandler with error-checking
458+
/// Encapsulates some of the common queries of the ExplicitDependencyBuildPlanner with error-checking
459459
/// on the dependency graph's structure.
460460
internal extension InterModuleDependencyGraph {
461461
func moduleInfo(of moduleId: ModuleDependencyId) throws -> ModuleInfo {
@@ -496,7 +496,7 @@ internal extension InterModuleDependencyGraph {
496496
}
497497

498498
// InterModuleDependencyGraph printing, useful for debugging
499-
private extension InterModuleDependencyGraph {
499+
internal extension InterModuleDependencyGraph {
500500
func prettyPrintString() throws -> String {
501501
let encoder = JSONEncoder()
502502
encoder.outputFormatting = [.prettyPrinted]
@@ -505,10 +505,10 @@ private extension InterModuleDependencyGraph {
505505
}
506506
}
507507

508-
// To keep the ExplicitModuleBuildHandler an implementation detail, provide an API
508+
// To keep the ExplicitDependencyBuildPlanner an implementation detail, provide an API
509509
// to access the dependency graph
510510
extension Driver {
511511
public var interModuleDependencyGraph: InterModuleDependencyGraph? {
512-
return explicitModuleBuildHandler?.dependencyGraph
512+
return explicitDependencyBuildPlanner?.dependencyGraph
513513
}
514514
}

‎Sources/SwiftDriver/Explicit Module Builds/ModuleDependencyScanning.swift‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,7 @@ extension Driver {
3232
// Pass in external target dependencies to be treated as placeholder dependencies by the scanner
3333
if let externalBuildArtifacts = externalBuildArtifacts {
3434
let dependencyPlaceholderMapFile =
35-
try serializeExternalDependencyArtifacts(externalTargetModulePathMap:
36-
externalBuildArtifacts.0)
35+
try serializeExternalDependencyArtifacts(externalBuildArtifacts: externalBuildArtifacts)
3736
commandLine.appendFlag("-placeholder-dependency-module-map-file")
3837
commandLine.appendPath(dependencyPlaceholderMapFile)
3938
}
@@ -54,18 +53,31 @@ extension Driver {
5453
}
5554

5655
/// Serialize a map of placeholder (external) dependencies for the dependency scanner.
57-
func serializeExternalDependencyArtifacts(externalTargetModulePathMap: ExternalTargetModulePathMap)
56+
func serializeExternalDependencyArtifacts(externalBuildArtifacts: ExternalBuildArtifacts)
5857
throws -> VirtualPath {
58+
let (externalTargetModulePathMap, externalModuleInfoMap) = externalBuildArtifacts
5959
var placeholderArtifacts: [SwiftModuleArtifactInfo] = []
60+
61+
// Explicit external targets
6062
for (moduleId, binaryModulePath) in externalTargetModulePathMap {
6163
placeholderArtifacts.append(
6264
SwiftModuleArtifactInfo(name: moduleId.moduleName,
6365
modulePath: binaryModulePath.description))
6466
}
67+
68+
// All other already-scanned Swift modules
69+
for (moduleId, moduleInfo) in externalModuleInfoMap
70+
where !externalTargetModulePathMap.keys.contains(moduleId) {
71+
guard case .swift(_) = moduleId else { continue }
72+
placeholderArtifacts.append(
73+
SwiftModuleArtifactInfo(name: moduleId.moduleName,
74+
modulePath: moduleInfo.modulePath))
75+
}
6576
let encoder = JSONEncoder()
6677
encoder.outputFormatting = [.prettyPrinted]
6778
let contents = try encoder.encode(placeholderArtifacts)
68-
return .temporaryWithKnownContents(.init("\(moduleOutputInfo.name)-placeholder-modules.json"), contents)
79+
return .temporaryWithKnownContents(.init("\(moduleOutputInfo.name)-placeholder-modules.json"),
80+
contents)
6981
}
7082

7183
mutating func performBatchDependencyScan(moduleInfos: [BatchScanModuleInfo])
@@ -163,6 +175,7 @@ extension Driver {
163175
let encoder = JSONEncoder()
164176
encoder.outputFormatting = [.prettyPrinted]
165177
let contents = try encoder.encode(moduleInfos)
166-
return .temporaryWithKnownContents(.init("\(moduleOutputInfo.name)-batch-module-scan.json"), contents)
178+
return .temporaryWithKnownContents(.init("\(moduleOutputInfo.name)-batch-module-scan.json"),
179+
contents)
167180
}
168181
}

‎Sources/SwiftDriver/Explicit Module Builds/PlaceholderDependencyResolution.swift‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,24 +45,46 @@ import Foundation
4545
throws {
4646
let externalTargetModulePathMap = externalBuildArtifacts.0
4747
let externalModuleInfoMap = externalBuildArtifacts.1
48-
let placeholderModules = modules.keys.filter {
48+
let placeholderFilter : (ModuleDependencyId) -> Bool = {
4949
if case .swiftPlaceholder(_) = $0 {
5050
return true
5151
}
5252
return false
5353
}
54+
var placeholderModules = modules.keys.filter(placeholderFilter)
5455

5556
// Resolve all target placeholder modules
5657
let placeholderTargetModules = placeholderModules.filter { externalTargetModulePathMap[$0] != nil }
5758
for moduleId in placeholderTargetModules {
5859
guard let placeholderModulePath = externalTargetModulePathMap[moduleId] else {
5960
throw Driver.Error.missingExternalDependency(moduleId.moduleName)
6061
}
61-
6262
try resolveTargetPlaceholder(placeholderId: moduleId,
6363
placeholderPath: placeholderModulePath,
6464
externalModuleInfoMap: externalModuleInfoMap)
6565
}
66+
67+
// Process remaining placeholders until there are none left
68+
placeholderModules = modules.keys.filter(placeholderFilter)
69+
while !placeholderModules.isEmpty {
70+
let moduleId = placeholderModules.first!
71+
let swiftModuleId = ModuleDependencyId.swift(moduleId.moduleName)
72+
73+
guard externalModuleInfoMap[swiftModuleId] != nil else {
74+
throw Driver.Error.missingExternalDependency(moduleId.moduleName)
75+
}
76+
let moduleInfo = externalModuleInfoMap[swiftModuleId]!
77+
78+
// Insert the resolved module, replacing the placeholder.
79+
try Self.mergeModule(swiftModuleId, moduleInfo, into: &modules)
80+
81+
// Traverse and add all of this external module's dependencies to the current graph.
82+
try resolvePlaceholderModuleDependencies(moduleId: swiftModuleId,
83+
externalModuleInfoMap: externalModuleInfoMap)
84+
85+
// Update the set of remaining placeholders to resolve
86+
placeholderModules = modules.keys.filter(placeholderFilter)
87+
}
6688
}
6789
}
6890

‎Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -424,19 +424,19 @@ extension Driver {
424424
/// to inputs and command line arguments of a compile job.
425425
func addExplicitModuleBuildArguments(inputs: inout [TypedVirtualPath],
426426
commandLine: inout [Job.ArgTemplate]) throws {
427-
guard var handler = explicitModuleBuildHandler else {
428-
fatalError("No handler in Explicit Module Build mode.")
427+
guard var dependencyPlanner = explicitDependencyBuildPlanner else {
428+
fatalError("No dependency planner in Explicit Module Build mode.")
429429
}
430-
try handler.resolveMainModuleDependencies(inputs: &inputs, commandLine: &commandLine)
430+
try dependencyPlanner.resolveMainModuleDependencies(inputs: &inputs, commandLine: &commandLine)
431431
}
432432

433433
/// In Explicit Module Build mode, distinguish between main module jobs and intermediate dependency build jobs,
434434
/// such as Swift modules built from .swiftmodule files and Clang PCMs.
435435
public func isExplicitMainModuleJob(job: Job) -> Bool {
436-
guard let handler = explicitModuleBuildHandler else {
437-
fatalError("No handler in Explicit Module Build mode.")
436+
guard let dependencyPlanner = explicitDependencyBuildPlanner else {
437+
fatalError("No dependency planner in Explicit Module Build mode.")
438438
}
439-
return job.moduleName == handler.dependencyGraph.mainModuleName
439+
return job.moduleName == dependencyPlanner.dependencyGraph.mainModuleName
440440
}
441441
}
442442

‎Sources/SwiftDriver/Jobs/Planning.swift‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -373,10 +373,10 @@ extension Driver {
373373
/// by re-scanning all Clang modules against all possible targets they will be built against.
374374
public mutating func generateExplicitModuleDependenciesJobs() throws -> [Job] {
375375
let dependencyGraph = try generateInterModuleDependencyGraph()
376-
explicitModuleBuildHandler =
377-
try ExplicitModuleBuildHandler(dependencyGraph: dependencyGraph,
376+
explicitDependencyBuildPlanner =
377+
try ExplicitDependencyBuildPlanner(dependencyGraph: dependencyGraph,
378378
toolchain: toolchain)
379-
return try explicitModuleBuildHandler!.generateExplicitModuleDependenciesBuildJobs()
379+
return try explicitDependencyBuildPlanner!.generateExplicitModuleDependenciesBuildJobs()
380380
}
381381

382382
private mutating func generateInterModuleDependencyGraph() throws -> InterModuleDependencyGraph {
@@ -390,7 +390,7 @@ extension Driver {
390390
recordedInputModificationDates: recordedInputModificationDates)
391391

392392
// Resolve placeholder dependencies in the dependency graph, if any.
393-
if externalBuildArtifacts != nil, !externalBuildArtifacts!.0.isEmpty {
393+
if externalBuildArtifacts != nil {
394394
try dependencyGraph.resolvePlaceholderDependencies(using: externalBuildArtifacts!)
395395
}
396396

@@ -491,7 +491,6 @@ extension Driver {
491491

492492
/// Plan a build by producing a set of jobs to complete the build.
493493
public mutating func planBuild() throws -> [Job] {
494-
495494
if let job = try immediateForwardingJob() {
496495
return [job]
497496
}

‎Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ private func checkExplicitModuleBuildJobDependencies(job: Job,
116116
XCTAssertEqual(dependencyArtifacts!.modulePath, prebuiltModuleDetails.compiledModulePath)
117117
case .clang(let clangDependencyDetails):
118118
let clangDependencyModulePathString =
119-
try ExplicitModuleBuildHandler.targetEncodedClangModuleFilePath(
119+
try ExplicitDependencyBuildPlanner.targetEncodedClangModuleFilePath(
120120
for: dependencyInfo, pcmArgs: pcmArgs)
121121
let clangDependencyModulePath =
122122
TypedVirtualPath(file: clangDependencyModulePathString, type: .pcm)
@@ -147,7 +147,7 @@ private func checkExplicitModuleBuildJobDependencies(job: Job,
147147
private func pcmArgsEncodedRelativeModulePath(for moduleName: String, with pcmArgs: [String]
148148
) throws -> RelativePath {
149149
return RelativePath(
150-
try ExplicitModuleBuildHandler.targetEncodedClangModuleName(for: moduleName,
150+
try ExplicitDependencyBuildPlanner.targetEncodedClangModuleName(for: moduleName,
151151
pcmArgs: pcmArgs) + ".pcm")
152152
}
153153

@@ -163,10 +163,10 @@ final class ExplicitModuleBuildTests: XCTestCase {
163163
try JSONDecoder().decode(
164164
InterModuleDependencyGraph.self,
165165
from: ModuleDependenciesInputs.fastDependencyScannerOutput.data(using: .utf8)!)
166-
driver.explicitModuleBuildHandler = try ExplicitModuleBuildHandler(dependencyGraph: moduleDependencyGraph,
166+
driver.explicitDependencyBuildPlanner = try ExplicitDependencyBuildPlanner(dependencyGraph: moduleDependencyGraph,
167167
toolchain: driver.toolchain)
168168
let modulePrebuildJobs =
169-
try driver.explicitModuleBuildHandler!.generateExplicitModuleDependenciesBuildJobs()
169+
try driver.explicitDependencyBuildPlanner!.generateExplicitModuleDependenciesBuildJobs()
170170
XCTAssertEqual(modulePrebuildJobs.count, 4)
171171
for job in modulePrebuildJobs {
172172
XCTAssertEqual(job.outputs.count, 1)
@@ -208,7 +208,8 @@ final class ExplicitModuleBuildTests: XCTestCase {
208208
[ModuleDependencyId.swiftPlaceholder("B"):AbsolutePath("/Somewhere/B.swiftmodule")]
209209
let externalModuleInfoMap: ModuleInfoMap = inputDependencyGraph.modules
210210

211-
// Construct a module dependency graph that will contain .swiftPlaceholder("B")
211+
// Construct a module dependency graph that will contain .swiftPlaceholder("B"),
212+
// .swiftPlaceholder("Swift"), .swiftPlaceholder("SwiftOnoneSupport")
212213
var moduleDependencyGraph =
213214
try JSONDecoder().decode(
214215
InterModuleDependencyGraph.self,
@@ -228,10 +229,20 @@ final class ExplicitModuleBuildTests: XCTestCase {
228229
// Plan explicit dependency jobs, after resolving placeholders to actual dependencies.
229230
try moduleDependencyGraph.resolvePlaceholderDependencies(
230231
using: (targetModulePathMap, externalModuleInfoMap))
231-
driver.explicitModuleBuildHandler = try ExplicitModuleBuildHandler(dependencyGraph: moduleDependencyGraph,
232-
toolchain: driver.toolchain)
232+
233+
// Ensure the graph no longer contains any placeholders
234+
XCTAssertFalse(moduleDependencyGraph.modules.keys.contains {
235+
if case .swiftPlaceholder(_) = $0 {
236+
return true
237+
}
238+
return false
239+
})
240+
241+
driver.explicitDependencyBuildPlanner =
242+
try ExplicitDependencyBuildPlanner(dependencyGraph: moduleDependencyGraph,
243+
toolchain: driver.toolchain)
233244
let modulePrebuildJobs =
234-
try driver.explicitModuleBuildHandler!.generateExplicitModuleDependenciesBuildJobs()
245+
try driver.explicitDependencyBuildPlanner!.generateExplicitModuleDependenciesBuildJobs()
235246

236247
// Verify that the dependency graph contains no placeholders.
237248
let placeholdersInTheGraph = driver.interModuleDependencyGraph!.modules.keys
@@ -291,7 +302,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
291302

292303
let jobs = try driver.planBuild()
293304
// Figure out which Triples to use.
294-
let dependencyGraph = driver.explicitModuleBuildHandler!.dependencyGraph
305+
let dependencyGraph = driver.explicitDependencyBuildPlanner!.dependencyGraph
295306
guard case .swift(let mainModuleSwiftDetails) = dependencyGraph.mainModule.details else {
296307
XCTFail("Main module does not have Swift details field")
297308
return

0 commit comments

Comments
 (0)