Skip to content

Commit 673548f

Browse files
artemcmartemcm
committed
[Explicit Module Builds] Re-scan all clang modules against all targets against which they will be built.
- Factor out placeholder dependency resolution out of the ExplicitModuleHandler into the Driver - Generate and post-process the dependency graph before passing it on to the ExplicitModuleHandler: - Run the Swift Fast Dependency Scanner. - Resolve placeholder dependencies, if any. - Re-scan and update Clang modules with a super-set of their dependencies against all compilation targets.
1 parent b590b0c commit 673548f

21 files changed

+383
-94
lines changed

‎Sources/SwiftDriver/Driver/Driver.swift‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@ public struct Driver {
205205
/// A collection describing external dependencies for the current main module that may be invisible to
206206
/// the driver itself, but visible to its clients (e.g. build systems like SwiftPM). Along with the external dependencies'
207207
/// module dependency graphs.
208-
internal var externalDependencyArtifactMap: ExternalDependencyArtifactMap? = nil
208+
@_spi(Testing) public var externalDependencyArtifactMap: ExternalDependencyArtifactMap? = nil
209209

210210
/// Handler for emitting diagnostics to stderr.
211211
public static let stderrDiagnosticsHandler: DiagnosticsEngine.DiagnosticsHandler = { diagnostic in
@@ -1005,6 +1005,9 @@ extension Driver {
10051005
case .scanDependencies:
10061006
compilerOutputType = .jsonDependencies
10071007

1008+
case .scanClangDependencies:
1009+
compilerOutputType = .jsonClangDependencies
1010+
10081011
default:
10091012
fatalError("unhandled output mode option \(outputOption)")
10101013
}

‎Sources/SwiftDriver/Explicit Module Builds/ClangVersionedDependencyResolution.swift‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
//===-------------- ClangVersionedDependencyResolution.swift --------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2020 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
import Foundation
14+
15+
/// A map from a module identifier to a set of module dependency graphs
16+
/// Used to compute distinct graphs corresponding to different target versions for a given clang module
17+
public typealias ModuleVersionedGraphMap = [ModuleDependencyId: [InterModuleDependencyGraph]]
18+
19+
internal extension Driver {
20+
// Dependency scanning results may vary depending on the target version specified on the
21+
// dependency scanning action. If the scanning action is performed at a fixed target, and the
22+
// scanned module is later compiled with a higher version target, miscomputation may occur
23+
// due to dependencies present only at the higher version number and thus not detected by
24+
// the dependency scanner. We must ensure to re-scan Clang modules at all targets at which
25+
// they will be compiled and record a super-set of the module's dependencies at all targets.
26+
/// For each clang module, compute its dependencies at all targets at which it will be compiled.
27+
mutating func resolveVersionedClangDependencies(dependencyGraph: inout InterModuleDependencyGraph)
28+
throws {
29+
// Traverse the dependency graph, collecting extraPCMArgs along each path
30+
// to all Clang modules, and compute a set of distinct PCMArgs across all paths to a
31+
// given Clang module in the graph.
32+
let modulePCMArgsSetMap = try dependencyGraph.computePCMArgSetsForClangModules()
33+
var moduleVersionedGraphMap: [ModuleDependencyId: [InterModuleDependencyGraph]] = [:]
34+
for (moduleId, pcmArgSet) in modulePCMArgsSetMap {
35+
for pcmArgs in pcmArgSet {
36+
let pcmSpecificDepGraph = try scanClangModule(moduleId: moduleId,
37+
pcmArgs: pcmArgs)
38+
if moduleVersionedGraphMap[moduleId] != nil {
39+
moduleVersionedGraphMap[moduleId]!.append(pcmSpecificDepGraph)
40+
} else {
41+
moduleVersionedGraphMap[moduleId] = [pcmSpecificDepGraph]
42+
}
43+
}
44+
}
45+
46+
try dependencyGraph.resolveVersionedClangModules(using: moduleVersionedGraphMap)
47+
}
48+
}
49+
50+
private extension InterModuleDependencyGraph {
51+
mutating func resolveVersionedClangModules(using versionedGraphMap: ModuleVersionedGraphMap)
52+
throws {
53+
for (moduleId, graphList) in versionedGraphMap {
54+
var currentModuleInfo = modules[moduleId]!
55+
for versionedGraph in graphList {
56+
guard let versionedModuleInfo = versionedGraph.modules[moduleId] else {
57+
throw Driver.Error.missingModuleDependency(moduleId.moduleName)
58+
}
59+
60+
versionedModuleInfo.directDependencies?.forEach { dependencyId in
61+
// If a not-seen-before dependency has been found, add it to the info
62+
// and its module to the main graph
63+
if !currentModuleInfo.directDependencies!.contains(dependencyId) {
64+
currentModuleInfo.directDependencies!.append(dependencyId)
65+
// In case this dependency is a module the "main" dependency graph does not contain,
66+
// add it.
67+
if modules[dependencyId] == nil {
68+
modules[dependencyId] = versionedGraph.modules[dependencyId]!
69+
}
70+
}
71+
}
72+
73+
}
74+
// Update the moduleInfo with the one whose dependencies consist of a super-set
75+
// of dependencies across all of the versioned dependency graphs
76+
modules[moduleId] = currentModuleInfo
77+
}
78+
}
79+
80+
/// DFS from the main module to all clang modules, accumulating distinct
81+
/// PCMArgs along all paths to a given Clang module
82+
func computePCMArgSetsForClangModules() throws -> [ModuleDependencyId : Set<[String]>] {
83+
let mainModuleId: ModuleDependencyId = .swift(mainModuleName)
84+
var pcmArgSetMap: [ModuleDependencyId : Set<[String]>] = [:]
85+
86+
func visit(_ moduleId: ModuleDependencyId,
87+
pathPCMArtSet: Set<[String]>,
88+
pcmArgSetMap: inout [ModuleDependencyId : Set<[String]>])
89+
throws {
90+
switch moduleId {
91+
case .swift:
92+
// Add extraPCMArgs of the visited node to the current path set
93+
// and proceed to visit all direct dependencies
94+
let modulePCMArgs = try swiftModulePCMArgs(of: moduleId)
95+
var newPathPCMArgSet = pathPCMArtSet
96+
newPathPCMArgSet.insert(modulePCMArgs)
97+
for dependencyId in try moduleInfo(of: moduleId).directDependencies! {
98+
try visit(dependencyId,
99+
pathPCMArtSet: newPathPCMArgSet,
100+
pcmArgSetMap: &pcmArgSetMap)
101+
}
102+
case .clang:
103+
// Add current path's PCMArgs to the SetMap and stop traversal
104+
if pcmArgSetMap[moduleId] != nil {
105+
pathPCMArtSet.forEach { pcmArgSetMap[moduleId]!.insert($0) }
106+
} else {
107+
pcmArgSetMap[moduleId] = pathPCMArtSet
108+
}
109+
return
110+
case .swiftPlaceholder:
111+
fatalError("Unresolved placeholder dependencies at planning stage: \(moduleId)")
112+
}
113+
}
114+
115+
try visit(mainModuleId,
116+
pathPCMArtSet: [],
117+
pcmArgSetMap: &pcmArgSetMap)
118+
return pcmArgSetMap
119+
}
120+
}

‎Sources/SwiftDriver/Explicit Module Builds/ExplicitModuleBuildHandler.swift‎

Lines changed: 4 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,6 @@ public typealias ExternalDependencyArtifactMap =
3434
/// The toolchain to be used for frontend job generation.
3535
private let toolchain: Toolchain
3636

37-
/// A collection of external dependency modules, and their binary module file paths and dependency graph.
38-
internal let externalDependencyArtifactMap: ExternalDependencyArtifactMap
39-
4037
/// The file system which we should interact with.
4138
/// FIXME: Our end goal is to not have any direct filesystem manipulation in here, but that's dependent on getting the
4239
/// dependency scanner/dependency job generation moved into a Job.
@@ -48,12 +45,11 @@ public typealias ExternalDependencyArtifactMap =
4845
/// dependency scanner/dependency job generation moved into a Job.
4946
private let temporaryDirectory: AbsolutePath
5047

51-
public init(dependencyGraph: InterModuleDependencyGraph, toolchain: Toolchain,
52-
fileSystem: FileSystem,
53-
externalDependencyArtifactMap: ExternalDependencyArtifactMap) throws {
48+
public init(dependencyGraph: InterModuleDependencyGraph,
49+
toolchain: Toolchain,
50+
fileSystem: FileSystem) throws {
5451
self.dependencyGraph = dependencyGraph
5552
self.toolchain = toolchain
56-
self.externalDependencyArtifactMap = externalDependencyArtifactMap
5753
self.fileSystem = fileSystem
5854
self.temporaryDirectory = try determineTempDirectory()
5955
}
@@ -119,11 +115,6 @@ public typealias ExternalDependencyArtifactMap =
119115
/// - Generate Job: S1
120116
///
121117
mutating public func generateExplicitModuleDependenciesBuildJobs() throws -> [Job] {
122-
// Resolve placeholder dependencies in the dependency graph, if any.
123-
if (!externalDependencyArtifactMap.isEmpty) {
124-
try resolvePlaceholderDependencies()
125-
}
126-
127118
// Compute jobs for all main module dependencies
128119
var mainModuleInputs: [TypedVirtualPath] = []
129120
var mainModuleCommandLine: [Job.ArgTemplate] = []
@@ -444,7 +435,7 @@ extension ExplicitModuleBuildHandler {
444435

445436
/// Encapsulates some of the common queries of the ExplicitModuleBuildeHandler with error-checking
446437
/// on the dependency graph's structure.
447-
private extension InterModuleDependencyGraph {
438+
internal extension InterModuleDependencyGraph {
448439
func moduleInfo(of moduleId: ModuleDependencyId) throws -> ModuleInfo {
449440
guard let moduleInfo = modules[moduleId] else {
450441
throw Driver.Error.missingModuleDependency(moduleId.moduleName)

‎Sources/SwiftDriver/Explicit Module Builds/ModuleDependencyScanning.swift‎

Lines changed: 64 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,11 @@
1111
//===----------------------------------------------------------------------===//
1212
import Foundation
1313
import TSCBasic
14+
import SwiftOptions
1415

1516
extension Driver {
1617
/// Precompute the dependencies for a given Swift compilation, producing a
17-
/// complete dependency graph including all Swift and C module files and
18+
/// dependency graph including all Swift and C module files and
1819
/// source files.
1920
mutating func dependencyScanningJob() throws -> Job {
2021
var inputs: [TypedVirtualPath] = []
@@ -73,4 +74,66 @@ extension Driver {
7374
try fileSystem.writeFileContents(placeholderMapFilePath, bytes: ByteString(contents))
7475
return placeholderMapFilePath
7576
}
77+
78+
/// Compute the dependencies for a given Clang module, by invoking the Clang dependency scanning action
79+
/// with the given module's name and a set of arguments (including the target version)
80+
mutating func clangDependencyScanningJob(moduleId: ModuleDependencyId,
81+
pcmArgs: [String]) throws -> Job {
82+
var inputs: [TypedVirtualPath] = []
83+
84+
// Aggregate the fast dependency scanner arguments
85+
var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
86+
commandLine.appendFlag("-frontend")
87+
commandLine.appendFlag("-scan-clang-dependencies")
88+
89+
try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs,
90+
bridgingHeaderHandling: .precompiled,
91+
moduleDependencyGraphUse: .dependencyScan)
92+
93+
// Ensure the `-target` option is inherited from the dependent Swift module's PCM args
94+
if let targetOptionIndex = pcmArgs.firstIndex(of: Option.target.spelling) {
95+
// PCM args are formulated as Clang command line options specified with:
96+
// -Xcc <option> -Xcc <option_value>
97+
assert(pcmArgs.count > targetOptionIndex + 1 && pcmArgs[targetOptionIndex + 1] == "-Xcc")
98+
let pcmArgTriple = Triple(pcmArgs[targetOptionIndex + 2])
99+
// Override the invocation's default target argument by appending the one extracted from
100+
// the pcmArgs
101+
commandLine.appendFlag(.target)
102+
commandLine.appendFlag(pcmArgTriple.triple)
103+
}
104+
105+
// Add the PCM args specific to this scan
106+
pcmArgs.forEach { commandLine.appendFlags($0) }
107+
108+
// This action does not require any input files, but all frontend actions require
109+
// at least one input so pick any input of the current compilation.
110+
let inputFile = inputFiles.first { $0.type == .swift }
111+
commandLine.appendPath(inputFile!.file)
112+
inputs.append(inputFile!)
113+
114+
commandLine.appendFlags("-module-name", moduleId.moduleName)
115+
// Construct the scanning job.
116+
return Job(moduleName: moduleOutputInfo.name,
117+
kind: .scanClangDependencies,
118+
tool: VirtualPath.absolute(try toolchain.getToolPath(.swiftCompiler)),
119+
commandLine: commandLine,
120+
displayInputs: inputs,
121+
inputs: inputs,
122+
outputs: [TypedVirtualPath(file: .standardOutput, type: .jsonDependencies)],
123+
supportsResponseFiles: true)
124+
}
125+
126+
mutating func scanClangModule(moduleId: ModuleDependencyId, pcmArgs: [String])
127+
throws -> InterModuleDependencyGraph {
128+
let clangDependencyScannerJob = try clangDependencyScanningJob(moduleId: moduleId,
129+
pcmArgs: pcmArgs)
130+
let forceResponseFiles = parsedOptions.hasArgument(.driverForceResponseFiles)
131+
132+
let dependencyGraph =
133+
try self.executor.execute(job: clangDependencyScannerJob,
134+
capturingJSONOutputAs: InterModuleDependencyGraph.self,
135+
forceResponseFiles: forceResponseFiles,
136+
recordedInputModificationDates: recordedInputModificationDates)
137+
return dependencyGraph
138+
}
76139
}

0 commit comments

Comments
 (0)