Skip to content

Commit 9653927

Browse files
rconnell9rconnell9
authored
Provide the downloadable metal toolchain to swiftbuild (#9434)
Provide the downloadable metal toolchain to swiftbuild using EXTERNAL_TOOLCHAINS_DIR env variable ### Motivation: #9202 [Parity] error: unable to spawn process 'metal' (No such file or directory) Requires CI to include Metal toolchain in order to enable Metal tests: #9443
1 parent c26009f commit 9653927

File tree

12 files changed

+241
-12
lines changed

12 files changed

+241
-12
lines changed

‎Fixtures/Metal/SimpleLibrary/Package.swift‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
// swift-tools-version: 6.2
2+
// The swift-tools-version declares the minimum version of Swift required to build this package.
3+
import PackageDescription
4+
5+
let package = Package(
6+
name: "MyRenderer",
7+
products: [
8+
.library(
9+
name: "MyRenderer",
10+
targets: ["MyRenderer"]),
11+
],
12+
targets: [
13+
.target(
14+
name: "MyRenderer",
15+
dependencies: ["MySharedTypes"]),
16+
17+
.target(name: "MySharedTypes")
18+
]
19+
)

‎Fixtures/Metal/SimpleLibrary/Sources/MyRenderer/Renderer.swift‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
import MySharedTypes
2+
3+
4+
let vertex = AAPLVertex(position: .init(250, -250), color: .init(1, 0, 0, 1))

‎Fixtures/Metal/SimpleLibrary/Sources/MyRenderer/Shaders.metal‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
// A relative path to SharedTypes.h.
2+
#import "../MySharedTypes/include/SharedTypes.h"
3+
4+
#include <metal_stdlib>
5+
using namespace metal;
6+
7+
vertex float4 simpleVertexShader(const device AAPLVertex *vertices [[buffer(0)]],
8+
uint vertexID [[vertex_id]]) {
9+
AAPLVertex in = vertices[vertexID];
10+
return float4(in.position.x, in.position.y, 0.0, 1.0);
11+
}
12+

‎Fixtures/Metal/SimpleLibrary/Sources/MySharedTypes/include/SharedTypes.h‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#ifndef SharedTypes_h
2+
#define SharedTypes_h
3+
4+
5+
#import <simd/simd.h>
6+
7+
8+
typedef struct {
9+
vector_float2 position;
10+
vector_float4 color;
11+
} AAPLVertex;
12+
13+
14+
#endif /* SharedTypes_h */

‎Fixtures/Metal/SimpleLibrary/Tests/MyRendererTests/MyRendererTests.swift‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import Testing
2+
@testable import MyRenderer
3+
4+
@Test func example() async throws {
5+
// Write your test here and use APIs like `#expect(...)` to check expected conditions.
6+
}

‎Package.swift‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1017,6 +1017,13 @@ let package = Package(
10171017
name: "SwiftBuildSupportTests",
10181018
dependencies: ["SwiftBuildSupport", "_InternalTestSupport", "_InternalBuildTestSupport"]
10191019
),
1020+
.testTarget(
1021+
name: "BuildMetalTests",
1022+
dependencies: [
1023+
"_InternalTestSupport",
1024+
"Basics"
1025+
]
1026+
),
10201027
// Examples (These are built to ensure they stay up to date with the API.)
10211028
.executableTarget(
10221029
name: "package-info",

‎Sources/PackageModel/Toolchain.swift‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,12 @@ public protocol Toolchain {
4747
/// The manifest and library locations used by this toolchain.
4848
var swiftPMLibrariesLocation: ToolchainConfiguration.SwiftPMLibrariesLocation { get }
4949

50+
/// Path to the Metal toolchain if available.
51+
var metalToolchainPath: AbsolutePath? { get }
52+
53+
// Metal toolchain ID if available.
54+
var metalToolchainId: String? { get }
55+
5056
/// Path of the `clang` compiler.
5157
func getClangCompiler() throws -> AbsolutePath
5258

‎Sources/PackageModel/ToolchainConfiguration.swift‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,14 @@ public struct ToolchainConfiguration {
4646
/// Currently computed only for Windows.
4747
public var swiftTestingPath: AbsolutePath?
4848

49+
/// Path to the Metal toolchain.
50+
/// This is optional and only available on Darwin platforms.
51+
public var metalToolchainPath: AbsolutePath?
52+
53+
/// Metal toolchain identifier
54+
/// This is optional and only available on Darwin platforms.
55+
public var metalToolchainId: String?
56+
4957
/// Creates the set of manifest resources associated with a `swiftc` executable.
5058
///
5159
/// - Parameters:
@@ -56,6 +64,8 @@ public struct ToolchainConfiguration {
5664
/// - swiftPMLibrariesRootPath: Custom path for SwiftPM libraries. Computed based on the compiler path by default.
5765
/// - sdkRootPath: Optional path to SDK root.
5866
/// - xctestPath: Optional path to XCTest.
67+
/// - swiftTestingPath: Optional path to swift-testing.
68+
/// - metalToolchainPath: Optional path to Metal toolchain.
5969
public init(
6070
librarianPath: AbsolutePath,
6171
swiftCompilerPath: AbsolutePath,
@@ -64,7 +74,9 @@ public struct ToolchainConfiguration {
6474
swiftPMLibrariesLocation: SwiftPMLibrariesLocation? = nil,
6575
sdkRootPath: AbsolutePath? = nil,
6676
xctestPath: AbsolutePath? = nil,
67-
swiftTestingPath: AbsolutePath? = nil
77+
swiftTestingPath: AbsolutePath? = nil,
78+
metalToolchainPath: AbsolutePath? = nil,
79+
metalToolchainId: String? = nil
6880
) {
6981
let swiftPMLibrariesLocation = swiftPMLibrariesLocation ?? {
7082
return .init(swiftCompilerPath: swiftCompilerPath)
@@ -78,6 +90,8 @@ public struct ToolchainConfiguration {
7890
self.sdkRootPath = sdkRootPath
7991
self.xctestPath = xctestPath
8092
self.swiftTestingPath = swiftTestingPath
93+
self.metalToolchainPath = metalToolchainPath
94+
self.metalToolchainId = metalToolchainId
8195
}
8296
}
8397

‎Sources/PackageModel/UserToolchain.swift‎

Lines changed: 62 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -931,6 +931,8 @@ public final class UserToolchain: Toolchain {
931931
)
932932
}
933933

934+
let metalToolchain = try? Self.deriveMetalToolchainPath(fileSystem: fileSystem, triple: triple, environment: environment)
935+
934936
self.configuration = .init(
935937
librarianPath: librarianPath,
936938
swiftCompilerPath: swiftCompilers.manifest,
@@ -939,7 +941,9 @@ public final class UserToolchain: Toolchain {
939941
swiftPMLibrariesLocation: swiftPMLibrariesLocation,
940942
sdkRootPath: self.swiftSDK.pathsConfiguration.sdkRootPath,
941943
xctestPath: xctestPath,
942-
swiftTestingPath: swiftTestingPath
944+
swiftTestingPath: swiftTestingPath,
945+
metalToolchainPath: metalToolchain?.path,
946+
metalToolchainId: metalToolchain?.identifier
943947
)
944948

945949
self.fileSystem = fileSystem
@@ -1071,6 +1075,55 @@ public final class UserToolchain: Toolchain {
10711075
return (platform, info)
10721076
}
10731077

1078+
private static func deriveMetalToolchainPath(
1079+
fileSystem: FileSystem,
1080+
triple: Basics.Triple,
1081+
environment: Environment
1082+
) throws -> (path: AbsolutePath, identifier: String)? {
1083+
guard triple.isDarwin() else {
1084+
return nil
1085+
}
1086+
1087+
let xcrunCmd = ["/usr/bin/xcrun", "--find", "metal"]
1088+
guard let output = try? AsyncProcess.checkNonZeroExit(arguments: xcrunCmd, environment: environment).spm_chomp() else {
1089+
return nil
1090+
}
1091+
1092+
guard let metalPath = try? AbsolutePath(validating: output) else {
1093+
return nil
1094+
}
1095+
1096+
guard let toolchainPath: AbsolutePath = {
1097+
var currentPath = metalPath
1098+
while currentPath != currentPath.parentDirectory {
1099+
if currentPath.basename == "Metal.xctoolchain" {
1100+
return currentPath
1101+
}
1102+
currentPath = currentPath.parentDirectory
1103+
}
1104+
return nil
1105+
}() else {
1106+
return nil
1107+
}
1108+
1109+
let toolchainInfoPlist = toolchainPath.appending(component: "ToolchainInfo.plist")
1110+
1111+
struct MetalToolchainInfo: Decodable {
1112+
let Identifier: String
1113+
}
1114+
1115+
let toolchainIdentifier: String
1116+
do {
1117+
let data: Data = try fileSystem.readFileContents(toolchainInfoPlist)
1118+
let info = try PropertyListDecoder().decode(MetalToolchainInfo.self, from: data)
1119+
toolchainIdentifier = info.Identifier
1120+
} catch {
1121+
return nil
1122+
}
1123+
1124+
return (path: toolchainPath.parentDirectory, identifier: toolchainIdentifier)
1125+
}
1126+
10741127
// TODO: We should have some general utility to find tools.
10751128
private static func deriveXCTestPath(
10761129
swiftSDK: SwiftSDK,
@@ -1254,6 +1307,14 @@ public final class UserToolchain: Toolchain {
12541307
configuration.sdkRootPath
12551308
}
12561309

1310+
public var metalToolchainPath: AbsolutePath? {
1311+
configuration.metalToolchainPath
1312+
}
1313+
1314+
public var metalToolchainId: String? {
1315+
configuration.metalToolchainId
1316+
}
1317+
12571318
public var swiftCompilerEnvironment: Environment {
12581319
configuration.swiftCompilerEnvironment
12591320
}

‎Sources/SwiftBuildSupport/SwiftBuildSystem.swift‎

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -64,14 +64,21 @@ package func withService<T>(
6464
public func createSession(
6565
service: SWBBuildService,
6666
name: String,
67-
toolchainPath: Basics.AbsolutePath,
67+
toolchain: Toolchain,
6868
packageManagerResourcesDirectory: Basics.AbsolutePath?
6969
) async throws-> (SWBBuildServiceSession, [SwiftBuildMessage.DiagnosticInfo]) {
70+
71+
var buildSessionEnv: [String: String]? = nil
72+
if let metalToolchainPath = toolchain.metalToolchainPath {
73+
buildSessionEnv = ["EXTERNAL_TOOLCHAINS_DIR": metalToolchainPath.pathString]
74+
}
75+
let toolchainPath = try toolchain.toolchainDir
76+
7077
// SWIFT_EXEC and SWIFT_EXEC_MANIFEST may need to be overridden in debug scenarios in order to pick up Open Source toolchains
7178
let sessionResult = if toolchainPath.components.contains(where: { $0.hasSuffix(".app") }) {
72-
await service.createSession(name: name, developerPath: nil, resourceSearchPaths: packageManagerResourcesDirectory.map { [$0.pathString] } ?? [], cachePath: nil, inferiorProductsPath: nil, environment: nil)
79+
await service.createSession(name: name, developerPath: nil, resourceSearchPaths: packageManagerResourcesDirectory.map { [$0.pathString] } ?? [], cachePath: nil, inferiorProductsPath: nil, environment: buildSessionEnv)
7380
} else {
74-
await service.createSession(name: name, swiftToolchainPath: toolchainPath.pathString, resourceSearchPaths: packageManagerResourcesDirectory.map { [$0.pathString] } ?? [], cachePath: nil, inferiorProductsPath: nil, environment: nil)
81+
await service.createSession(name: name, swiftToolchainPath: toolchainPath.pathString, resourceSearchPaths: packageManagerResourcesDirectory.map { [$0.pathString] } ?? [], cachePath: nil, inferiorProductsPath: nil, environment: buildSessionEnv)
7582
}
7683
switch sessionResult {
7784
case (.success(let session), let diagnostics):
@@ -84,14 +91,14 @@ public func createSession(
8491
func withSession(
8592
service: SWBBuildService,
8693
name: String,
87-
toolchainPath: Basics.AbsolutePath,
94+
toolchain: Toolchain,
8895
packageManagerResourcesDirectory: Basics.AbsolutePath?,
8996
body: @escaping (
9097
_ session: SWBBuildServiceSession,
9198
_ diagnostics: [SwiftBuild.SwiftBuildMessage.DiagnosticInfo]
9299
) async throws -> Void
93100
) async throws {
94-
let (session, diagnostics) = try await createSession(service: service, name: name, toolchainPath: toolchainPath, packageManagerResourcesDirectory: packageManagerResourcesDirectory)
101+
let (session, diagnostics) = try await createSession(service: service, name: name, toolchain: toolchain, packageManagerResourcesDirectory: packageManagerResourcesDirectory)
95102
do {
96103
try await body(session, diagnostics)
97104
} catch let bodyError {
@@ -546,7 +553,7 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem {
546553

547554
var serializedDiagnosticPathsByTargetName: [String: [Basics.AbsolutePath]] = [:]
548555
do {
549-
try await withSession(service: service, name: self.buildParameters.pifManifest.pathString, toolchainPath: self.buildParameters.toolchain.toolchainDir, packageManagerResourcesDirectory: self.packageManagerResourcesDirectory) { session, _ in
556+
try await withSession(service: service, name: self.buildParameters.pifManifest.pathString, toolchain: self.buildParameters.toolchain, packageManagerResourcesDirectory: self.packageManagerResourcesDirectory) { session, _ in
550557
self.outputStream.send("Building for \(self.buildParameters.configuration == .debug ? "debugging" : "production")...\n")
551558

552559
// Load the workspace, and set the system information to the default
@@ -881,16 +888,20 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem {
881888
if setToolchainSetting {
882889
// If the SwiftPM toolchain corresponds to a toolchain registered with the lower level build system, add it to the toolchain stack.
883890
// Otherwise, apply overrides for each component of the SwiftPM toolchain.
884-
if let toolchainID = try await session.lookupToolchain(at: buildParameters.toolchain.toolchainDir.pathString) {
885-
settings["TOOLCHAINS"] = "\(toolchainID.rawValue) $(inherited)"
886-
} else {
891+
let toolchainID = try await session.lookupToolchain(at: buildParameters.toolchain.toolchainDir.pathString)
892+
if toolchainID == nil {
887893
// FIXME: This list of overrides is incomplete.
888894
// An error with determining the override should not be fatal here.
889895
settings["CC"] = try? buildParameters.toolchain.getClangCompiler().pathString
890896
// Always specify the path of the effective Swift compiler, which was determined in the same way as for the
891897
// native build system.
892898
settings["SWIFT_EXEC"] = buildParameters.toolchain.swiftCompilerPath.pathString
893899
}
900+
901+
let overrideToolchains = [buildParameters.toolchain.metalToolchainId, toolchainID?.rawValue].compactMap { $0 }
902+
if !overrideToolchains.isEmpty {
903+
settings["TOOLCHAINS"] = (overrideToolchains + ["$(inherited)"]).joined(separator: " ")
904+
}
894905
}
895906

896907
for sanitizer in buildParameters.sanitizers.sanitizers {
@@ -1250,7 +1261,7 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem {
12501261
package func createLongLivedSession(name: String) async throws -> LongLivedBuildServiceSession {
12511262
let service = try await SWBBuildService(connectionMode: .inProcessStatic(swiftbuildServiceEntryPoint))
12521263
do {
1253-
let (session, diagnostics) = try await createSession(service: service, name: name, toolchainPath: buildParameters.toolchain.toolchainDir, packageManagerResourcesDirectory: packageManagerResourcesDirectory)
1264+
let (session, diagnostics) = try await createSession(service: service, name: name, toolchain: buildParameters.toolchain, packageManagerResourcesDirectory: packageManagerResourcesDirectory)
12541265
let teardownHandler = {
12551266
try await session.close()
12561267
await service.close()

0 commit comments

Comments
 (0)