Add field to dependencies to filter them by platform (#176)

* Add field to dependencies to filter them by platform

This let us declare that certain dependencies are needed at build/runtime only
on some platforms.

* Propagate `filter_platforms` through `registry_resolve!`

* [Dependencies] Make the new field a vector of platforms instead of a function

* Add test about `AnyPlatform` matching all platforms

* [Dependencies] Add serialisation of `platforms` field

* [Dependencies] Bring back `filter_platforms` function

Co-authored-by: Valentin Churavy <vchuravy@users.noreply.github.com>

Author: giordano

Project.toml

@@ -1,7 +1,8 @@
 name = "BinaryBuilderBase"
 uuid = "7f725544-6523-48cd-82d1-3fa08ff4056e"
 authors = ["Elliot Saba <staticfloat@gmail.com>"]
-version = "1.1.2"
+version = "1.2.0"
+
 
 [deps]
 CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"

src/Dependencies.jl

@@ -1,7 +1,8 @@
 using UUIDs
 
 export Dependency, BuildDependency, HostBuildDependency,
-    is_host_dependency, is_target_dependency, is_build_dependency, is_runtime_dependency
+    is_host_dependency, is_target_dependency, is_build_dependency, is_runtime_dependency,
+    filter_platforms
 
 
 # Pkg.PackageSpec return different types in different Julia versions so...
@@ -54,7 +55,7 @@ Return whether `dep` is a runtime dependency or not.
 is_runtime_dependency
 
 """
-    Dependency(dep::Union{PackageSpec,String}, build_version; compat)
+    Dependency(dep::Union{PackageSpec,String}, build_version; compat, platforms)
 
 Define a binary dependency that is necessary to build the package and load the
 generated JLL package.  The argument can be either a string with the name of the
@@ -67,12 +68,19 @@ The optional keyword argument `compat` can be used to specify a string for use
 in the `Project.toml` of the generated Julia package.  If `compat` is non-empty
 and `build_version` is not passed, the latter defaults to the minimum version
 compatible with the `compat` specifier.
+
+The optional keyword argument `platforms` is a vector of `AbstractPlatform`s
+which indicates for which platforms the dependency should be used.  By default
+`platforms=[AnyPlatform()]`, to mean that the dependency is compatible with all
+platforms.
 """
 struct Dependency <: AbstractDependency
     pkg::PkgSpec
     build_version::Union{VersionNumber,Nothing}
     compat::String  # semver string for use in Project.toml of the JLL
-    function Dependency(pkg::PkgSpec, build_version = nothing; compat::String = "")
+    platforms::Vector{<:AbstractPlatform}
+    function Dependency(pkg::PkgSpec, build_version = nothing; compat::String = "",
+                        platforms::Vector{<:AbstractPlatform}=[AnyPlatform()])
         if length(compat) > 0
             spec = PKG_VERSIONS.semver_spec(compat) # verify compat is valid
             if build_version === nothing
@@ -88,45 +96,64 @@ struct Dependency <: AbstractDependency
                 throw(ArgumentError("PackageSpec version and compat for $(pkg) are incompatible"))
             end
         end
-        new(pkg, build_version, compat)
+        new(pkg, build_version, compat, platforms)
     end
 end
-function Dependency(dep::AbstractString, build_version = nothing; compat::String = "")
-    return Dependency(PackageSpec(; name = dep), build_version, compat = compat)
+function Dependency(dep::AbstractString, build_version = nothing;
+                    compat::String = "", platforms::Vector{<:AbstractPlatform}=[AnyPlatform()])
+    return Dependency(PackageSpec(; name = dep), build_version; compat, platforms)
 end
 is_host_dependency(::Dependency) = false
 is_build_dependency(::Dependency) = true
 is_runtime_dependency(::Dependency) = true
 
 """
-    BuildDependency(dep::Union{PackageSpec,String})
+    BuildDependency(dep::Union{PackageSpec,String}; platforms)
 
 Define a binary dependency that is necessary only to build the package.  The
-argument can be either a string with the name of the JLL package or a
+`dep` argument can be either a string with the name of the JLL package or a
 `Pkg.PackageSpec`.
+
+The optional keyword argument `platforms` is a vector of `AbstractPlatform`s
+which indicates for which platforms the dependency should be used.  By default
+`platforms=[AnyPlatform()]`, to mean that the dependency is compatible with all
+platforms.
 """
 struct BuildDependency <: AbstractDependency
     pkg::PkgSpec
+    platforms::Vector{<:AbstractPlatform}
+    BuildDependency(pkg::PkgSpec; platforms::Vector{<:AbstractPlatform}=[AnyPlatform()]) =
+        new(pkg, platforms)
 end
-BuildDependency(dep::AbstractString) = BuildDependency(PackageSpec(; name = dep))
+BuildDependency(dep::AbstractString; platforms::Vector{<:AbstractPlatform}=[AnyPlatform()]) =
+    BuildDependency(PackageSpec(; name = dep); platforms)
 is_host_dependency(::BuildDependency) = false
 is_build_dependency(::BuildDependency) = true
 is_runtime_dependency(::BuildDependency) = false
 
 """
-    HostBuildDependency(dep::Union{PackageSpec,String})
+    HostBuildDependency(dep::Union{PackageSpec,String}; platforms)
 
 Define a binary dependency that is necessary only to build the package.
 Different from the [`BuildDependency`](@ref), the artifact for the host
 platform will be installed, instead of that for the target platform.
 
-The argument can be either a string with the name of the JLL package or a
+The `dep` argument can be either a string with the name of the JLL package or a
 `Pkg.PackageSpec`.
+
+The optional keyword argument `platforms` is a vector of `AbstractPlatform`s
+which indicates for which platforms the dependency should be used.  By default
+`platforms=[AnyPlatform()]`, to mean that the dependency is compatible with all
+platforms.
 """
 struct HostBuildDependency <: AbstractDependency
     pkg::PkgSpec
+    platforms::Vector{<:AbstractPlatform}
+    HostBuildDependency(pkg::PkgSpec; platforms::Vector{<:AbstractPlatform}=[AnyPlatform()]) =
+        new(pkg, platforms)
 end
-HostBuildDependency(dep::AbstractString) = HostBuildDependency(PackageSpec(; name = dep))
+HostBuildDependency(dep::AbstractString; platforms::Vector{<:AbstractPlatform}=[AnyPlatform()]) =
+    HostBuildDependency(PackageSpec(; name = dep); platforms)
 is_host_dependency(::HostBuildDependency) = true
 is_build_dependency(::HostBuildDependency) = true
 is_runtime_dependency(::HostBuildDependency) = false
@@ -145,12 +172,20 @@ end
 getname(x::PkgSpec) = x.name
 getname(x::AbstractDependency) = getname(getpkg(x))
 
+"""
+    filter_platforms(deps::AbstractVector{<:AbstractDependency}, p::AbstractPlatform)
+
+Filter the dependencies `deps` which are compatible with platform `p`.
+"""
+filter_platforms(deps::AbstractVector{<:AbstractDependency}, p::AbstractPlatform) =
+    [dep for dep in deps if any(x -> platforms_match(x, p), dep.platforms)]
+
 # Wrapper around `Pkg.Types.registry_resolve!` which keeps the type of the
 # dependencies.  TODO: improve this
 function registry_resolve!(ctx, dependencies::Vector{<:AbstractDependency})
     resolved_dependencies = Pkg.Types.registry_resolve!(ctx, getpkg.(dependencies))
     for idx in eachindex(dependencies)
-        dependencies[idx] = typeof(dependencies[idx])(resolved_dependencies[idx])
+        dependencies[idx] = typeof(dependencies[idx])(resolved_dependencies[idx]; platforms=dependencies[idx].platforms)
     end
     return dependencies
 end
@@ -222,7 +257,9 @@ for (type, type_descr) in ((Dependency, "dependency"), (BuildDependency, "buildd
                                "compat" => getcompat(d),
                                "version-major" => major(version(d)),
                                "version-minor" => minor(version(d)),
-                               "version-patch" => patch(version(d)))
+                               "version-patch" => patch(version(d)),
+                               "platforms" => triplet.(d.platforms),
+                               )
 end
 
 # When deserialiasing the JSON file, the dependencies are in the form of
@@ -235,12 +272,13 @@ function dependencify(d::Dict)
         version = VersionNumber(d["version-major"], d["version-minor"], d["version-patch"])
         version = version == v"0" ? nothing : version
         spec = PackageSpec(; name = d["name"], uuid = uuid, version = version)
+        platforms = parse_platform.(d["platforms"])
         if d["type"] == "dependency"
-            return Dependency(spec; compat = compat)
+            return Dependency(spec; compat, platforms)
         elseif d["type"] == "builddependency"
-            return BuildDependency(spec)
+            return BuildDependency(spec; platforms)
         elseif d["type"] == "hostdependency"
-            return HostBuildDependency(spec)
+            return HostBuildDependency(spec; platforms)
         end
     end
     error("Cannot convert to dependency")

src/Platforms.jl

@@ -39,6 +39,11 @@ Get the executable extension for the given Platform.  Includes the leading `.`.
 platform_exeext(p::AbstractPlatform) = Sys.iswindows(p) ? ".exe" : ""
 
 
+# Helper parsing function: it extends `parse(Platform, p)` by supporting
+# `AnyPlatform` as well.
+parse_platform(p::String) = p == "any" ? AnyPlatform() : parse(Platform, p; validate_strict=true)
+
+
 # Recursively test for key presence in nested dicts
 function haskeys(d, keys...)
     for key in keys

test/dependencies.jl

@@ -17,7 +17,7 @@ end
 
 @testset "Dependencies" begin
     name = "Foo_jll"
-    dep = Dependency(PackageSpec(; name = name))
+    dep = Dependency(PackageSpec(; name = name); platforms=supported_platforms(; experimental=true, exclude=!Sys.isapple))
     @test Dependency(name) == dep
     @test !is_host_dependency(dep)
     @test is_target_dependency(dep)
@@ -36,7 +36,7 @@ end
     @test getcompat(dep_buildver) == ""
 
     # the same but with compat info
-    dep_buildver = Dependency(PackageSpec(; name = name), build_version, compat = "~1.2")
+    dep_buildver = Dependency(PackageSpec(; name = name), build_version; compat = "~1.2", platforms=[Platform("x86_64", "linux"; cxxstring_abi="cxx11")])
     @test Dependency(name, build_version) == dep_buildver
     @test getname(dep_buildver) == name
     @test getpkg(dep_buildver) == PackageSpec(; name = name, version = build_version)
@@ -74,27 +74,32 @@ end
     @test getname(PackageSpec(; name = host_name)) == host_name
     @test getpkg(host_dep) == PackageSpec(; name = host_name)
 
+    @testset "Filter dependencies by platform" begin
+        @test filter_platforms([dep, dep_buildver, dep_compat], Platform("x86_64", "linux"; cxxstring_abi="cxx03")) == [dep_compat]
+        @test filter_platforms([dep, dep_buildver, dep_compat], Platform("x86_64", "macos")) == [dep, dep_compat]
+    end
+
     @testset "JSON (de)serialization" begin
         jdep = JSON.lower(dep)
-        @test jdep == Dict("type" => "dependency", "name" => name, "uuid" => nothing, "compat" => "", "version-major" => 0x0, "version-minor" => 0x0, "version-patch" => 0x0)
+        @test jdep == Dict("type" => "dependency", "name" => name, "uuid" => nothing, "compat" => "", "version-major" => 0x0, "version-minor" => 0x0, "version-patch" => 0x0, "platforms" => ["x86_64-apple-darwin", "aarch64-apple-darwin"])
         @test dependencify(jdep) == dep
 
         jdep_buildver = JSON.lower(dep_buildver)
-        @test jdep_buildver == Dict("type" => "dependency", "name" => name, "uuid" => nothing, "compat" => "~1.2", "version-major" => 0x0, "version-minor" => 0x0, "version-patch" => 0x0)
+        @test jdep_buildver == Dict("type" => "dependency", "name" => name, "uuid" => nothing, "compat" => "~1.2", "version-major" => 0x0, "version-minor" => 0x0, "version-patch" => 0x0, "platforms" => ["x86_64-linux-gnu-cxx11"])
         # the build_version is currently not serialized, so the following test fails
         @test_broken dependencify(jdep_buildver) == dep_buildver
 
         jbuild_dep = JSON.lower(build_dep)
-        @test jbuild_dep == Dict("type" => "builddependency", "name" => build_name, "uuid" => nothing, "compat" => "", "version-major" => 0x0, "version-minor" => 0x0, "version-patch" => 0x0)
+        @test jbuild_dep == Dict("type" => "builddependency", "name" => build_name, "uuid" => nothing, "compat" => "", "version-major" => 0x0, "version-minor" => 0x0, "version-patch" => 0x0, "platforms" => ["any"])
         @test dependencify(jbuild_dep) == build_dep
 
         jhost_dep = JSON.lower(host_dep)
-        @test jhost_dep == Dict("type" => "hostdependency", "name" => host_name, "uuid" => nothing, "compat" => "", "version-major" => 0x0, "version-minor" => 0x0, "version-patch" => 0x0)
+        @test jhost_dep == Dict("type" => "hostdependency", "name" => host_name, "uuid" => nothing, "compat" => "", "version-major" => 0x0, "version-minor" => 0x0, "version-patch" => 0x0, "platforms" => ["any"])
         @test dependencify(jhost_dep) == host_dep
 
         full_dep = Dependency(PackageSpec(; name = "Baz_jll", uuid = "00000000-1111-2222-3333-444444444444", version = "3.1.4"))
         jfull_dep = JSON.lower(full_dep)
-        @test jfull_dep == Dict("type" => "dependency", "name" => "Baz_jll", "uuid" => "00000000-1111-2222-3333-444444444444", "compat" => "", "version-major" => 0x3, "version-minor" => 0x1, "version-patch" => 0x4)
+        @test jfull_dep == Dict("type" => "dependency", "name" => "Baz_jll", "uuid" => "00000000-1111-2222-3333-444444444444", "compat" => "", "version-major" => 0x3, "version-minor" => 0x1, "version-patch" => 0x4, "platforms" => ["any"])
         @test dependencify(jfull_dep) == full_dep
         @test_throws ErrorException dependencify(Dict("type" => "git"))
     end

test/platforms.jl

@@ -53,6 +53,10 @@ end
         )
     @test BinaryBuilderBase.choose_shards(AnyPlatform()) == BinaryBuilderBase.choose_shards(default_host_platform)
     @test BinaryBuilderBase.aatriplet(AnyPlatform()) == BinaryBuilderBase.aatriplet(default_host_platform)
+
+    # Make sure `AnyPlatform` matches all platforms we can possibly support.
+    @test all(p -> platforms_match(AnyPlatform(), p),
+              expand_microarchitectures(expand_gfortran_versions(expand_cxxstring_abis(supported_platforms(; experimental=true)))))
 end
 
 @testset "Target properties" begin