Refactor pigz support (#113)

* Refactor pigz support

* Add tests

Author: omus

src/ArchiveUtils.jl

@@ -151,9 +151,9 @@ end
 
 # Stripped down copy of `Pkg.Artifacts.archive_artifact` that supports a custom `package`
 # function
-function _archive_artifact(hash::SHA1, tarball_path::String;
-                           honor_overrides::Bool=false,
-                           package::Function=_package_fast)
+function archive_artifact(hash::SHA1, tarball_path::String;
+                          honor_overrides::Bool=false,
+                          package::Function=package)
 
     if !artifact_exists(hash)
         error("Unable to archive artifact $(bytes2hex(hash.bytes)): does not exist!")
@@ -168,12 +168,20 @@ function _archive_artifact(hash::SHA1, tarball_path::String;
     end
 end
 
-# Copy of `Pkg.PlatformEngines.package` but using `pigz` instead of `7z`
-function _package_fast(src_dir::AbstractString, tarball_path::AbstractString)
+# An improved version of `Pkg.PlatformEngines.package`
+function package(src_dir::AbstractString, tarball_path::AbstractString;
+                 format::AbstractString="gzip")
     rm(tarball_path, force=true)
-    pigz() do pigz_exe
-        open(pipeline(`$pigz_exe -9`, stdout=tarball_path), write=true) do io
-            Tar.create(src_dir, io)
-        end
+
+    # Note: For compressing gzip files we use pigz since it uses threading where as p7zip
+    # does not.
+    compress_cmd = if format == "gzip"
+        pipeline(`$(pigz()) -9`, stdout=tarball_path)
+    else
+        pipeline(`$(p7zip()) a -si -t$format -mx9 $tarball_path`, stdout=devnull)
+    end
+
+    open(compress_cmd, write=true) do io
+        Tar.create(src_dir, io)
     end
 end

src/Prefix.jl

@@ -173,7 +173,7 @@ function package(prefix::Prefix,
         @info("Tree hash of contents of $(basename(out_path)): $(tree_hash)")
     end
 
-    tarball_hash = _archive_artifact(tree_hash, out_path; honor_overrides=false)
+    tarball_hash = archive_artifact(tree_hash, out_path; honor_overrides=false)
     if verbose
         @info("SHA256 of $(basename(out_path)): $(tarball_hash)")
     end

test/archive_utils.jl

@@ -0,0 +1,64 @@
+using BinaryBuilderBase: archive_artifact, package
+using Pkg.Artifacts: create_artifact, remove_artifact, with_artifacts_directory
+using Pkg: PlatformEngines
+
+@testset "Archive Utils" begin
+    @testset "package" begin
+        mktempdir() do prefix
+            # Create random files
+            mkpath(joinpath(prefix, "bin"))
+            mkpath(joinpath(prefix, "lib"))
+            mkpath(joinpath(prefix, "etc"))
+            bar_path = joinpath(prefix, "bin", "bar.sh")
+            open(bar_path, "w") do f
+                write(f, "#!/bin/sh\n")
+                write(f, "echo yolo\n")
+            end
+            baz_path = joinpath(prefix, "lib", "baz.so")
+            open(baz_path, "w") do f
+                write(f, "this is not an actual .so\n")
+            end
+
+            qux_path = joinpath(prefix, "etc", "qux.conf")
+            open(qux_path, "w") do f
+                write(f, "use_julia=true\n")
+            end
+
+            mktempdir() do output_dir
+                for (format, ext) in [("gzip", "gz"), ("xz", "xz")]
+                    tarball_path =  joinpath(output_dir, "foo.tar.$ext")
+                    package(prefix, tarball_path; format=format)
+                    @test isfile(tarball_path)
+
+                    # Test that we can inspect the contents of the tarball
+                    contents = PlatformEngines.list_tarball_files(tarball_path)
+                    @test "bin/bar.sh" in contents
+                    @test "lib/baz.so" in contents
+                    @test "etc/qux.conf" in contents
+                end
+            end
+        end
+    end
+
+    @testset "Artifact archival" begin
+        mktempdir() do art_dir
+            with_artifacts_directory(art_dir) do
+                hash = create_artifact(p -> touch(joinpath(p, "foo")))
+                tarball_path = joinpath(art_dir, "foo.tar.gz")
+                archive_artifact(hash, tarball_path)
+                @test "foo" in PlatformEngines.list_tarball_files(tarball_path)
+                rm(tarball_path)
+
+                # Test custom `package` function and ensure failure if no `tarball_path` file
+                # is created.
+                package_alt(src_dir, tarball_path) = nothing
+                @test !isfile(tarball_path)
+                @test_throws SystemError archive_artifact(hash, tarball_path, package=package_alt)
+
+                # Test archiving something that doesn't exist fails
+                remove_artifact(hash)
+                @test_throws ErrorException archive_artifact(hash, tarball_path)
+            end
+        end
+    end
+end

test/runtests.jl

@@ -1,5 +1,6 @@
 # Run all our tests
 include("compat.jl")
+include("archive_utils.jl")
 include("dependencies.jl")
 include("platforms.jl")
 include("prefix.jl")