[Auditor] Add a pass to normalise timestamps in Windows import libraries (#1259)

giordano · web-flow · commit 38b664ed793b · 2023-01-21T17:46:19.000Z
* [Auditor] Add a pass to normalise timestamps in Windows import libraries

This should ensure reproducibility of import libraries.

* Run repro tests for Windows libraries on i686

This avoids downloading yet another compiler shard.

* Force `Pkg.gc`
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -63,6 +63,7 @@ jobs:
   steps:
   - bash: |
       set -e
+      $(JULIA) -e 'using Pkg; Pkg.gc()'
       $(JULIA) --check-bounds=yes --inline=yes -e 'using Pkg; Pkg.Registry.update(); Pkg.instantiate(); Pkg.test(coverage=true)'
       $(JULIA) -e 'using Pkg; Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder())' || true
     name: Test
diff --git a/src/Auditor.jl b/src/Auditor.jl
@@ -201,7 +201,9 @@ function audit(prefix::Prefix, src_name::AbstractString = "";
             continue
         end
         # remove it
-        @info("Removing libtool file $f")
+        if verbose
+            @info("Removing libtool file $f")
+        end
         rm(f; force=true)
     end
 
@@ -245,6 +247,16 @@ function audit(prefix::Prefix, src_name::AbstractString = "";
                 mv(f, joinpath(prefix, "bin", basename(f)))
             end
         end
+
+        # Normalise timestamp of Windows import libraries.
+        import_libraries = collect_files(prefix, endswith(".dll.a"))
+        for implib in import_libraries
+            if verbose
+                @info("Normalising timestamps in import library $(implib)")
+            end
+            normalise_implib_timestamp(implib)
+        end
+
     end
 
     # Check that we're providing a license file
diff --git a/src/auditor/extra_checks.jl b/src/auditor/extra_checks.jl
@@ -1,6 +1,8 @@
 # This file contains some extra checks that don't fall into the other
 # categories, for example because they're very platform-specific.
 
+using Dates: DateTime, datetime2unix
+
 function check_os_abi(oh::ObjectHandle, p::AbstractPlatform, rest...; verbose::Bool = false, kwargs...)
     if Sys.isfreebsd(p)
         if oh.ei.osabi != ELF.ELFOSABI_FREEBSD
@@ -29,3 +31,19 @@ function check_os_abi(oh::ObjectHandle, p::AbstractPlatform, rest...; verbose::B
     end
     return true
 end
+
+# Problem: import libraries on Windows embed information about temporary object files,
+# including their modification time on disk, which makes import libraries non-reproducible:
+# <https://github.com/JuliaPackaging/BinaryBuilder.jl/issues/1245>.
+function normalise_implib_timestamp(path::AbstractString)
+    # Format of the object file info is something like
+    #     /      1674301251  0     0     644     286       `
+    # where `1674301251` is the Unix timestamp of the modification time of the library, we
+    # normalise it to another timestamp.  NOTE: it appears that changing the timestamp width
+    # would break the library, so we use another fixed 10-digit wide timestamp.  10-digit
+    # Unix timestamps span between 2001 and 2286, so we should be good for a while.
+    timestamp = trunc(Int, datetime2unix(DateTime(2013, 2, 13, 0, 49, 0))) # Easter egg
+    newlib = replace(read(path, String), r"(?<HEAD>/ +)(\d{10})(?<TAIL> +\d+ +\d+ +\d+ +\d+ +`)" => SubstitutionString("\\g<HEAD>$(timestamp)\\g<TAIL>"))
+    # Write back the file to disk.
+    write(path, newlib)
+end
diff --git a/test/auditing.jl b/test/auditing.jl
@@ -306,7 +306,8 @@ end
                     # The products we expect to be build
                     libfoo_products,
                     # No dependencies
-                    Dependency[]
+                    Dependency[];
+                    verbose = true,
                 )
             end
 
@@ -726,6 +727,54 @@ end
     end
 end
 
+# References:
+# * https://github.com/JuliaPackaging/BinaryBuilder.jl/issues/1232
+# * https://github.com/JuliaPackaging/BinaryBuilder.jl/issues/1245
+@testset "Auditor - Reproducible libraries on Windows" begin
+    platform = Platform("i686", "windows")
+    expected_git_shas = Dict(
+        v"4" => Base.SHA1("64017544630a472081ba10879e58901d5e3b53a0"),
+        v"6" => Base.SHA1("31afae67c528151860c18128696b19c85309b556"),
+    )
+    @testset "gcc version $(gcc_version)" for gcc_version in (v"4", v"6")
+        mktempdir() do build_path
+            build_output_meta = nothing
+            product = LibraryProduct("libfoo", :libfoo)
+            @test_logs (:info, r"Normalising timestamps in import library") match_mode=:any begin
+                build_output_meta = autobuild(
+                    build_path,
+                    "implib",
+                    v"1.0.0",
+                    # No sources
+                    FileSource[],
+                    # Build a library without execution permissions
+                    raw"""
+                    mkdir -p "${libdir}"
+                    cd "${libdir}"
+                    echo 'int foo(){ return 42; }' | cc -x c -shared - -o libfoo.${dlext} -Wl,--out-implib,libfoo.${dlext}.a
+                    """,
+                    # Build for Windows
+                    [platform],
+                    # Ensure our library product is built
+                    [product],
+                    # No dependencies
+                    Dependency[];
+                    verbose = true,
+                    require_license = false,
+                    preferred_gcc_version = gcc_version,
+                )
+            end
+
+            # Extract our platform's build
+            @test haskey(build_output_meta, platform)
+            tarball_path, tarball_hash = build_output_meta[platform][1:2]
+            @test isfile(tarball_path)
+            # Ensure reproducibility of build
+            @test build_output_meta[platform][3] == expected_git_shas[gcc_version]
+        end
+    end
+end
+
 @testset "Auditor - other checks" begin
     platform = Platform("armv7l", "linux"; call_abi = "eabihf", libc = "glibc")
     mktempdir() do build_path
