Merge remote-tracking branch 'shlevy/prebuilt-depends' into hkm/prebuilt-depends

Author: hamishmack

builder/comp-builder.nix

@@ -90,6 +90,65 @@
 # LLVM
 , useLLVM ? ghc.useLLVM or false
 , smallAddressSpace ? false
+
+# Note [prebuilt dependencies]
+#
+# Typical cabal project planning starts with the libraries that come with
+# the compiler and then plans to build every other needed dependency from
+# source (fetched through hackage repositories or source-repository
+# dependencies). In cases where some library that isn't part of the compiler
+# is only available as a pre-built shared object file (such as for some
+# closed-source module from a vendor, or in principle a component whose
+# compilation is extremely expensive), we need to be able to tell cabal
+# about additional prebuilt dependencies to include in its plan and link to
+# as needed at build time.
+#
+# This can be done by passing the needed libraries in prebuilt-depends. During
+# cabal planning and builds, these libraries (and their dependencies) will be
+# present in the ghc-pkg database that cabal will draw from for its dependency
+# resolution, thereby skipping lookup from hackage or building from source.
+#
+# The entries in the prebuilt dependencies list may have dependencies that are
+# part of the compiler-provided package set, or may have overlap with each other.
+# GHC can actually handle this use case fine, since types from different packages
+# (even of the same name) will not unify, so at worst you will get a compile error,
+# but cabal will need to choose one for the packages you are building. The entries
+# in the list are given priority over the compiler-provided ones, with the later
+# entries having greater priority than the earlier ones.
+#
+# For your build to succeed, your prebuilt-dependencies must meet the following:
+#
+# 1. They are built with the same compiler version and RTS way.
+# 2. They have all of their dependencies within the package db directory or
+#    included as propagatedBuildInputs
+# 3. They must include the `envDep` and `exactDep` files that make-config.files.nix
+#    expects for configuring cabal precisely.
+#
+# The recommended way to meet this requirement is to build the relevant libraries
+# with haskell.nix too, since it sets up the dependencies appropriately. An example
+# workflow would be:
+#
+# 1. Build libraries foo and bar with haskell.nix (the same plan)
+# 2. Note down the store paths for foo and bar library outputs
+# 3. Make a full nix export of those store paths (using e.g. `nix-store --export $(nix-store --query --requisites $barPath $fooPath) > foobar.closure`
+# 4. On the consumer machine, import the store paths (e.g. `nix-store --import foobar.closure` and then add gc roots)
+# 5. In the consumer haskell.nix build, add the imported store paths to your prebuilt-depends. E.g.:
+#
+#      prebuilt-depends = let foo = {
+#        # We need to make this look like a call to derivation for stdenv to do the right thing
+#        name = "foo-lib-foo-0.1.0.0";
+#        type = "derivation";
+#        outputs = [ "out" ];
+#        out = foo;
+#        all = [ foo ];
+#        # $fooPath is already in the store due to the import, so we use the storePath
+#        # builtin to add it as a source dependency to the build. Note that this does
+#        # not work in pure evaluation mode, you must use --impure with flakes. An
+#        # alternative would be to bundle up all of the needed libraries into tarballs that
+#        # are fetched and unpacked as proper fixed-output derivations.
+#        outPath = builtins.storePath $fooPath;
+#      }; in [ foo ]; # Could also do the same for bar of course
+, prebuilt-depends ? []
 }:
 # makeOverridable is called here after all the `? DEFAULT` arguments
 # will have been applied.  This makes sure that `c.override (oldAttrs: {...})`
@@ -160,6 +219,7 @@ let self =
 , enableTSanRTS
 , useLLVM
 , smallAddressSpace
+, prebuilt-depends
 }@drvArgs:
 
 let
@@ -213,7 +273,7 @@ let
   configFiles = makeConfigFiles {
     component = componentForSetup;
     inherit (package) identifier;
-    inherit fullName flags needsProfiling enableDWARF;
+    inherit fullName flags needsProfiling enableDWARF prebuilt-depends;
   };
 
   enableFeature = enable: feature:
@@ -852,5 +912,6 @@ in drv; in self) {
           enableDWARF
           enableTSanRTS
           useLLVM
-          smallAddressSpace;
+          smallAddressSpace
+          prebuilt-depends;
 }

builder/hspkg-builder.nix

@@ -58,6 +58,7 @@ let
     inherit allComponent componentId component package name src flags setup cabalFile cabal-generator patches
             shellHook
             ;
+    inherit (config) prebuilt-depends;
   };
 
 in rec {

builder/make-config-files.nix

@@ -1,6 +1,6 @@
 { stdenv, lib, haskellLib, ghc, nonReinstallablePkgs, runCommand, writeText, writeScript }@defaults:
 
-{ identifier, component, fullName, flags ? {}, needsProfiling ? false, enableDWARF ? false, chooseDrv ? drv: drv, nonReinstallablePkgs ? defaults.nonReinstallablePkgs }:
+{ identifier, component, fullName, flags ? {}, needsProfiling ? false, enableDWARF ? false, chooseDrv ? drv: drv, nonReinstallablePkgs ? defaults.nonReinstallablePkgs, prebuilt-depends ? [] }:
 
 let
   # Sort and remove duplicates from nonReinstallablePkgs.
@@ -57,7 +57,7 @@ let
       ((if needsProfiling then (x: map (p: p.profiled or p) x) else x: x)
       (map haskellLib.dependToLib component.depends))
     )
-  );
+  ) ++ prebuilt-depends;
   script = ''
     ${target-pkg} init $configFiles/${packageCfgDir}
 
@@ -149,17 +149,6 @@ let
       echo "allow-older: ${identifier.name}:*" >> $configFiles/cabal.config
     ''}
 
-    for p in "''${pkgsHostTarget[@]}"; do
-      if [ -e $p/envDep ]; then
-        cat $p/envDep >> $configFiles/ghc-environment
-      fi
-      ${ lib.optionalString component.doExactConfig ''
-        if [ -d $p/exactDep ]; then
-          cat $p/exactDep/configure-flags >> $configFiles/configure-flags
-          cat $p/exactDep/cabal.config >> $configFiles/cabal.config
-        fi
-      ''}
-    done
     for p in ${lib.concatStringsSep " " nonReinstallablePkgs'}; do
       if [ -e $ghcDeps/envDeps/$p ]; then
         cat $ghcDeps/envDeps/$p >> $configFiles/ghc-environment
@@ -173,6 +162,23 @@ let
       fi
     done
   ''
+  # We put the packages in buildInputs *after* the GHC deps on the command line
+  # to ensure that any prebuilt dependencies (see Note [prebuilt dependencies])
+  # take priority over the GHC-provided ones. We can't relink the prebuilt
+  # libraries, so this is the most likely to avoid conflicts.
+  + ''
+    for p in "''${pkgsHostTarget[@]}"; do
+      if [ -e $p/envDep ]; then
+        cat $p/envDep >> $configFiles/ghc-environment
+      fi
+      ${ lib.optionalString component.doExactConfig ''
+        if [ -d $p/exactDep ]; then
+          cat $p/exactDep/configure-flags >> $configFiles/configure-flags
+          cat $p/exactDep/cabal.config >> $configFiles/cabal.config
+        fi
+      ''}
+    done
+  ''
   # This code originates in the `generic-builder.nix` from nixpkgs.  However GHC has been fixed
   # to drop unused libraries referenced from libraries; and this patch is usually included in the
   # nixpkgs's GHC builds.  This doesn't sadly make this stupid hack unnecessary.  It resurfaces in

lib/call-cabal-project-to-nix.nix

@@ -62,6 +62,7 @@
 , evalPackages
 , supportHpack ? false      # Run hpack on package.yaml files with no .cabal file
 , ignorePackageYaml ? false # Ignore package.yaml files even if they exist
+, prebuilt-depends ? []
 , ...
 }@args:
 let
@@ -371,6 +372,7 @@ let
   };
 
   dummy-ghc-pkg-dump = evalPackages.runCommand "dummy-ghc-pkg-dump" {
+      buildInputs = prebuilt-depends;
       nativeBuildInputs = [
         evalPackages.haskell-nix.nix-tools-unchecked.exes.cabal2json
         evalPackages.jq
@@ -535,6 +537,15 @@ let
                 LAST_PKG="ghcjs-th"
               ''
           }
+          for l in "''${pkgsHostTarget[@]}"; do
+            if [ -d "$l/package.conf.d" ]; then
+              files=("$l/package.conf.d/"*.conf)
+              for file in "''${files[@]}"; do
+                 cat "$file" >> $out
+                 echo '---' >> $out
+              done
+            fi
+          done
           for pkg in $PKGS; do
             varname="$(echo $pkg | tr "-" "_")"
             ver="VER_$varname"

modules/cabal-project.nix

@@ -144,5 +144,14 @@ in {
       type = nullOr (listOf unspecified);
       default = [];
     };
+    prebuilt-depends = mkOption {
+      type = listOf package;
+      default = [];
+      description = ''
+        pre-built (perhaps proprietary) Haskell packages to make available as dependencies
+
+        See Note [prebuilt dependencies] for more details
+      '';
+    };
   };
 }

modules/component-driver.nix

@@ -37,6 +37,15 @@ in
     default = [];
     description = "pkgs to globally provide to Setup.hs builds";
   };
+  options.prebuilt-depends = lib.mkOption {
+    type = lib.types.listOf lib.types.package;
+    default = [];
+    description = ''
+      pre-built (perhaps proprietary) Haskell packages to make available as dependencies
+
+      See Note [prebuilt dependencies] for more details
+    '';
+  };
 
   # Dependencies (with reinstallable-lib:ghc)
   #

overlays/haskell.nix

@@ -699,6 +699,7 @@ final: prev: {
                         in if ghc.isHaskellNixCompiler or false then ghc.override { ghcEvalPackages = evalPackages; } else ghc;
                       compiler.nix-name = final.lib.mkForce config.compiler-nix-name;
                       evalPackages = final.lib.mkDefault evalPackages;
+                      inherit (config) prebuilt-depends;
                     } ];
                   extra-hackages = config.extra-hackages or [] ++ callProjectResults.extra-hackages;
                 };