Skip to content

Commit c71fcd2

Browse files
hasufellhasufell
committed
Parallelize GHC builds
1 parent e09f540 commit c71fcd2

File tree

16 files changed

+779
-383
lines changed

16 files changed

+779
-383
lines changed

‎.cirrus.yml‎

Lines changed: 73 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ compute_engine_instance:
66
disk: 100 # Gb
77

88
build_task:
9-
name: build
109
timeout_in: 120m
1110
only_if: $CIRRUS_TAG != ''
1211
env:
@@ -20,59 +19,83 @@ build_task:
2019
RUNNER_OS: "FreeBSD"
2120
ADD_CABAL_ARGS: "--enable-split-sections"
2221
GITHUB_WORKSPACE: ${CIRRUS_WORKING_DIR}
22+
CABAL_CACHE_NONFATAL: "yes"
23+
matrix:
24+
- name: build-ghc-8.10.7
25+
env:
26+
GHC_VERSION: 8.10.7
27+
- name: build-ghc-9.0.2
28+
env:
29+
GHC_VERSION: 9.0.2
30+
- name: build-ghc-9.2.5
31+
env:
32+
GHC_VERSION: 9.2.5
2333
install_script: pkg install -y hs-cabal-install git bash misc/compat10x misc/compat11x misc/compat12x gmake patchelf tree
2434
script:
2535
- tzsetup Etc/GMT
2636
- adjkerntz -a
2737
- bash .github/scripts/build.sh
28-
- bash .github/scripts/test.sh
29-
# binaries_cache: &binaries_cache
30-
# folder: out
31-
# fingerprint_script:
32-
# - ghc --numeric-version
33-
# - cat cabal.project
34-
# - cat haskell-language-server.cabal
35-
# - cat dist-newstyle/cache/plan.json
38+
- tar caf out.tar.xz out/ store/
3639
binaries_artifacts:
37-
path: "out/*"
40+
path: "out.tar.xz"
41+
42+
43+
bindist_task:
44+
name: bindist
45+
depends_on:
46+
- build-ghc-8.10.7
47+
- build-ghc-9.0.2
48+
- build-ghc-9.2.5
49+
timeout_in: 120m
50+
only_if: $CIRRUS_TAG != ''
51+
env:
52+
TARBALL_EXT: "tar.xz"
53+
ARCH: 64
54+
ARTIFACT: "x86_64-freebsd"
55+
DISTRO: "na"
56+
RUNNER_OS: "FreeBSD"
57+
GITHUB_WORKSPACE: ${CIRRUS_WORKING_DIR}
58+
install_script: pkg install -y hs-cabal-install git bash misc/compat10x misc/compat11x misc/compat12x gmake patchelf tree unzip
59+
script:
60+
- tzsetup Etc/GMT
61+
- adjkerntz -a
62+
63+
- curl -o binaries-8.10.7.tar.xz -L https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID}/build-ghc-8.10.7/binaries/out.tar.xz
64+
- tar xvf binaries-8.10.7.tar.xz
65+
- rm -f binaries-8.10.7.tar.xz
66+
67+
- curl -o binaries-9.0.2.tar.xz -L https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID}/build-ghc-9.0.2/binaries/out.tar.xz
68+
- tar xvf binaries-9.0.2.tar.xz
69+
- rm -f binaries-9.0.2.tar.xz
70+
71+
- curl -o binaries-9.2.5.tar.xz -L https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID}/build-ghc-9.2.5/binaries/out.tar.xz
72+
- tar xvf binaries-9.2.5.tar.xz
73+
- rm -f binaries-9.2.5.tar.xz
74+
75+
- bash .github/scripts/bindist.sh
76+
bindist_artifacts:
77+
path: "./out/*.tar.xz"
78+
79+
test_task:
80+
name: test
81+
depends_on:
82+
- bindist
83+
timeout_in: 120m
84+
only_if: $CIRRUS_TAG != ''
85+
env:
86+
TARBALL_EXT: "tar.xz"
87+
ARCH: 64
88+
ARTIFACT: "x86_64-freebsd"
89+
DISTRO: "na"
90+
RUNNER_OS: "FreeBSD"
91+
GITHUB_WORKSPACE: ${CIRRUS_WORKING_DIR}
92+
install_script: pkg install -y hs-cabal-install git bash misc/compat10x misc/compat11x misc/compat12x gmake patchelf tree unzip
93+
script:
94+
- tzsetup Etc/GMT
95+
- adjkerntz -a
96+
97+
- curl -O -L https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID}/bindist/bindist.zip
98+
- unzip bindist.zip
99+
100+
- bash .github/scripts/test.sh
38101

39-
# release_task:
40-
# name: "Release"
41-
# only_if: $CIRRUS_TAG != ''
42-
# depends_on: build
43-
# env:
44-
# GITHUB_TOKEN: ENCRYPTED[9d9b54424cb18abc9067436c729a77e0486805eff1903bc5b2c591696850f8db7cebac3f29cfa3b119ebb6fd2e98a839]
45-
# install_script: pkg install -y curl bash jq
46-
# binaries_cache: *binaries_cache
47-
# upload_script: |
48-
# #!/bin/bash
49-
#
50-
# if [[ "$CIRRUS_RELEASE" == "" ]]; then
51-
# NUM_TRIES=0
52-
# until [ $NUM_TRIES -eq 20 ]
53-
# do
54-
# echo "Retrying to find a release associated with this tag"
55-
# CIRRUS_RELEASE=$(curl -sL https://api.github.com/repos/$CIRRUS_REPO_FULL_NAME/releases/tags/$CIRRUS_TAG | jq -r '.id')
56-
# [[ "$CIRRUS_RELEASE" != "null" ]] && break
57-
# NUM_TRIES=$((NUM_TRIES+1))
58-
# sleep 30
59-
# done
60-
# fi
61-
#
62-
# if [[ "$GITHUB_TOKEN" == "" ]]; then
63-
# echo "Please provide GitHub access token via GITHUB_TOKEN environment variable!"
64-
# exit 1
65-
# fi
66-
#
67-
# for fpath in out/*
68-
# do
69-
# echo "Uploading $fpath..."
70-
# name=$(basename "$fpath")
71-
# url_to_upload="https://uploads.github.com/repos/$CIRRUS_REPO_FULL_NAME/releases/$CIRRUS_RELEASE/assets?name=$name"
72-
# echo "Uploading to $url_to_upload"
73-
# curl -X POST \
74-
# --data-binary @$fpath \
75-
# --header "Authorization: token $GITHUB_TOKEN" \
76-
# --header "Content-Type: application/x-xz-compressed-tar" \
77-
# $url_to_upload
78-
# done

‎.github/scripts/bindist.sh‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/bin/bash
2+
3+
set -eux
4+
5+
. .github/scripts/env.sh
6+
. .github/scripts/common.sh
7+
8+
# ensure ghcup
9+
if ! command -v ghcup ; then
10+
install_ghcup
11+
fi
12+
13+
# create tarball/zip
14+
case "${TARBALL_EXT}" in
15+
zip)
16+
HLS_VERSION="$(grep '^version:' haskell-language-server.cabal | awk '{ print $2 }')"
17+
(
18+
cd "$CI_PROJECT_DIR/out/${ARTIFACT}"
19+
zip "$CI_PROJECT_DIR/out/haskell-language-server-${HLS_VERSION}-${ARTIFACT}.zip" haskell-language-server-*
20+
)
21+
;;
22+
tar.xz)
23+
GHCS="$(cd "$CI_PROJECT_DIR/out/${ARTIFACT}" && rm -f ./*.json && for ghc in * ; do printf "%s " "$ghc" ; done)"
24+
emake --version
25+
emake GHCUP=ghcup ARTIFACT="${ARTIFACT}" GHCS="${GHCS}" bindist
26+
emake GHCUP=ghcup ARTIFACT="${ARTIFACT}" bindist-tar
27+
emake GHCUP=ghcup GHCS="${GHCS}" clean-ghcs
28+
;;
29+
*)
30+
fail "Unknown TARBALL_EXT: ${TARBALL_EXT}"
31+
;;
32+
esac

‎.github/scripts/brew.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,6 @@ export HOMEBREW_LOGS=$CI_PROJECT_DIR/.brew_logs
2222
mkdir -p /private/tmp/.brew_tmp
2323
export HOMEBREW_TEMP=/private/tmp/.brew_tmp
2424

25-
brew update
25+
#brew update
2626
brew install ${1+"$@"}
2727

‎.github/scripts/build.sh‎

Lines changed: 25 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -22,75 +22,40 @@ download_cabal_cache "$HOME/.local/bin/cabal-cache"
2222

2323
# build
2424
ecabal update
25+
ghcup install ghc "${GHC_VERSION}"
26+
ghcup set ghc "${GHC_VERSION}"
27+
"ghc-${GHC_VERSION}" --info
28+
"ghc" --info
2529

26-
mkdir -p "$CI_PROJECT_DIR/plan.json"
27-
mkdir -p "$CI_PROJECT_DIR/out"
30+
mkdir -p "$CI_PROJECT_DIR/out/${ARTIFACT}"
31+
mkdir -p "$CI_PROJECT_DIR/out/plan.json"
2832

2933
case "$(uname)" in
3034
MSYS_*|MINGW*)
31-
for ghc in $(cat bindist/ghcs-Msys) ; do
32-
GHC_VERSION="$(echo "${ghc}" | tr -d '\r')"
33-
args=( -O2 -w "ghc-$GHC_VERSION" --project-file cabal.project --disable-profiling --disable-tests --enable-executable-stripping ${ADD_CABAL_ARGS})
34-
ghcup install ghc "${GHC_VERSION}"
35-
ghcup set ghc "${GHC_VERSION}"
36-
"ghc-${GHC_VERSION}" --info
37-
"ghc" --info
38-
# Shorten binary names
39-
# due to MAX_PATH issues on windows
40-
sed -i.bak -e 's/haskell-language-server/hls/g' \
41-
-e 's/haskell_language_server/hls/g' \
42-
haskell-language-server.cabal cabal.project
43-
sed -i.bak -e 's/Paths_haskell_language_server/Paths_hls/g' \
44-
src/**/*.hs exe/*.hs
45-
46-
47-
# shellcheck disable=SC2068
48-
build_with_cache ${args[@]} exe:hls exe:hls-wrapper
49-
cp dist-newstyle/cache/plan.json "$CI_PROJECT_DIR/plan.json/${ARTIFACT}-ghc-${GHC_VERSION}-plan.json"
50-
51-
# shellcheck disable=SC2068
52-
cp "$(cabal list-bin -v0 ${args[@]} exe:hls)" "$CI_PROJECT_DIR/out/haskell-language-server-${GHC_VERSION}"${ext}
53-
# shellcheck disable=SC2068
54-
cp "$(cabal list-bin -v0 ${args[@]} exe:hls-wrapper)" "$CI_PROJECT_DIR/out/haskell-language-server-wrapper"${ext}
55-
ghcup rm ghc "${GHC_VERSION}"
56-
done
35+
args=( -O2 -w "ghc-$GHC_VERSION" --project-file cabal.project --disable-profiling --disable-tests --enable-executable-stripping ${ADD_CABAL_ARGS})
36+
37+
# Shorten binary names
38+
# due to MAX_PATH issues on windows
39+
sed -i.bak -e 's/haskell-language-server/hls/g' \
40+
-e 's/haskell_language_server/hls/g' \
41+
haskell-language-server.cabal cabal.project
42+
sed -i.bak -e 's/Paths_haskell_language_server/Paths_hls/g' \
43+
src/**/*.hs exe/*.hs
44+
45+
# shellcheck disable=SC2068
46+
build_with_cache ${args[@]} exe:hls exe:hls-wrapper
47+
cp dist-newstyle/cache/plan.json "$CI_PROJECT_DIR/out/plan.json/${ARTIFACT}-ghc-${GHC_VERSION}-plan.json"
48+
49+
# shellcheck disable=SC2068
50+
cp "$(cabal list-bin -v0 ${args[@]} exe:hls)" "$CI_PROJECT_DIR/out/${ARTIFACT}/haskell-language-server-${GHC_VERSION}${ext}"
51+
# shellcheck disable=SC2068
52+
cp "$(cabal list-bin -v0 ${args[@]} exe:hls-wrapper)" "$CI_PROJECT_DIR/out/${ARTIFACT}/haskell-language-server-wrapper${ext}"
5753
;;
5854
*)
5955
sed -i.bak -e '/DELETE MARKER FOR CI/,/END DELETE/d' cabal.project # see comment in cabal.project
6056
emake --version
61-
emake GHCUP=ghcup CABAL_CACHE_BIN=cabal-cache.sh S3_HOST="${S3_HOST}" S3_KEY="${ARTIFACT}" hls
62-
emake GHCUP=ghcup bindist
63-
(
64-
cd out/
65-
for pj in *plan.json ; do
66-
mv "${pj}" "$CI_PROJECT_DIR/plan.json/${ARTIFACT}-${pj}"
67-
done
68-
)
69-
rm -rf out/*.*.*
57+
emake GHCUP=ghcup CABAL_CACHE_BIN=cabal-cache.sh S3_HOST="${S3_HOST}" S3_KEY="${ARTIFACT}" GHC_VERSION="${GHC_VERSION}" hls-ghc
7058
;;
7159
esac
7260

73-
# create tarball/zip
74-
TARBALL_PREFIX="haskell-language-server"
75-
case "${TARBALL_EXT}" in
76-
zip)
77-
HLS_VERSION="$("$CI_PROJECT_DIR/out/haskell-language-server-wrapper" --numeric-version)"
78-
(
79-
cd "$CI_PROJECT_DIR/out/"
80-
zip "${TARBALL_PREFIX}-${HLS_VERSION}-${ARTIFACT}.zip" haskell-language-server-*
81-
find . -mindepth 1 -maxdepth 1 \! -name '*.zip' -exec rm -rf '{}' \;
82-
)
83-
;;
84-
tar.xz)
85-
emake --version
86-
HLS_VERSION="$(emake -s -C out/bindist/haskell-language-server-* version)"
87-
emake TARBALL="${TARBALL_PREFIX}-${HLS_VERSION}-${ARTIFACT}.tar.xz" bindist-tar
88-
emake GHCUP=ghcup clean-ghcs
89-
find out -mindepth 1 -maxdepth 1 \! -name '*.tar.xz' -exec rm -rf '{}' \;
90-
;;
91-
*)
92-
fail "Unknown TARBALL_EXT: ${TARBALL_EXT}"
93-
;;
94-
esac
9561

96-
mv "$CI_PROJECT_DIR"/plan.json/* out/

‎.github/scripts/cabal-cache.sh‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/bin/sh
1+
#!/usr/bin/env bash
22

33
case "$(uname -s)" in
44
MSYS_*|MINGW*)
@@ -9,11 +9,12 @@ case "$(uname -s)" in
99
;;
1010
esac
1111

12-
if [ -n "${CABAL_CACHE_DISABLE}" ] ; then
12+
if [ "${CABAL_CACHE_DISABLE}" = "yes" ] ; then
1313
echo "cabal-cache disabled (CABAL_CACHE_DISABLE set)"
14-
elif [ -n "${CABAL_CACHE_NONFATAL}" ] ; then
15-
"cabal-cache${ext}" "$@" || echo "cabal-cache failed (CABAL_CACHE_NONFATAL set)"
14+
elif [ "${CABAL_CACHE_NONFATAL}" = "yes" ] ; then
15+
time "cabal-cache${ext}" "$@" || echo "cabal-cache failed (CABAL_CACHE_NONFATAL set)"
1616
else
17-
exec "cabal-cache${ext}" "$@"
17+
time "cabal-cache${ext}" "$@"
18+
exit $?
1819
fi
1920

‎.github/scripts/common.sh‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,6 @@ install_ghcup() {
139139
else
140140
curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | BOOTSTRAP_HASKELL_MINIMAL=1 sh
141141
source "$(dirname "${GHCUP_BIN}")/env"
142-
ghcup install ghc --set "${BOOTSTRAP_HASKELL_GHC_VERSION}"
143142
ghcup install cabal --set "${BOOTSTRAP_HASKELL_CABAL_VERSION}"
144143
fi
145144
}

‎.github/scripts/env.sh‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ fi
1111
export PATH="$HOME/.local/bin:$PATH"
1212

1313
export BOOTSTRAP_HASKELL_NONINTERACTIVE=1
14-
export BOOTSTRAP_HASKELL_GHC_VERSION="${GHC_VER:-recommended}"
1514
export BOOTSTRAP_HASKELL_CABAL_VERSION="${CABAL_VER:-recommended}"
1615
export BOOTSTRAP_HASKELL_ADJUST_CABAL_CONFIG=yes
1716
export BOOTSTRAP_HASKELL_INSTALL_NO_STACK=yes

‎.github/scripts/tar.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
3+
set -eux
4+
5+
. .github/scripts/env.sh
6+
7+
ls -lah
8+
ls -lah out/
9+
ls -lah store/
10+
11+
tar cvf "out-${ARTIFACT}-${GHC_VERSION}.tar" out/ store/

‎.github/scripts/untar.sh‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
3+
set -eux
4+
5+
for bindist in out-*.tar ; do
6+
tar xf "${bindist}"
7+
done

0 commit comments

Comments
 (0)