Cache xeus-python build (#87)

* Cache xeus-python build

* Add a script to build xeus_python only if the Dockerfile changed

* Iterate

* Iterate

* Prevent making two builds in the check-release workflow

check-manifest will not use our cache, it makes an isolated build, so
let's not build twice...

* Wrong file extension

* Trigger build

Author: martinRenou

.github/workflows/build.yml

@@ -47,14 +47,26 @@ jobs:
           restore-keys: |
             yarn-
 
+      - name: Setup xeus-python build cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            src/python_data.data
+            src/python_data.js
+            src/web_worker_kernel.ts
+            src/worker.ts
+            src/xpython_wasm.js
+            src/xpython_wasm.wasm
+            src/xpython_wasm.hash
+          key: ${{ hashFiles('Dockerfile') }}
+
       - name: Install dependencies
         run: python -m pip install -U jupyterlab~=3.1 check-manifest
+
       - name: Build the extension
         run: |
           set -eux
           jlpm
-          # build separately for now
-          jlpm run build:wasm
           jlpm run build
           jlpm run eslint:check
           python -m pip install .
@@ -64,8 +76,6 @@ jobs:
           # TODO: add JupyterLite browser check
           # python -m jupyterlab.browser_check
 
-          check-manifest -v
-
           pip install build
           python -m build --sdist
           cp dist/*.tar.gz jupyterlite-xeus-python.tar.gz

.github/workflows/check-release.yml

@@ -46,9 +46,6 @@ jobs:
       - name: Upgrade packaging dependencies
         run: |
           pip install --upgrade pip setuptools wheel jupyter-packaging~=0.10 --user
-      - name: Install Dependencies
-        run: |
-          pip install .
       - name: Check Release
         uses: jupyter-server/jupyter_releaser/.github/actions/check-release@v1
         with:

.gitignore

@@ -117,6 +117,10 @@ python_data.js
 python_data.data
 xpython_wasm.js
 xpython_wasm.wasm
+xpython_wasm.hash
+
+src/worker.ts
+src/web_worker_kernel.ts
 
 # Labextension
 share

build_xeus_python.js

@@ -0,0 +1,35 @@
+const { existsSync, readFileSync, writeFileSync } = require('fs');
+const crypto = require('crypto');
+const path = require('path');
+const { execSync } = require('child_process');
+
+const VERSION_FILE_NAME = 'xpython_wasm.hash';
+const VERSION_FILE_PATH = path.join('src', VERSION_FILE_NAME);
+
+const hashSum = crypto.createHash('sha256');
+
+hashSum.update(readFileSync('Dockerfile'));
+const buildHash = hashSum.digest('hex');
+
+let needsRebuild = true;
+
+if (existsSync(VERSION_FILE_PATH)) {
+  const currentHash = readFileSync(VERSION_FILE_PATH, {
+    encoding: 'utf8',
+    flag: 'r'
+  });
+
+  if (currentHash === buildHash) {
+    needsRebuild = false;
+  }
+}
+
+if (needsRebuild) {
+  execSync('docker build -t jupyterlite-xeus-kernel .');
+
+  execSync(
+    'docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) jupyterlite-xeus-kernel copy_output.sh'
+  );
+
+  writeFileSync(VERSION_FILE_PATH, buildHash);
+}

package.json

@@ -29,19 +29,15 @@
     "url": "https://github.com/jupyterlite/xeus-python-kernel.git"
   },
   "scripts": {
-    "build:dockerimage": "docker build -t jupyterlite-xeus-kernel . ",
-    "build:dockerimage_no_cache": "docker build -t jupyterlite-xeus-kernel --no-cache . ",
-    "build:emscripten": "echo $(pwd) && docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) jupyterlite-xeus-kernel copy_output.sh",
     "copy-files": "copyfiles -u 1 src/xpython_wasm.wasm src/xpython_wasm.js src/python_data.js src/python_data.data lib",
-    "build:wasm": "jlpm run build:dockerimage && jlpm run build:emscripten",
-    "build:wasm_no_cache": "jlpm run build:dockerimage_no_cache && jlpm run build:emscripten",
     "build:worker": "webpack --config worker.webpack.config.js --mode=development",
+    "build": "jlpm run build:dockerimage && jlpm run build:lib && jlpm run build:worker && jlpm run copy-files && jlpm run build:labextension:dev",
+    "build:dockerimage": "node build_xeus_python.js",
     "build:worker:prod": "webpack --config worker.webpack.config.js --mode=production",
-    "build": "jlpm run build:lib && jlpm run build:worker && jlpm run copy-files && jlpm run build:labextension:dev",
-    "build:prod": "jlpm run clean && jlpm run build:wasm && jlpm run build:lib && jlpm run build:worker:prod && jlpm run copy-files && jlpm run build:labextension",
+    "build:prod": "jlpm run clean && jlpm run build build:dockerimage && jlpm run build:lib && jlpm run build:worker:prod && jlpm run copy-files && jlpm run build:labextension",
     "build:labextension": "jupyter labextension build .",
     "build:labextension:dev": "jupyter labextension build --development True .",
-    "build:lib": " jlpm run sed-worker && tsc",
+    "build:lib": "jlpm run sed-worker && tsc",
     "sed-worker": "shx sed -i \"s/XEUS_KERNEL_FILE/'.\\/xpython_wasm.js'/\" src/worker.ts && shx sed -i \"s/LANGUAGE_DATA_FILE/'.\\/python_data.js'/\" src/worker.ts ",
     "clean": "jlpm run clean:lib",
     "clean:lib": "rimraf lib tsconfig.tsbuildinfo",

yarn.lock

@@ -2433,6 +2433,18 @@ glob@^6.0.1:
     once "^1.3.0"
     path-is-absolute "^1.0.0"
 
+glob@^7.0.0:
+  version "7.2.3"
+  resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b"
+  integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==
+  dependencies:
+    fs.realpath "^1.0.0"
+    inflight "^1.0.4"
+    inherits "2"
+    minimatch "^3.1.1"
+    once "^1.3.0"
+    path-is-absolute "^1.0.0"
+
 glob@^7.0.5, glob@^7.1.3, glob@^7.1.4:
   version "7.2.0"
   resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023"
@@ -2755,6 +2767,11 @@ internal-slot@^1.0.3:
     has "^1.0.3"
     side-channel "^1.0.4"
 
+interpret@^1.0.0:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.4.0.tgz#665ab8bc4da27a774a40584e812e3e0fa45b1a1e"
+  integrity sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==
+
 interpret@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/interpret/-/interpret-2.2.0.tgz#1a78a0b5965c40a5416d007ad6f50ad27c417df9"
@@ -2802,6 +2819,13 @@ is-core-module@^2.8.1:
   dependencies:
     has "^1.0.3"
 
+is-core-module@^2.9.0:
+  version "2.11.0"
+  resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.11.0.tgz#ad4cb3e3863e814523c96f3f58d26cc570ff0144"
+  integrity sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw==
+  dependencies:
+    has "^1.0.3"
+
 is-date-object@^1.0.1:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.5.tgz#0841d5536e724c25597bf6ea62e1bd38298df31f"
@@ -3448,7 +3472,7 @@ mini-css-extract-plugin@~1.3.2:
     schema-utils "^3.0.0"
     webpack-sources "^1.1.0"
 
-"minimatch@2 || 3", minimatch@^3.0.3, minimatch@^3.0.4:
+"minimatch@2 || 3", minimatch@^3.0.3, minimatch@^3.0.4, minimatch@^3.1.1:
   version "3.1.2"
   resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"
   integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==
@@ -3474,6 +3498,11 @@ minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6, minimist@~1.2.0:
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44"
   integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==
 
+minimist@^1.2.3:
+  version "1.2.7"
+  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.7.tgz#daa1c4d91f507390437c6a8bc01078e7000c4d18"
+  integrity sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==
+
 minipass-collect@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/minipass-collect/-/minipass-collect-1.0.2.tgz#22b813bf745dc6edba2576b940022ad6edc8c617"
@@ -4209,6 +4238,13 @@ readable-stream@~2.3.6:
     string_decoder "~1.1.1"
     util-deprecate "~1.0.1"
 
+rechoir@^0.6.2:
+  version "0.6.2"
+  resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384"
+  integrity sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==
+  dependencies:
+    resolve "^1.1.6"
+
 rechoir@^0.7.0:
   version "0.7.1"
   resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.7.1.tgz#9478a96a1ca135b5e88fc027f03ee92d6c645686"
@@ -4344,6 +4380,15 @@ resolve-from@^5.0.0:
   resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-5.0.0.tgz#c35225843df8f776df21c57557bc087e9dfdfc69"
   integrity sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==
 
+resolve@^1.1.6:
+  version "1.22.1"
+  resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.1.tgz#27cb2ebb53f91abb49470a928bba7558066ac177"
+  integrity sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw==
+  dependencies:
+    is-core-module "^2.9.0"
+    path-parse "^1.0.7"
+    supports-preserve-symlinks-flag "^1.0.0"
+
 resolve@^1.10.0, resolve@^1.20.0, resolve@^1.9.0:
   version "1.22.0"
   resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.0.tgz#5e0b8c67c15df57a89bdbabe603a002f21731198"
@@ -4576,6 +4621,23 @@ shell-quote@^1.6.1:
   resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.7.3.tgz#aa40edac170445b9a431e17bb62c0b881b9c4123"
   integrity sha512-Vpfqwm4EnqGdlsBFNmHhxhElJYrdfcxPThu+ryKS5J8L/fhAwLazFZtq+S+TWZ9ANj2piSQLGj6NQg+lKPmxrw==
 
+shelljs@^0.8.5:
+  version "0.8.5"
+  resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.8.5.tgz#de055408d8361bed66c669d2f000538ced8ee20c"
+  integrity sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==
+  dependencies:
+    glob "^7.0.0"
+    interpret "^1.0.0"
+    rechoir "^0.6.2"
+
+shx@^0.3.0:
+  version "0.3.4"
+  resolved "https://registry.yarnpkg.com/shx/-/shx-0.3.4.tgz#74289230b4b663979167f94e1935901406e40f02"
+  integrity sha512-N6A9MLVqjxZYcVn8hLmtneQWIJtp8IKzMP4eMnx+nqkvXoqinUPCbUFLp2UcWTEIUONhlk0ewxr/jaVGlc+J+g==
+  dependencies:
+    minimist "^1.2.3"
+    shelljs "^0.8.5"
+
 side-channel@^1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf"