From a54127fe22f2d629cd55a61feaf95abc9df2baad Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 21 Jul 2025 15:27:11 -0500 Subject: [PATCH 01/16] refresh build, update lambdaric --- .github/workflows/build.yml | 51 ++++++++++++++++++++++++++++++------- docker/Dockerfile.runner | 2 +- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ab5e168..ebfafe2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,13 +1,8 @@ name: Build on: - push: - pull_request: - workflow_dispatch: - schedule: - - cron: '40 7 * * 2' - - + push: + pull_request: jobs: @@ -17,7 +12,7 @@ jobs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - id: set-matrix - run: echo "::set-output name=matrix::{\"include\":[{\"type\":\"amd64\",\"image\":\"amazon/aws-lambda-provided:al2\", \"arch\":\"x86_64\"},{\"type\":\"arm64\",\"image\":\"amazon/aws-lambda-provided:al2.2023.12.14.13\", \"arch\":\"arm64\"}] }" + run: echo "::set-output name=matrix::{\"include\":[{\"type\":\"amd64\",\"image\":\"amazon/aws-lambda-provided:al2\", \"arch\":\"x86_64\"},{\"type\":\"arm64\",\"image\":\"amazon/aws-lambda-provided:al2.2025.07.17.11-arm64\", \"arch\":\"arm64\"}] }" containers: @@ -41,11 +36,24 @@ jobs: echo "are we pushing packages" ${{ env.PUSH_PACKAGES }} echo "event_name" ${{ github.event_name }} echo "ref" ${{ github.ref }} + + - name: Set up Docker + uses: docker/setup-docker-action@v4 + with: + daemon-config: | + { + "debug": false, + "features": { + "containerd-snapshotter": true + } + } + - name: Setup Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 with: version: latest + - if: ${{ env.PUSH_PACKAGES == 'true' }} name: Login to GitHub Container Registry uses: docker/login-action@v3 @@ -53,6 +61,7 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Prepare id: prep run: | @@ -62,8 +71,24 @@ jobs: fi echo ::set-output name=BUILD_DATE::$(date -u +'%Y-%m-%dT%H:%M:%SZ') echo ::set-output name=VERSION::${VERSION} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract container metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ghcr.io/pdal/pdal + docker.io/pdal/pdal + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + - name: Build image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: push: ${{ env.PUSH_PACKAGES == 'true' }} builder: ${{ steps.buildx.outputs.name }} @@ -85,3 +110,11 @@ jobs: org.opencontainers.image.version=${{ steps.prep.outputs.VERSION }} org.opencontainers.image.created=${{ steps.prep.outputs.BUILD_DATE }} + - name: Generate artifact attestation + if: ${{ env.PUSH_PACKAGES == 'true' }} + uses: actions/attest-build-provenance@v2 + with: + subject-name: ghcr.io/pdal/pdal + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: ${{ env.PUSH_PACKAGES == 'true' }} + diff --git a/docker/Dockerfile.runner b/docker/Dockerfile.runner index c4a8bdf..c3317e3 100644 --- a/docker/Dockerfile.runner +++ b/docker/Dockerfile.runner @@ -56,7 +56,7 @@ ENV PATH $PATH:${CONDAENV}/bin ENV LD_LIBRARY_PATH=${CONDAENV}/lib ENV HOME=/var/task/ -RUN /var/task/bin/python -m pip install awslambdaric==2.0.11 +RUN /var/task/bin/python -m pip install awslambdaric==3.1.1 ADD https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie-${RIE_ARCH} /usr/bin/aws-lambda-rie RUN chmod +x /usr/bin/aws-lambda-rie From edbb59ff30384d32c78dc042fd18a0b1201bcd59 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 21 Jul 2025 16:32:42 -0500 Subject: [PATCH 02/16] point to actual container location --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ebfafe2..0fb9ba3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -93,7 +93,7 @@ jobs: push: ${{ env.PUSH_PACKAGES == 'true' }} builder: ${{ steps.buildx.outputs.name }} context: . - file: ./Dockerfile + file: ./docker/Dockerfile.runner platforms: linux/${{ matrix.type}} build-args: | LAMBDA_IMAGE=${{ matrix.image }} From f7f0666956686321a12eb241922d9d8dd6a5345c Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 21 Jul 2025 16:37:18 -0500 Subject: [PATCH 03/16] build from ./docker context --- .github/workflows/build.yml | 2 +- docker/Dockerfile.runner | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0fb9ba3..6960fba 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -92,7 +92,7 @@ jobs: with: push: ${{ env.PUSH_PACKAGES == 'true' }} builder: ${{ steps.buildx.outputs.name }} - context: . + context: docker file: ./docker/Dockerfile.runner platforms: linux/${{ matrix.type}} build-args: | diff --git a/docker/Dockerfile.runner b/docker/Dockerfile.runner index c3317e3..912577b 100644 --- a/docker/Dockerfile.runner +++ b/docker/Dockerfile.runner @@ -1,7 +1,7 @@ ARG LAMBDA_IMAGE="amazon/aws-lambda-provided:al2" ARG RIE_ARCH="amd64" -FROM --platform=$TARGETPLATFORM condaforge/miniforge3:latest as condasetup +FROM condaforge/miniforge3:latest AS condasetup LABEL MAINTAINER="Howard Butler " ARG TARGETPLATFORM @@ -32,7 +32,7 @@ RUN conda-pack -n ${CONDA_ENV_NAME} --dest-prefix=/var/task -o /tmp/env.tar && \ rm /tmp/env.tar -FROM --platform=$TARGETPLATFORM ${LAMBDA_IMAGE:?} as al2 +FROM --platform=$TARGETPLATFORM ${LAMBDA_IMAGE:?} AS al2 ARG RIE_ARCH ARG LAMBDA_IMAGE From fbf2b3f5c06cd3c2458faea953040b9a810ae048 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 21 Jul 2025 17:59:23 -0500 Subject: [PATCH 04/16] swing again --- .github/workflows/build.yml | 2 +- docker/Dockerfile.runner | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6960fba..0fb9ba3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -92,7 +92,7 @@ jobs: with: push: ${{ env.PUSH_PACKAGES == 'true' }} builder: ${{ steps.buildx.outputs.name }} - context: docker + context: . file: ./docker/Dockerfile.runner platforms: linux/${{ matrix.type}} build-args: | diff --git a/docker/Dockerfile.runner b/docker/Dockerfile.runner index 912577b..6c36e25 100644 --- a/docker/Dockerfile.runner +++ b/docker/Dockerfile.runner @@ -13,8 +13,8 @@ RUN printf "I'm building for TARGETPLATFORM=${TARGETPLATFORM}" \ && printf "With uname -s : " && uname -s \ && printf "and uname -m : " && uname -mm -ENV CONDA_ENV_NAME "pdal" -ENV CONDAENV "/opt/conda/envs/${CONDA_ENV_NAME}" +ENV CONDA_ENV_NAME="pdal" +ENV CONDAENV="/opt/conda/envs/${CONDA_ENV_NAME}" # Create the environment: @@ -43,16 +43,16 @@ ENV TARGETARCH=${TARGETARCH:-amd64} -ENV CONDAENV "/var/task" -ENV CONDA_PREFIX "/var/task" -ENV TARGETPLATFORM "${TARGETPLATFORM}" +ENV CONDAENV="/var/task" +ENV CONDA_PREFIX="/var/task" +ENV TARGETPLATFORM="${TARGETPLATFORM}" COPY --from=condasetup /venv ${CONDAENV} -ENV PROJ_LIB ${CONDAENV}/share/proj +ENV PROJ_LIB=${CONDAENV}/share/proj ENV PROJ_NETWORK=TRUE -ENV PATH $PATH:${CONDAENV}/bin +ENV PATH=$PATH:${CONDAENV}/bin ENV LD_LIBRARY_PATH=${CONDAENV}/lib ENV HOME=/var/task/ @@ -64,6 +64,6 @@ RUN chmod +x /usr/bin/aws-lambda-rie WORKDIR /var/task COPY python-entry.sh ./ -COPY handlers/python/ /var/task/lib/python3.11/site-packages/pdal_lambda +COPY ./docker/handlers/python/ /var/task/lib/python3.11/site-packages/pdal_lambda COPY root-bashrc /root/.bashrc ENTRYPOINT [ "/var/task/python-entry.sh" ] From eff950b1bcaa814e34aed13d75cf9f185896c752 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 21 Jul 2025 18:05:12 -0500 Subject: [PATCH 05/16] pointer to file --- docker/Dockerfile.runner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.runner b/docker/Dockerfile.runner index 6c36e25..e5cc870 100644 --- a/docker/Dockerfile.runner +++ b/docker/Dockerfile.runner @@ -65,5 +65,5 @@ RUN chmod +x /usr/bin/aws-lambda-rie WORKDIR /var/task COPY python-entry.sh ./ COPY ./docker/handlers/python/ /var/task/lib/python3.11/site-packages/pdal_lambda -COPY root-bashrc /root/.bashrc +COPY ./docker/root-bashrc /root/.bashrc ENTRYPOINT [ "/var/task/python-entry.sh" ] From 0d39dc37e3bf22689e503dfe5dae02fa741af203 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 21 Jul 2025 18:14:20 -0500 Subject: [PATCH 06/16] some day we will succeed --- docker/Dockerfile.runner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.runner b/docker/Dockerfile.runner index e5cc870..aa71004 100644 --- a/docker/Dockerfile.runner +++ b/docker/Dockerfile.runner @@ -64,6 +64,6 @@ RUN chmod +x /usr/bin/aws-lambda-rie WORKDIR /var/task COPY python-entry.sh ./ -COPY ./docker/handlers/python/ /var/task/lib/python3.11/site-packages/pdal_lambda +COPY ./handlers/python/ /var/task/lib/python3.11/site-packages/pdal_lambda COPY ./docker/root-bashrc /root/.bashrc ENTRYPOINT [ "/var/task/python-entry.sh" ] From 9cb3d9ef89669c8d2a07a6efb7514581dcfa2022 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 21 Jul 2025 21:55:11 -0500 Subject: [PATCH 07/16] another path issue --- docker/Dockerfile.runner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.runner b/docker/Dockerfile.runner index aa71004..a1bb145 100644 --- a/docker/Dockerfile.runner +++ b/docker/Dockerfile.runner @@ -63,7 +63,7 @@ RUN chmod +x /usr/bin/aws-lambda-rie WORKDIR /var/task -COPY python-entry.sh ./ +COPY ./docker/python-entry.sh ./ COPY ./handlers/python/ /var/task/lib/python3.11/site-packages/pdal_lambda COPY ./docker/root-bashrc /root/.bashrc ENTRYPOINT [ "/var/task/python-entry.sh" ] From d197650e4d93aa8005fc5c07b96fe23ec4fe62aa Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 21 Jul 2025 22:04:25 -0500 Subject: [PATCH 08/16] more paths --- docker/Dockerfile.runner | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.runner b/docker/Dockerfile.runner index a1bb145..0fc0766 100644 --- a/docker/Dockerfile.runner +++ b/docker/Dockerfile.runner @@ -18,11 +18,11 @@ ENV CONDAENV="/opt/conda/envs/${CONDA_ENV_NAME}" # Create the environment: -COPY build-environment.yml . +COPY ./docker/build-environment.yml . RUN conda env create -f build-environment.yml RUN mamba update --all -y -COPY run-environment.yml . +COPY ./docker/run-environment.yml . RUN conda env create -f run-environment.yml From be63c18c7f0d24d0e4c882e273b2de3f057e8b2a Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 22 Jul 2025 08:21:17 -0500 Subject: [PATCH 09/16] need more permission --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0fb9ba3..9fbc5df 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,7 +19,10 @@ jobs: needs: config runs-on: ubuntu-latest permissions: + contents: read packages: write + attestations: write + id-token: write strategy: fail-fast: true matrix: ${{fromJson(needs.config.outputs.matrix)}} From 59010e5cb63c348b8367721a6b3e0272241522cb Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 22 Jul 2025 08:48:46 -0500 Subject: [PATCH 10/16] name push job --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9fbc5df..170611c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -92,6 +92,7 @@ jobs: - name: Build image uses: docker/build-push-action@v6 + id: push with: push: ${{ env.PUSH_PACKAGES == 'true' }} builder: ${{ steps.buildx.outputs.name }} From 927778a0de139d63ae1f0e5b491563a0017baa68 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 22 Jul 2025 09:07:11 -0500 Subject: [PATCH 11/16] remove set-output usage --- .github/workflows/build.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 170611c..8c6b2ed 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - id: set-matrix - run: echo "::set-output name=matrix::{\"include\":[{\"type\":\"amd64\",\"image\":\"amazon/aws-lambda-provided:al2\", \"arch\":\"x86_64\"},{\"type\":\"arm64\",\"image\":\"amazon/aws-lambda-provided:al2.2025.07.17.11-arm64\", \"arch\":\"arm64\"}] }" + run: echo "name=matrix::{\"include\":[{\"type\":\"amd64\",\"image\":\"amazon/aws-lambda-provided:al2\", \"arch\":\"x86_64\"},{\"type\":\"arm64\",\"image\":\"amazon/aws-lambda-provided:al2.2025.07.17.11-arm64\", \"arch\":\"arm64\"}] }" containers: @@ -72,8 +72,9 @@ jobs: if [[ $GITHUB_REF == refs/tags/* ]]; then VERSION=${GITHUB_REF/refs\/tags\//} fi - echo ::set-output name=BUILD_DATE::$(date -u +'%Y-%m-%dT%H:%M:%SZ') - echo ::set-output name=VERSION::${VERSION} + BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + echo "BUILD_DATE=$BUILD_DATE" >> $GITHUB_OUTPUT + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT # Extract metadata (tags, labels) for Docker # https://github.com/docker/metadata-action From 5f2a75b7101e1641a377c0f4a276f22436e1df58 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 22 Jul 2025 09:11:45 -0500 Subject: [PATCH 12/16] try matrix again --- .github/workflows/build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8c6b2ed..f50a50d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - id: set-matrix - run: echo "name=matrix::{\"include\":[{\"type\":\"amd64\",\"image\":\"amazon/aws-lambda-provided:al2\", \"arch\":\"x86_64\"},{\"type\":\"arm64\",\"image\":\"amazon/aws-lambda-provided:al2.2025.07.17.11-arm64\", \"arch\":\"arm64\"}] }" + run: echo "matrix={\"include\":[{\"type\":\"amd64\",\"image\":\"amazon/aws-lambda-provided:al2\", \"arch\":\"x86_64\"},{\"type\":\"arm64\",\"image\":\"amazon/aws-lambda-provided:al2.2025.07.17.11-arm64\", \"arch\":\"arm64\"}] }" >> $GITHUB_OUTPUT containers: @@ -84,7 +84,6 @@ jobs: with: images: | ghcr.io/pdal/pdal - docker.io/pdal/pdal tags: | type=ref,event=branch type=ref,event=pr From 4182efc78890c39a6678dd68351e772496c4beca Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 22 Jul 2025 09:17:52 -0500 Subject: [PATCH 13/16] upload to correct registry --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f50a50d..2e48a05 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -118,7 +118,7 @@ jobs: if: ${{ env.PUSH_PACKAGES == 'true' }} uses: actions/attest-build-provenance@v2 with: - subject-name: ghcr.io/pdal/pdal + subject-name: ghcr.io/pdal/lambda subject-digest: ${{ steps.push.outputs.digest }} push-to-registry: ${{ env.PUSH_PACKAGES == 'true' }} From 44c4ce492075664d48a9d6a7eec05452712dace4 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 22 Jul 2025 10:14:45 -0500 Subject: [PATCH 14/16] try setting annotations --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2e48a05..bb53f83 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -89,6 +89,9 @@ jobs: type=ref,event=pr type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index + - name: Build image uses: docker/build-push-action@v6 @@ -105,6 +108,7 @@ jobs: tags: | ghcr.io/pdal/lambda:${{ steps.prep.outputs.VERSION }} ghcr.io/pdal/lambda:latest + annotations: ${{ steps.meta.outputs.annotations }} labels: | org.opencontainers.image.title=${{ github.event.repository.name }} org.opencontainers.image.description=${{ github.event.repository.description }} From 8ecf8fe3a79dd9e00eeb701eedce6f2eb64c641e Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 22 Jul 2025 11:36:40 -0500 Subject: [PATCH 15/16] clean up merge --- .github/workflows/build.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1c49b15..d32b05c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - id: set-matrix - run: echo "::set-output name=matrix::{\"include\":[{\"type\":\"amd64\",\"image\":\"amazon/aws-lambda-provided:al2\", \"arch\":\"x86_64\"},{\"type\":\"arm64\",\"image\":\"amazon/aws-lambda-provided:al2.2025.07.17.11-arm64\", \"arch\":\"arm64\"}] }" + run: echo "name=matrix::{\"include\":[{\"type\":\"amd64\",\"image\":\"amazon/aws-lambda-provided:al2\", \"arch\":\"x86_64\"},{\"type\":\"arm64\",\"image\":\"amazon/aws-lambda-provided:al2.2025.07.17.11-arm64\", \"arch\":\"arm64\"}] }" >> $GITHUB_OUTPUT containers: needs: config @@ -93,14 +93,10 @@ jobs: env: DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index - - - name: Build image - uses: docker/build-push-action@v6 - id: push - - name: Build image uses: docker/build-push-action@v6 with: + id: push push: ${{ env.PUSH_PACKAGES == 'true' }} builder: ${{ steps.buildx.outputs.name }} context: . From f90548836a3250793e8994e39de7fa12b7554985 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 22 Jul 2025 11:40:25 -0500 Subject: [PATCH 16/16] fix matrix defn --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d32b05c..d36b9f7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - id: set-matrix - run: echo "name=matrix::{\"include\":[{\"type\":\"amd64\",\"image\":\"amazon/aws-lambda-provided:al2\", \"arch\":\"x86_64\"},{\"type\":\"arm64\",\"image\":\"amazon/aws-lambda-provided:al2.2025.07.17.11-arm64\", \"arch\":\"arm64\"}] }" >> $GITHUB_OUTPUT + run: echo "matrix={\"include\":[{\"type\":\"amd64\",\"image\":\"amazon/aws-lambda-provided:al2\", \"arch\":\"x86_64\"},{\"type\":\"arm64\",\"image\":\"amazon/aws-lambda-provided:al2.2025.07.17.11-arm64\", \"arch\":\"arm64\"}] }" >> $GITHUB_OUTPUT containers: needs: config