From 9a2d6dab1c9ca6c22ab095a93e6345169d87b190 Mon Sep 17 00:00:00 2001 From: Moe Mahhouk Date: Mon, 16 Jun 2025 17:45:20 +0000 Subject: [PATCH 01/33] chore: refactor reproducible builds --- Cargo.toml | 7 ----- Dockerfile.reproducible | 35 ++++++----------------- Makefile | 61 ++++++++++++++++++++++++++++++----------- 3 files changed, 53 insertions(+), 50 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4850b2f56c4..d044b2d20a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -286,12 +286,5 @@ lto = "fat" codegen-units = 1 incremental = false -[profile.reproducible] -inherits = "release" -debug = false -panic = "abort" -codegen-units = 1 -overflow-checks = true - [patch.crates-io] quick-protobuf = { git = "https://github.com/sigp/quick-protobuf.git", rev = "681f413312404ab6e51f0b46f39b0075c6f4ebfd" } diff --git a/Dockerfile.reproducible b/Dockerfile.reproducible index df57616874d..f8f26f1b32e 100644 --- a/Dockerfile.reproducible +++ b/Dockerfile.reproducible @@ -1,44 +1,25 @@ -# Define the Rust image as an argument with a default to x86_64 Rust 1.82 image based on Debian Bullseye -ARG RUST_IMAGE="rust:1.82-bullseye@sha256:ac7fe7b0c9429313c0fe87d3a8993998d1fe2be9e3e91b5e2ec05d3a09d87128" +ARG RUST_IMAGE="rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1" + FROM ${RUST_IMAGE} AS builder # Install specific version of the build dependencies RUN apt-get update && apt-get install -y libclang-dev=1:11.0-51+nmu5 cmake=3.18.4-2+deb11u1 -# Add target architecture argument with default value ARG RUST_TARGET="x86_64-unknown-linux-gnu" # Copy the project to the container -COPY . /app +COPY ./ /app WORKDIR /app -# Get the latest commit timestamp and set SOURCE_DATE_EPOCH (default it to 0 if not passed) -ARG SOURCE_DATE=0 - -# Set environment variables for reproducibility -ARG RUSTFLAGS="-C link-arg=-Wl,--build-id=none -C metadata='' --remap-path-prefix $(pwd)=." -ENV SOURCE_DATE_EPOCH=$SOURCE_DATE \ - CARGO_INCREMENTAL=0 \ - LC_ALL=C \ - TZ=UTC \ - RUSTFLAGS="${RUSTFLAGS}" - -# Set the default features if not provided -ARG FEATURES="gnosis,slasher-lmdb,slasher-mdbx,slasher-redb,jemalloc" - -# Set the default profile if not provided -ARG PROFILE="reproducible" - # Build the project with the reproducible settings -RUN cargo build --bin lighthouse \ - --features "${FEATURES}" \ - --profile "${PROFILE}" \ - --locked \ - --target "${RUST_TARGET}" +RUN make build-reproducible -RUN mv /app/target/${RUST_TARGET}/${PROFILE}/lighthouse /lighthouse +# Move the binary to a standard location +RUN mv /app/target/${RUST_TARGET}/release/lighthouse /lighthouse # Create a minimal final image with just the binary FROM gcr.io/distroless/cc-debian12:nonroot-6755e21ccd99ddead6edc8106ba03888cbeed41a COPY --from=builder /lighthouse /lighthouse + +EXPOSE 30303 30303/udp 9001 8545 8546 ENTRYPOINT [ "/lighthouse" ] diff --git a/Makefile b/Makefile index 75b6811b743..a58dcfc4f90 100644 --- a/Makefile +++ b/Makefile @@ -82,36 +82,65 @@ build-lcli-aarch64: build-lcli-riscv64: cross build --bin lcli --target riscv64gc-unknown-linux-gnu --features "portable" --profile "$(CROSS_PROFILE)" --locked -# extracts the current source date for reproducible builds -SOURCE_DATE := $(shell git log -1 --pretty=%ct) +# Environment variables for reproducible builds +# Initialize RUSTFLAGS +RUST_BUILD_FLAGS = +# Remove build ID from the binary to ensure reproducibility across builds +RUST_BUILD_FLAGS += -C link-arg=-Wl,--build-id=none +# Remove metadata hash from symbol names to ensure reproducible builds +RUST_BUILD_FLAGS += -C metadata='' + +# Set timestamp from last git commit for reproducible builds +SOURCE_DATE ?= $(shell git log -1 --pretty=%ct) + +# Disable incremental compilation to avoid non-deterministic artifacts +CARGO_INCREMENTAL_VAL = 0 +# Set C locale for consistent string handling and sorting +LOCALE_VAL = C +# Set UTC timezone for consistent time handling across builds +TZ_VAL = UTC + +# Default features for lighthouse +FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx,slasher-redb,jemalloc + +# Default profile +PROFILE ?= release + +# Default target architecture +RUST_TARGET ?= x86_64-unknown-linux-gnu + +# Default images for different architectures +RUST_IMAGE_AMD64 ?= rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1 +RUST_IMAGE_ARM64 ?= rust:1.86-bullseye@sha256:36053eabadeb701e3e0406610a2ce72ccfa10b7828963cd08cffdcf660518b27 -# Default image for x86_64 -RUST_IMAGE_AMD64 ?= rust:1.82-bullseye@sha256:ac7fe7b0c9429313c0fe87d3a8993998d1fe2be9e3e91b5e2ec05d3a09d87128 +.PHONY: build-reproducible +build-reproducible: ## Build the lighthouse binary into `target` directory with reproducible builds + SOURCE_DATE_EPOCH=$(SOURCE_DATE) \ + RUSTFLAGS="${RUST_BUILD_FLAGS} --remap-path-prefix $$(pwd)=." \ + CARGO_INCREMENTAL=${CARGO_INCREMENTAL_VAL} \ + LC_ALL=${LOCALE_VAL} \ + TZ=${TZ_VAL} \ + cargo build --bin lighthouse --features "$(FEATURES)" --profile "$(PROFILE)" --locked --target $(RUST_TARGET) -# Reproducible build for x86_64 -build-reproducible-x86_64: +.PHONY: build-reproducible-x86_64 +build-reproducible-x86_64: ## Build reproducible x86_64 Docker image DOCKER_BUILDKIT=1 docker build \ --build-arg RUST_TARGET="x86_64-unknown-linux-gnu" \ --build-arg RUST_IMAGE=$(RUST_IMAGE_AMD64) \ - --build-arg SOURCE_DATE=$(SOURCE_DATE) \ - -f Dockerfile.reproducible \ + -f Dockerfile.test \ -t lighthouse:reproducible-amd64 . -# Default image for arm64 -RUST_IMAGE_ARM64 ?= rust:1.82-bullseye@sha256:3c1b8b6487513ad4e753d008b960260f5bcc81bf110883460f6ed3cd72bf439b - -# Reproducible build for aarch64 -build-reproducible-aarch64: +.PHONY: build-reproducible-aarch64 +build-reproducible-aarch64: ## Build reproducible aarch64 Docker image DOCKER_BUILDKIT=1 docker build \ --platform linux/arm64 \ --build-arg RUST_TARGET="aarch64-unknown-linux-gnu" \ --build-arg RUST_IMAGE=$(RUST_IMAGE_ARM64) \ - --build-arg SOURCE_DATE=$(SOURCE_DATE) \ -f Dockerfile.reproducible \ -t lighthouse:reproducible-arm64 . -# Build both architectures -build-reproducible-all: build-reproducible-x86_64 build-reproducible-aarch64 +.PHONY: build-reproducible-all +build-reproducible-all: build-reproducible-x86_64 build-reproducible-aarch64 ## Build both x86_64 and aarch64 reproducible Docker images # Create a `.tar.gz` containing a binary for a specific target. define tarball_release_binary From 10a3b59790945de0416c7117e87f7c112c7fcc49 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Mon, 16 Jun 2025 22:00:31 +0000 Subject: [PATCH 02/33] Add release-reproducible CI workflow --- .github/workflows/release-reproducible.yml | 131 +++++++++++++++++++++ Makefile | 2 +- 2 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/release-reproducible.yml diff --git a/.github/workflows/release-reproducible.yml b/.github/workflows/release-reproducible.yml new file mode 100644 index 00000000000..80f7c670306 --- /dev/null +++ b/.github/workflows/release-reproducible.yml @@ -0,0 +1,131 @@ +# This workflow is for building and pushing reproducible Docker images for releases. +name: release-reproducible + +on: + push: + tags: + - v* + workflow_dispatch: + inputs: + dry_run: + description: "Enable dry run mode (builds images but skips push to registry)" + type: boolean + default: false + +env: + DOCKER_REPRODUCIBLE_IMAGE_NAME: ${{ github.repository_owner }}/lighthouse-reproducible + DOCKER_PASSWORD: ${{ secrets.DH_KEY }} + DOCKER_USERNAME: ${{ secrets.DH_ORG }} + +jobs: + extract-version: + name: extract version + runs-on: ubuntu-latest + steps: + - name: Extract version + run: echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT + id: extract_version + outputs: + VERSION: ${{ steps.extract_version.outputs.VERSION }} + + build-reproducible: + name: build and push reproducible images + runs-on: ubuntu-latest + needs: extract-version + strategy: + matrix: + arch: [amd64, arm64] + include: + - arch: amd64 + rust_target: x86_64-unknown-linux-gnu + rust_image: rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1 + platform: linux/amd64 + - arch: arm64 + rust_target: aarch64-unknown-linux-gnu + rust_image: rust:1.86-bullseye@sha256:36053eabadeb701e3e0406610a2ce72ccfa10b7828963cd08cffdcf660518b27 + platform: linux/arm64 + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + if: ${{ github.event.inputs.dry_run != 'true' }} + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKER_PASSWORD }} + + - name: Build reproducible image (${{ matrix.arch }}) + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile.reproducible + platforms: ${{ matrix.platform }} + push: ${{ github.event.inputs.dry_run != 'true' }} + tags: | + ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }} + build-args: | + RUST_TARGET=${{ matrix.rust_target }} + RUST_IMAGE=${{ matrix.rust_image }} + cache-from: type=gha,scope=${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ matrix.arch }} + provenance: false + env: + DOCKER_BUILD_RECORD_UPLOAD: false + + create-manifest: + name: create multi-arch manifest + runs-on: ubuntu-latest + needs: [extract-version, build-reproducible] + if: ${{ github.event.inputs.dry_run != 'true' }} + steps: + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKER_PASSWORD }} + + - name: Create and push multi-arch manifest + run: | + # Create manifest for version tag + docker manifest create \ + ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }} \ + ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}-amd64 \ + ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}-arm64 + + docker manifest push ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }} + + # Create manifest for latest tag + docker manifest create \ + ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:latest \ + ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}-amd64 \ + ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}-arm64 + + docker manifest push ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:latest + + dry-run-summary: + name: dry run summary + runs-on: ubuntu-latest + needs: [build-reproducible, extract-version] + if: ${{ github.event.inputs.dry_run == 'true' }} + steps: + - name: Summarize dry run + run: | + echo "## ๐Ÿงช Reproducible Build Dry Run Summary" + echo "" + echo "โœ… Successfully completed dry run for version ${{ needs.extract-version.outputs.VERSION }}" + echo "" + echo "### What would happen in a real release:" + echo "- Multi-arch reproducible Docker images would be built" + echo "- Images would be pushed to Docker Hub as:" + echo " - \`${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}\`" + echo " - \`${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:latest\`" + echo "" + echo "### Architectures built:" + echo "- linux/amd64 (x86_64-unknown-linux-gnu)" + echo "- linux/arm64 (aarch64-unknown-linux-gnu)" + echo "" + echo "### Next Steps" + echo "To perform a real release, push a git tag (e.g., \`git tag v4.6.0 && git push origin v4.6.0\`)" diff --git a/Makefile b/Makefile index a58dcfc4f90..caae9f8ac2a 100644 --- a/Makefile +++ b/Makefile @@ -127,7 +127,7 @@ build-reproducible-x86_64: ## Build reproducible x86_64 Docker image DOCKER_BUILDKIT=1 docker build \ --build-arg RUST_TARGET="x86_64-unknown-linux-gnu" \ --build-arg RUST_IMAGE=$(RUST_IMAGE_AMD64) \ - -f Dockerfile.test \ + -f Dockerfile.reproducible \ -t lighthouse:reproducible-amd64 . .PHONY: build-reproducible-aarch64 From a4348b2ef3a833e09df2ace1d90d077481c9b3fd Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Mon, 16 Jun 2025 22:10:06 +0000 Subject: [PATCH 03/33] Add reproducible-build workflow to catch regressions --- .github/workflows/reproducible-build.yml | 178 +++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 .github/workflows/reproducible-build.yml diff --git a/.github/workflows/reproducible-build.yml b/.github/workflows/reproducible-build.yml new file mode 100644 index 00000000000..f2115eba271 --- /dev/null +++ b/.github/workflows/reproducible-build.yml @@ -0,0 +1,178 @@ +name: reproducible-build + +on: + workflow_dispatch: {} + schedule: + # Run every 2 days at 1 AM UTC + - cron: "0 1 */2 * *" + pull_request: + # Also run on PRs that might affect reproducible builds + paths: + - "Makefile" + - "Dockerfile.reproducible" + - ".github/workflows/reproducible-build.yml" + - "Cargo.toml" + - "Cargo.lock" + +jobs: + build-x86_64: + name: test reproducible builds (x86_64) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + with: + target: x86_64-unknown-linux-gnu + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y libclang-dev cmake + + - name: Install cargo-cache + run: cargo install cargo-cache + + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + key: reproducible-build-x86_64 + + - name: Build Lighthouse (first build) + run: | + make build-reproducible RUST_TARGET=x86_64-unknown-linux-gnu + cp target/x86_64-unknown-linux-gnu/release/lighthouse lighthouse-build-1 + sha256sum lighthouse-build-1 > lighthouse-build-1.sha256 + + - name: Clean build artifacts and cache + run: | + make clean + cargo cache -a + rm -rf target/ + + - name: Build Lighthouse (second build) + run: | + make build-reproducible RUST_TARGET=x86_64-unknown-linux-gnu + cp target/x86_64-unknown-linux-gnu/release/lighthouse lighthouse-build-2 + sha256sum lighthouse-build-2 > lighthouse-build-2.sha256 + + - name: Compare binaries + run: | + echo "=== Build 1 SHA256 ===" + cat lighthouse-build-1.sha256 + echo "=== Build 2 SHA256 ===" + cat lighthouse-build-2.sha256 + echo "=== Binary Comparison ===" + if cmp lighthouse-build-1 lighthouse-build-2; then + echo "โœ… Binaries are identical - reproducible build PASSED" + else + echo "โŒ Binaries differ - reproducible build FAILED" + exit 1 + fi + + - name: Upload build artifacts (on failure) + if: failure() + uses: actions/upload-artifact@v4 + with: + name: failed-reproducible-builds-x86_64 + path: | + lighthouse-build-1 + lighthouse-build-2 + lighthouse-build-1.sha256 + lighthouse-build-2.sha256 + + build-aarch64: + name: test reproducible builds (aarch64) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + with: + target: aarch64-unknown-linux-gnu + + - name: Install build dependencies and cross-compilation tools + run: | + sudo apt-get update + sudo apt-get install -y libclang-dev cmake gcc-aarch64-linux-gnu + + - name: Install cargo-cache + run: cargo install cargo-cache + + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + key: reproducible-build-aarch64 + + - name: Build Lighthouse (first build) + run: | + make build-reproducible RUST_TARGET=aarch64-unknown-linux-gnu + cp target/aarch64-unknown-linux-gnu/release/lighthouse lighthouse-build-1-arm64 + sha256sum lighthouse-build-1-arm64 > lighthouse-build-1-arm64.sha256 + + - name: Clean build artifacts and cache + run: | + make clean + cargo cache -a + rm -rf target/ + + - name: Build Lighthouse (second build) + run: | + make build-reproducible RUST_TARGET=aarch64-unknown-linux-gnu + cp target/aarch64-unknown-linux-gnu/release/lighthouse lighthouse-build-2-arm64 + sha256sum lighthouse-build-2-arm64 > lighthouse-build-2-arm64.sha256 + + - name: Compare binaries + run: | + echo "=== Build 1 SHA256 (ARM64) ===" + cat lighthouse-build-1-arm64.sha256 + echo "=== Build 2 SHA256 (ARM64) ===" + cat lighthouse-build-2-arm64.sha256 + echo "=== Binary Comparison ===" + if cmp lighthouse-build-1-arm64 lighthouse-build-2-arm64; then + echo "โœ… ARM64 binaries are identical - reproducible build PASSED" + else + echo "โŒ ARM64 binaries differ - reproducible build FAILED" + exit 1 + fi + + - name: Upload build artifacts (on failure) + if: failure() + uses: actions/upload-artifact@v4 + with: + name: failed-reproducible-builds-aarch64 + path: | + lighthouse-build-1-arm64 + lighthouse-build-2-arm64 + lighthouse-build-1-arm64.sha256 + lighthouse-build-2-arm64.sha256 + + summary: + name: reproducible build summary + runs-on: ubuntu-latest + needs: [build-x86_64, build-aarch64] + if: always() + steps: + - name: Report results + run: | + echo "## ๐Ÿ”„ Reproducible Build Test Results" + echo "" + if [[ "${{ needs.build-x86_64.result }}" == "success" ]]; then + echo "โœ… **x86_64**: Reproducible builds PASSED" + else + echo "โŒ **x86_64**: Reproducible builds FAILED" + fi + + if [[ "${{ needs.build-aarch64.result }}" == "success" ]]; then + echo "โœ… **aarch64**: Reproducible builds PASSED" + else + echo "โŒ **aarch64**: Reproducible builds FAILED" + fi + + echo "" + if [[ "${{ needs.build-x86_64.result }}" == "success" ]] && [[ "${{ needs.build-aarch64.result }}" == "success" ]]; then + echo "๐ŸŽ‰ **Overall**: All reproducible builds are working correctly!" + else + echo "โš ๏ธ **Overall**: Some reproducible builds failed - check the logs above" + exit 1 + fi From 608caaba9e6296134e8ad35d1227cb2c5e0e8ca8 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Mon, 16 Jun 2025 22:23:21 +0000 Subject: [PATCH 04/33] fixing linting issues --- .github/workflows/release-reproducible.yml | 69 +++++++++++--------- .github/workflows/reproducible-build.yml | 74 ++++++++++++---------- 2 files changed, 82 insertions(+), 61 deletions(-) diff --git a/.github/workflows/release-reproducible.yml b/.github/workflows/release-reproducible.yml index 80f7c670306..10f668869ad 100644 --- a/.github/workflows/release-reproducible.yml +++ b/.github/workflows/release-reproducible.yml @@ -1,4 +1,3 @@ -# This workflow is for building and pushing reproducible Docker images for releases. name: release-reproducible on: @@ -8,12 +7,14 @@ on: workflow_dispatch: inputs: dry_run: - description: "Enable dry run mode (builds images but skips push to registry)" + description: >- + Enable dry run mode (builds images but skips push to registry) type: boolean default: false env: - DOCKER_REPRODUCIBLE_IMAGE_NAME: ${{ github.repository_owner }}/lighthouse-reproducible + DOCKER_REPRODUCIBLE_IMAGE_NAME: >- + ${{ github.repository_owner }}/lighthouse-reproducible DOCKER_PASSWORD: ${{ secrets.DH_KEY }} DOCKER_USERNAME: ${{ secrets.DH_ORG }} @@ -23,7 +24,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Extract version - run: echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT + run: >- + echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT id: extract_version outputs: VERSION: ${{ steps.extract_version.outputs.VERSION }} @@ -38,42 +40,46 @@ jobs: include: - arch: amd64 rust_target: x86_64-unknown-linux-gnu - rust_image: rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1 + rust_image: >- + rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1 platform: linux/amd64 - arch: arm64 rust_target: aarch64-unknown-linux-gnu - rust_image: rust:1.86-bullseye@sha256:36053eabadeb701e3e0406610a2ce72ccfa10b7828963cd08cffdcf660518b27 + rust_image: >- + rust:1.86-bullseye@sha256:36053eabadeb701e3e0406610a2ce72ccfa10b7828963cd08cffdcf660518b27 platform: linux/arm64 steps: - uses: actions/checkout@v4 - + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - + - name: Log in to Docker Hub if: ${{ github.event.inputs.dry_run != 'true' }} uses: docker/login-action@v3 with: username: ${{ env.DOCKER_USERNAME }} password: ${{ env.DOCKER_PASSWORD }} - + - name: Build reproducible image (${{ matrix.arch }}) uses: docker/build-push-action@v6 + env: + IMAGE_BASE: ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }} + VERSION: ${{ needs.extract-version.outputs.VERSION }} + ARCH: ${{ matrix.arch }} + DOCKER_BUILD_RECORD_UPLOAD: false with: context: . file: ./Dockerfile.reproducible platforms: ${{ matrix.platform }} push: ${{ github.event.inputs.dry_run != 'true' }} - tags: | - ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }} + tags: ${{ env.IMAGE_BASE }}:${{ env.VERSION }}-${{ env.ARCH }} build-args: | RUST_TARGET=${{ matrix.rust_target }} RUST_IMAGE=${{ matrix.rust_image }} cache-from: type=gha,scope=${{ matrix.arch }} cache-to: type=gha,mode=max,scope=${{ matrix.arch }} provenance: false - env: - DOCKER_BUILD_RECORD_UPLOAD: false create-manifest: name: create multi-arch manifest @@ -86,24 +92,26 @@ jobs: with: username: ${{ env.DOCKER_USERNAME }} password: ${{ env.DOCKER_PASSWORD }} - + - name: Create and push multi-arch manifest run: | + IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }} + VERSION=${{ needs.extract-version.outputs.VERSION }} # Create manifest for version tag docker manifest create \ - ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }} \ - ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}-amd64 \ - ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}-arm64 - - docker manifest push ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }} - + ${IMAGE_NAME}:${VERSION} \ + ${IMAGE_NAME}:${VERSION}-amd64 \ + ${IMAGE_NAME}:${VERSION}-arm64 + + docker manifest push ${IMAGE_NAME}:${VERSION} + # Create manifest for latest tag docker manifest create \ - ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:latest \ - ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}-amd64 \ - ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}-arm64 - - docker manifest push ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:latest + ${IMAGE_NAME}:latest \ + ${IMAGE_NAME}:${VERSION}-amd64 \ + ${IMAGE_NAME}:${VERSION}-arm64 + + docker manifest push ${IMAGE_NAME}:latest dry-run-summary: name: dry run summary @@ -113,19 +121,22 @@ jobs: steps: - name: Summarize dry run run: | + IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }} + VERSION=${{ needs.extract-version.outputs.VERSION }} echo "## ๐Ÿงช Reproducible Build Dry Run Summary" echo "" - echo "โœ… Successfully completed dry run for version ${{ needs.extract-version.outputs.VERSION }}" + echo "โœ… Successfully completed dry run for version ${VERSION}" echo "" echo "### What would happen in a real release:" echo "- Multi-arch reproducible Docker images would be built" echo "- Images would be pushed to Docker Hub as:" - echo " - \`${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}\`" - echo " - \`${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:latest\`" + echo " - \`${IMAGE_NAME}:${VERSION}\`" + echo " - \`${IMAGE_NAME}:latest\`" echo "" echo "### Architectures built:" echo "- linux/amd64 (x86_64-unknown-linux-gnu)" echo "- linux/arm64 (aarch64-unknown-linux-gnu)" echo "" echo "### Next Steps" - echo "To perform a real release, push a git tag (e.g., \`git tag v4.6.0 && git push origin v4.6.0\`)" + echo "To perform a real release, push a git tag" + echo "(e.g., \`git tag v4.6.0 && git push origin v4.6.0\`)" diff --git a/.github/workflows/reproducible-build.yml b/.github/workflows/reproducible-build.yml index f2115eba271..843e9246966 100644 --- a/.github/workflows/reproducible-build.yml +++ b/.github/workflows/reproducible-build.yml @@ -3,10 +3,8 @@ name: reproducible-build on: workflow_dispatch: {} schedule: - # Run every 2 days at 1 AM UTC - cron: "0 1 */2 * *" pull_request: - # Also run on PRs that might affect reproducible builds paths: - "Makefile" - "Dockerfile.reproducible" @@ -20,19 +18,19 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - + - uses: dtolnay/rust-toolchain@stable with: target: x86_64-unknown-linux-gnu - + - name: Install build dependencies run: | sudo apt-get update sudo apt-get install -y libclang-dev cmake - + - name: Install cargo-cache run: cargo install cargo-cache - + - uses: Swatinem/rust-cache@v2 with: cache-on-failure: true @@ -40,22 +38,26 @@ jobs: - name: Build Lighthouse (first build) run: | - make build-reproducible RUST_TARGET=x86_64-unknown-linux-gnu - cp target/x86_64-unknown-linux-gnu/release/lighthouse lighthouse-build-1 + make build-reproducible \ + RUST_TARGET=x86_64-unknown-linux-gnu + cp target/x86_64-unknown-linux-gnu/release/lighthouse \ + lighthouse-build-1 sha256sum lighthouse-build-1 > lighthouse-build-1.sha256 - + - name: Clean build artifacts and cache run: | make clean cargo cache -a rm -rf target/ - + - name: Build Lighthouse (second build) run: | - make build-reproducible RUST_TARGET=x86_64-unknown-linux-gnu - cp target/x86_64-unknown-linux-gnu/release/lighthouse lighthouse-build-2 + make build-reproducible \ + RUST_TARGET=x86_64-unknown-linux-gnu + cp target/x86_64-unknown-linux-gnu/release/lighthouse \ + lighthouse-build-2 sha256sum lighthouse-build-2 > lighthouse-build-2.sha256 - + - name: Compare binaries run: | echo "=== Build 1 SHA256 ===" @@ -69,7 +71,7 @@ jobs: echo "โŒ Binaries differ - reproducible build FAILED" exit 1 fi - + - name: Upload build artifacts (on failure) if: failure() uses: actions/upload-artifact@v4 @@ -86,19 +88,19 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - + - uses: dtolnay/rust-toolchain@stable with: target: aarch64-unknown-linux-gnu - + - name: Install build dependencies and cross-compilation tools run: | sudo apt-get update sudo apt-get install -y libclang-dev cmake gcc-aarch64-linux-gnu - + - name: Install cargo-cache run: cargo install cargo-cache - + - uses: Swatinem/rust-cache@v2 with: cache-on-failure: true @@ -106,22 +108,28 @@ jobs: - name: Build Lighthouse (first build) run: | - make build-reproducible RUST_TARGET=aarch64-unknown-linux-gnu - cp target/aarch64-unknown-linux-gnu/release/lighthouse lighthouse-build-1-arm64 - sha256sum lighthouse-build-1-arm64 > lighthouse-build-1-arm64.sha256 - + make build-reproducible \ + RUST_TARGET=aarch64-unknown-linux-gnu + cp target/aarch64-unknown-linux-gnu/release/lighthouse \ + lighthouse-build-1-arm64 + sha256sum lighthouse-build-1-arm64 > \ + lighthouse-build-1-arm64.sha256 + - name: Clean build artifacts and cache run: | make clean cargo cache -a rm -rf target/ - + - name: Build Lighthouse (second build) run: | - make build-reproducible RUST_TARGET=aarch64-unknown-linux-gnu - cp target/aarch64-unknown-linux-gnu/release/lighthouse lighthouse-build-2-arm64 - sha256sum lighthouse-build-2-arm64 > lighthouse-build-2-arm64.sha256 - + make build-reproducible \ + RUST_TARGET=aarch64-unknown-linux-gnu + cp target/aarch64-unknown-linux-gnu/release/lighthouse \ + lighthouse-build-2-arm64 + sha256sum lighthouse-build-2-arm64 > \ + lighthouse-build-2-arm64.sha256 + - name: Compare binaries run: | echo "=== Build 1 SHA256 (ARM64) ===" @@ -135,7 +143,7 @@ jobs: echo "โŒ ARM64 binaries differ - reproducible build FAILED" exit 1 fi - + - name: Upload build artifacts (on failure) if: failure() uses: actions/upload-artifact@v4 @@ -162,17 +170,19 @@ jobs: else echo "โŒ **x86_64**: Reproducible builds FAILED" fi - + if [[ "${{ needs.build-aarch64.result }}" == "success" ]]; then echo "โœ… **aarch64**: Reproducible builds PASSED" else echo "โŒ **aarch64**: Reproducible builds FAILED" fi - + echo "" - if [[ "${{ needs.build-x86_64.result }}" == "success" ]] && [[ "${{ needs.build-aarch64.result }}" == "success" ]]; then + if [[ "${{ needs.build-x86_64.result }}" == "success" ]] \ + && [[ "${{ needs.build-aarch64.result }}" == "success" ]]; then echo "๐ŸŽ‰ **Overall**: All reproducible builds are working correctly!" else - echo "โš ๏ธ **Overall**: Some reproducible builds failed - check the logs above" + echo "โš ๏ธ **Overall**: Some reproducible builds failed" + echo "Check the logs above" exit 1 fi From b946c49d6e9e8d9092663b19030b7a2ddbafc66c Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Tue, 17 Jun 2025 18:18:10 +0000 Subject: [PATCH 05/33] Add necessary setup for reproducible debian packaging --- Makefile | 140 ++++++++++++++++++++++++++++++++++ lighthouse/Cargo.toml | 30 ++++++++ lighthouse/lighthouse.service | 34 +++++++++ 3 files changed, 204 insertions(+) create mode 100644 lighthouse/lighthouse.service diff --git a/Makefile b/Makefile index caae9f8ac2a..96ccdc80538 100644 --- a/Makefile +++ b/Makefile @@ -341,3 +341,143 @@ clean: cargo clean make -C $(EF_TESTS) clean make -C $(STATE_TRANSITION_VECTORS) clean + +.PHONY: install-cargo-deb deb-cargo deb-cargo-x86_64 deb-cargo-aarch64 deb-cargo-all test-deb-reproducible install-deb-local remove-deb-local clean-deb help-deb +## Install cargo-deb if not present +install-cargo-deb: + @if ! command -v cargo-deb &> /dev/null; then \ + echo "Installing cargo-deb..."; \ + cargo install cargo-deb; \ + else \ + echo "cargo-deb already installed"; \ + fi + +## Build .deb package using cargo-deb with reproducible settings +deb-cargo: install-cargo-deb build-reproducible + @echo "Building .deb package with cargo-deb..." + @if [ ! -f "lighthouse/lighthouse.service" ]; then \ + echo "โŒ lighthouse.service not found in lighthouse/ directory"; \ + exit 1; \ + fi + @if [ ! -f "README.md" ]; then \ + echo "โŒ README.md not found in current directory"; \ + exit 1; \ + fi + + cd lighthouse && \ + SOURCE_DATE_EPOCH=$(SOURCE_DATE) \ + CARGO_INCREMENTAL=$(CARGO_INCREMENTAL_VAL) \ + LC_ALL=$(LOCALE_VAL) \ + TZ=$(TZ_VAL) \ + cargo deb --target $(RUST_TARGET) --no-build + + @echo "โœ… Package built successfully!" + @find target/$(RUST_TARGET)/debian -name "*.deb" -exec ls -la {} \; + +## Build .deb for specific architectures +deb-cargo-x86_64: + $(MAKE) deb-cargo RUST_TARGET=x86_64-unknown-linux-gnu + +deb-cargo-aarch64: + $(MAKE) deb-cargo RUST_TARGET=aarch64-unknown-linux-gnu + +deb-cargo-all: deb-cargo-x86_64 deb-cargo-aarch64 + +## Test reproducibility of cargo-deb packages +test-deb-reproducible: + @echo "Testing cargo-deb package reproducibility..." + @if ! command -v diffoscope &> /dev/null; then \ + echo "Installing diffoscope..."; \ + sudo apt-get update; \ + sudo apt-get install -y diffoscope binutils-multiarch; \ + fi + + @echo "Building first package..." + @rm -f lighthouse_*.deb lighthouse-deb-*.deb + @$(MAKE) clean || true + @$(MAKE) deb-cargo + @FIRST_PACKAGE=$$(find target/$(RUST_TARGET)/debian -name "*.deb" | head -1); \ + if [ -n "$$FIRST_PACKAGE" ]; then \ + cp "$$FIRST_PACKAGE" ./lighthouse-deb-build-1.deb; \ + else \ + echo "โŒ First package not found"; exit 1; \ + fi + + @echo "Building second package..." + @$(MAKE) clean || true + @$(MAKE) deb-cargo + @SECOND_PACKAGE=$$(find target/$(RUST_TARGET)/debian -name "*.deb" | head -1); \ + if [ -n "$$SECOND_PACKAGE" ]; then \ + cp "$$SECOND_PACKAGE" ./lighthouse-deb-build-2.deb; \ + else \ + echo "โŒ Second package not found"; exit 1; \ + fi + + @echo "Comparing packages..." + @echo "=== Package sizes ===" + @ls -la lighthouse-deb-build-*.deb + @echo "=== SHA256 checksums ===" + @sha256sum lighthouse-deb-build-*.deb + + @if cmp -s lighthouse-deb-build-1.deb lighthouse-deb-build-2.deb; then \ + echo "โœ… SUCCESS: cargo-deb packages are identical!"; \ + echo "โœ… Reproducible build PASSED"; \ + else \ + echo "โŒ FAILED: cargo-deb packages differ"; \ + echo "Running detailed analysis with diffoscope..."; \ + diffoscope --text lighthouse-deb-build-1.deb lighthouse-deb-build-2.deb > cargo-deb-diff.txt || true; \ + echo "Differences saved to cargo-deb-diff.txt"; \ + echo "โŒ Reproducible build FAILED"; \ + exit 1; \ + fi + +## Install .deb package locally for testing +install-deb-local: + @PACKAGE=$$(find target/$(RUST_TARGET)/debian -name "*.deb" | head -1); \ + if [ -n "$$PACKAGE" ]; then \ + echo "Installing lighthouse package: $$PACKAGE"; \ + sudo dpkg -i "$$PACKAGE"; \ + echo "Fixing dependencies if needed..."; \ + sudo apt-get install -f; \ + echo "โœ… Package installed successfully!"; \ + echo ""; \ + echo "The lighthouse service is now available but not started."; \ + echo "Your systemd service file handles user creation declaratively."; \ + echo ""; \ + echo "To check service status: systemctl status lighthouse"; \ + else \ + echo "โŒ No .deb package found. Run 'make deb-cargo' first."; \ + fi + +## Remove installed lighthouse package +remove-deb-local: + @echo "Removing lighthouse package..." + sudo dpkg -r lighthouse || true + sudo systemctl daemon-reload || true + +## Clean up debian packaging artifacts +clean-deb: + @echo "Cleaning up debian packaging artifacts..." + rm -f lighthouse_*.deb lighthouse-deb-*.deb *-diff.txt + rm -rf target/*/debian/ + +## Show help for debian packaging +help-deb: + @echo "Clean Debian Packaging Targets:" + @echo " install-cargo-deb - Install cargo-deb tool" + @echo " deb-cargo - Build .deb package with cargo-deb" + @echo " deb-cargo-x86_64 - Build x86_64 .deb package" + @echo " deb-cargo-aarch64 - Build aarch64 .deb package" + @echo " deb-cargo-all - Build all architectures" + @echo " test-deb-reproducible - Test reproducibility" + @echo " install-deb-local - Install .deb package locally" + @echo " remove-deb-local - Remove installed package" + @echo " clean-deb - Clean up packaging artifacts" + @echo "" + @echo "Prerequisites:" + @echo " - lighthouse/lighthouse.service file" + @echo " - README.md file in current directory" + @echo "" + @echo "Quick start:" + @echo " make deb-cargo - Build .deb for current RUST_TARGET" + @echo " make install-deb-local - Test the package" diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index 3ca93aedf7a..956a0d202fb 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -93,3 +93,33 @@ zeroize = { workspace = true } [[test]] name = "lighthouse_tests" path = "tests/main.rs" + +[package.metadata.deb] +# Package metadata +maintainer = "Sigma Prime " +extended-description = """\ +Lighthouse is a Rust implementation of the Ethereum beacon chain, \ +built by Sigma Prime. It implements the official Ethereum 2.0 specification.""" + +# Dependencies +depends = "$auto, systemd" +section = "net" +priority = "optional" + +# System integration +systemd-units = { enable = false, start = false } + +# Assets to include in the package +assets = [ + # Binary - use target/release/ prefix (cargo-deb will replace with actual path) + ["target/release/lighthouse", "usr/bin/", "755"], + + # Systemd service (in lighthouse directory) + ["lighthouse.service", "lib/systemd/system/", "644"], + + # Documentation (in parent directory) + ["../README.md", "usr/share/doc/lighthouse/", "644"], +] + +# Use default features - let Makefile handle the building +default-features = false diff --git a/lighthouse/lighthouse.service b/lighthouse/lighthouse.service new file mode 100644 index 00000000000..dfc2b4f9f52 --- /dev/null +++ b/lighthouse/lighthouse.service @@ -0,0 +1,34 @@ +[Unit] +Description=Lighthouse Ethereum Beacon Chain Client +Documentation=https://lighthouse-book.sigmaprime.io/ +After=network.target +Wants=network.target + +[Service] +Type=simple +DynamicUser=true +User=lighthouse +Group=eth +Restart=always +RestartSec=5 +TimeoutStopSec=180 +Environment="RUST_LOG=info" +ExecStart=/usr/bin/lighthouse \ + bn \ + --checkpoint-sync-url https://sync-mainnet.beaconcha.in \ + --execution-endpoint http://localhost:8551 \ + --execution-jwt /etc/jwt.hex \ + --suggested-fee-recipient 0x0 \ + --always-prepare-payload \ + --prepare-payload-lookahead 8000 \ + --disable-deposit-contract-sync \ + --http \ + --port 9000 \ + --http-port 3500 \ + --metrics \ + --metrics-address 127.0.0.1 \ + --metrics-port 5054 \ + --datadir /mnt/data/lighthouse + +[Install] +WantedBy=default.target From d58a9c1f9c2f7ff8aa92e6e12fd80801af0496c8 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Tue, 17 Jun 2025 18:37:50 +0000 Subject: [PATCH 06/33] add CI workflow to build and upload deb packages as assets --- .github/workflows/release-deb.yml | 312 ++++++++++++++++++++++++++++++ 1 file changed, 312 insertions(+) create mode 100644 .github/workflows/release-deb.yml diff --git a/.github/workflows/release-deb.yml b/.github/workflows/release-deb.yml new file mode 100644 index 00000000000..b32ac845f2c --- /dev/null +++ b/.github/workflows/release-deb.yml @@ -0,0 +1,312 @@ +name: release-deb + +on: + push: + tags: + - v* + workflow_dispatch: + inputs: + dry_run: + description: >- + Enable dry run mode (builds packages but skips release upload) + type: boolean + default: false + +jobs: + extract-version: + name: extract version + runs-on: ubuntu-latest + steps: + - name: Extract version + run: >- + echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT + id: extract_version + outputs: + VERSION: ${{ steps.extract_version.outputs.VERSION }} + + build-deb: + name: build reproducible deb packages + runs-on: ubuntu-latest + needs: extract-version + strategy: + matrix: + arch: [x86_64, aarch64] + include: + - arch: x86_64 + rust_target: x86_64-unknown-linux-gnu + gcc_package: gcc + - arch: aarch64 + rust_target: aarch64-unknown-linux-gnu + gcc_package: gcc-aarch64-linux-gnu + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + with: + target: ${{ matrix.rust_target }} + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y libclang-dev cmake ${{ matrix.gcc_package }} + + - name: Install cargo-deb + run: cargo install cargo-deb + + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + key: deb-build-${{ matrix.arch }} + + - name: Build reproducible .deb package + run: | + make deb-cargo RUST_TARGET=${{ matrix.rust_target }} + + - name: Find and rename .deb package + run: | + VERSION=${{ needs.extract-version.outputs.VERSION }} + DEB_FILE=$(find target/${{ matrix.rust_target }}/debian -name "*.deb" | head -1) + if [ -n "$DEB_FILE" ]; then + # Extract package info + DEB_NAME=$(dpkg-deb -f "$DEB_FILE" Package) + DEB_VERSION=$(dpkg-deb -f "$DEB_FILE" Version) + DEB_ARCH=$(dpkg-deb -f "$DEB_FILE" Architecture) + + # Create standardized filename + NEW_NAME="lighthouse_${VERSION#v}_${{ matrix.arch }}.deb" + cp "$DEB_FILE" "$NEW_NAME" + + echo "DEB_FILE=$NEW_NAME" >> $GITHUB_ENV + echo "DEB_SIZE=$(stat -f%z "$NEW_NAME" 2>/dev/null || stat -c%s "$NEW_NAME")" >> $GITHUB_ENV + + # Generate checksums + sha256sum "$NEW_NAME" > "$NEW_NAME.sha256" + sha512sum "$NEW_NAME" > "$NEW_NAME.sha512" + + echo "Package built: $NEW_NAME" + echo "Size: $(du -h "$NEW_NAME" | cut -f1)" + echo "SHA256: $(cat "$NEW_NAME.sha256")" + else + echo "โŒ No .deb package found" + exit 1 + fi + + - name: Test package installation (dry run) + run: | + echo "Testing package metadata and dependencies..." + dpkg-deb -I "$DEB_FILE" + echo "" + echo "Package contents:" + dpkg-deb -c "$DEB_FILE" + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: lighthouse-deb-${{ matrix.arch }} + path: | + lighthouse_*_${{ matrix.arch }}.deb + lighthouse_*_${{ matrix.arch }}.deb.sha256 + lighthouse_*_${{ matrix.arch }}.deb.sha512 + retention-days: 7 + + test-reproducibility: + name: test deb reproducibility + runs-on: ubuntu-latest + needs: extract-version + strategy: + matrix: + arch: [x86_64, aarch64] + include: + - arch: x86_64 + rust_target: x86_64-unknown-linux-gnu + gcc_package: gcc + - arch: aarch64 + rust_target: aarch64-unknown-linux-gnu + gcc_package: gcc-aarch64-linux-gnu + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + with: + target: ${{ matrix.rust_target }} + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y libclang-dev cmake ${{ matrix.gcc_package }} diffoscope + + - name: Install cargo-deb and cargo-cache + run: | + cargo install cargo-deb + cargo install cargo-cache + + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + key: deb-reproducible-${{ matrix.arch }} + + - name: Test reproducible deb build + run: | + make test-deb-reproducible RUST_TARGET=${{ matrix.rust_target }} + + - name: Upload reproducibility test artifacts (on failure) + if: failure() + uses: actions/upload-artifact@v4 + with: + name: failed-reproducible-deb-${{ matrix.arch }} + path: | + lighthouse-deb-build-*.deb + *-diff.txt + + create-release: + name: create github release + runs-on: ubuntu-latest + needs: [extract-version, build-deb, test-reproducibility] + if: ${{ github.event.inputs.dry_run != 'true' }} + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Prepare release assets + run: | + mkdir -p release-assets + find artifacts -name "*.deb" -exec cp {} release-assets/ \; + find artifacts -name "*.sha256" -exec cp {} release-assets/ \; + find artifacts -name "*.sha512" -exec cp {} release-assets/ \; + + echo "Release assets:" + ls -la release-assets/ + + - name: Generate release notes + run: | + VERSION=${{ needs.extract-version.outputs.VERSION }} + cat > release-notes.md << EOF + # Lighthouse ${VERSION} - Debian Packages + + This release includes reproducible Debian packages for Lighthouse ${VERSION}. + + ## Available Packages + + - **x86_64**: \`lighthouse_${VERSION#v}_x86_64.deb\` + - **aarch64**: \`lighthouse_${VERSION#v}_aarch64.deb\` + + ## Installation + + ### Ubuntu/Debian (x86_64): + \`\`\`bash + wget https://github.com/${{ github.repository }}/releases/download/${VERSION}/lighthouse_${VERSION#v}_x86_64.deb + sudo dpkg -i lighthouse_${VERSION#v}_x86_64.deb + sudo apt-get install -f # Fix any dependency issues + \`\`\` + + ### Ubuntu/Debian (ARM64): + \`\`\`bash + wget https://github.com/${{ github.repository }}/releases/download/${VERSION}/lighthouse_${VERSION#v}_aarch64.deb + sudo dpkg -i lighthouse_${VERSION#v}_aarch64.deb + sudo apt-get install -f # Fix any dependency issues + \`\`\` + + ## Verification + + All packages include SHA256 and SHA512 checksums for verification: + + \`\`\`bash + # Verify SHA256 + sha256sum -c lighthouse_${VERSION#v}_x86_64.deb.sha256 + + # Verify SHA512 + sha512sum -c lighthouse_${VERSION#v}_x86_64.deb.sha512 + \`\`\` + + ## System Service + + After installation, Lighthouse can be managed as a systemd service: + + \`\`\`bash + # Enable and start the service + sudo systemctl enable lighthouse + sudo systemctl start lighthouse + + # Check status + sudo systemctl status lighthouse + + # View logs + sudo journalctl -u lighthouse -f + \`\`\` + + ## Reproducible Builds + + These packages are built using reproducible build techniques. You can verify the build process by checking out the source code at tag \`${VERSION}\` and running: + + \`\`\`bash + make deb-cargo RUST_TARGET=x86_64-unknown-linux-gnu + make test-deb-reproducible RUST_TARGET=x86_64-unknown-linux-gnu + \`\`\` + + ## Package Details + + - **Maintainer**: Sigma Prime + - **Dependencies**: Automatically managed by dpkg/apt + - **Service Integration**: Includes systemd service unit + - **Build Method**: Reproducible builds with cargo-deb + + For more information, see the [Lighthouse documentation](https://lighthouse-book.sigmaprime.io/). + EOF + + - name: Create Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ needs.extract-version.outputs.VERSION }} + name: "Lighthouse ${{ needs.extract-version.outputs.VERSION }} - Debian Packages" + body_path: release-notes.md + files: release-assets/* + draft: false + prerelease: ${{ contains(needs.extract-version.outputs.VERSION, 'beta') || contains(needs.extract-version.outputs.VERSION, 'alpha') || contains(needs.extract-version.outputs.VERSION, 'rc') }} + generate_release_notes: true + + dry-run-summary: + name: dry run summary + runs-on: ubuntu-latest + needs: [extract-version, build-deb, test-reproducibility] + if: ${{ github.event.inputs.dry_run == 'true' }} + steps: + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Summarize dry run + run: | + VERSION=${{ needs.extract-version.outputs.VERSION }} + echo "## ๐Ÿงช Debian Package Release Dry Run Summary" + echo "" + echo "โœ… Successfully completed dry run for version ${VERSION}" + echo "" + echo "### Built Packages:" + find artifacts -name "*.deb" -exec basename {} \; | sort + echo "" + echo "### Package Sizes:" + find artifacts -name "*.deb" -exec ls -lh {} \; | awk '{print $9 ": " $5}' + echo "" + echo "### Checksums Generated:" + find artifacts -name "*.sha256" -exec basename {} \; | sort + find artifacts -name "*.sha512" -exec basename {} \; | sort + echo "" + echo "### What would happen in a real release:" + echo "- Packages would be uploaded to GitHub Releases" + echo "- Release notes would be automatically generated" + echo "- Users could install via:" + echo " \`wget + dpkg -i lighthouse_${VERSION#v}_x86_64.deb\`" + echo "" + echo "### Reproducibility Test Results:" + echo "โœ… All reproducibility tests passed" + echo "" + echo "### Next Steps:" + echo "To perform a real release, push a git tag:" + echo "\`git tag ${VERSION} && git push origin ${VERSION}\`" From af4c59be377983e855d995276f7917a01383f656 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Wed, 18 Jun 2025 08:26:09 +0000 Subject: [PATCH 07/33] merge the deb packaging workflow into release.yml to avoid conflicts --- .github/workflows/release-deb.yml | 312 ------------------------------ .github/workflows/release.yml | 149 +++++++++++++- 2 files changed, 142 insertions(+), 319 deletions(-) delete mode 100644 .github/workflows/release-deb.yml diff --git a/.github/workflows/release-deb.yml b/.github/workflows/release-deb.yml deleted file mode 100644 index b32ac845f2c..00000000000 --- a/.github/workflows/release-deb.yml +++ /dev/null @@ -1,312 +0,0 @@ -name: release-deb - -on: - push: - tags: - - v* - workflow_dispatch: - inputs: - dry_run: - description: >- - Enable dry run mode (builds packages but skips release upload) - type: boolean - default: false - -jobs: - extract-version: - name: extract version - runs-on: ubuntu-latest - steps: - - name: Extract version - run: >- - echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT - id: extract_version - outputs: - VERSION: ${{ steps.extract_version.outputs.VERSION }} - - build-deb: - name: build reproducible deb packages - runs-on: ubuntu-latest - needs: extract-version - strategy: - matrix: - arch: [x86_64, aarch64] - include: - - arch: x86_64 - rust_target: x86_64-unknown-linux-gnu - gcc_package: gcc - - arch: aarch64 - rust_target: aarch64-unknown-linux-gnu - gcc_package: gcc-aarch64-linux-gnu - steps: - - uses: actions/checkout@v4 - - - uses: dtolnay/rust-toolchain@stable - with: - target: ${{ matrix.rust_target }} - - - name: Install build dependencies - run: | - sudo apt-get update - sudo apt-get install -y libclang-dev cmake ${{ matrix.gcc_package }} - - - name: Install cargo-deb - run: cargo install cargo-deb - - - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: true - key: deb-build-${{ matrix.arch }} - - - name: Build reproducible .deb package - run: | - make deb-cargo RUST_TARGET=${{ matrix.rust_target }} - - - name: Find and rename .deb package - run: | - VERSION=${{ needs.extract-version.outputs.VERSION }} - DEB_FILE=$(find target/${{ matrix.rust_target }}/debian -name "*.deb" | head -1) - if [ -n "$DEB_FILE" ]; then - # Extract package info - DEB_NAME=$(dpkg-deb -f "$DEB_FILE" Package) - DEB_VERSION=$(dpkg-deb -f "$DEB_FILE" Version) - DEB_ARCH=$(dpkg-deb -f "$DEB_FILE" Architecture) - - # Create standardized filename - NEW_NAME="lighthouse_${VERSION#v}_${{ matrix.arch }}.deb" - cp "$DEB_FILE" "$NEW_NAME" - - echo "DEB_FILE=$NEW_NAME" >> $GITHUB_ENV - echo "DEB_SIZE=$(stat -f%z "$NEW_NAME" 2>/dev/null || stat -c%s "$NEW_NAME")" >> $GITHUB_ENV - - # Generate checksums - sha256sum "$NEW_NAME" > "$NEW_NAME.sha256" - sha512sum "$NEW_NAME" > "$NEW_NAME.sha512" - - echo "Package built: $NEW_NAME" - echo "Size: $(du -h "$NEW_NAME" | cut -f1)" - echo "SHA256: $(cat "$NEW_NAME.sha256")" - else - echo "โŒ No .deb package found" - exit 1 - fi - - - name: Test package installation (dry run) - run: | - echo "Testing package metadata and dependencies..." - dpkg-deb -I "$DEB_FILE" - echo "" - echo "Package contents:" - dpkg-deb -c "$DEB_FILE" - - - name: Upload build artifacts - uses: actions/upload-artifact@v4 - with: - name: lighthouse-deb-${{ matrix.arch }} - path: | - lighthouse_*_${{ matrix.arch }}.deb - lighthouse_*_${{ matrix.arch }}.deb.sha256 - lighthouse_*_${{ matrix.arch }}.deb.sha512 - retention-days: 7 - - test-reproducibility: - name: test deb reproducibility - runs-on: ubuntu-latest - needs: extract-version - strategy: - matrix: - arch: [x86_64, aarch64] - include: - - arch: x86_64 - rust_target: x86_64-unknown-linux-gnu - gcc_package: gcc - - arch: aarch64 - rust_target: aarch64-unknown-linux-gnu - gcc_package: gcc-aarch64-linux-gnu - steps: - - uses: actions/checkout@v4 - - - uses: dtolnay/rust-toolchain@stable - with: - target: ${{ matrix.rust_target }} - - - name: Install build dependencies - run: | - sudo apt-get update - sudo apt-get install -y libclang-dev cmake ${{ matrix.gcc_package }} diffoscope - - - name: Install cargo-deb and cargo-cache - run: | - cargo install cargo-deb - cargo install cargo-cache - - - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: true - key: deb-reproducible-${{ matrix.arch }} - - - name: Test reproducible deb build - run: | - make test-deb-reproducible RUST_TARGET=${{ matrix.rust_target }} - - - name: Upload reproducibility test artifacts (on failure) - if: failure() - uses: actions/upload-artifact@v4 - with: - name: failed-reproducible-deb-${{ matrix.arch }} - path: | - lighthouse-deb-build-*.deb - *-diff.txt - - create-release: - name: create github release - runs-on: ubuntu-latest - needs: [extract-version, build-deb, test-reproducibility] - if: ${{ github.event.inputs.dry_run != 'true' }} - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts - - - name: Prepare release assets - run: | - mkdir -p release-assets - find artifacts -name "*.deb" -exec cp {} release-assets/ \; - find artifacts -name "*.sha256" -exec cp {} release-assets/ \; - find artifacts -name "*.sha512" -exec cp {} release-assets/ \; - - echo "Release assets:" - ls -la release-assets/ - - - name: Generate release notes - run: | - VERSION=${{ needs.extract-version.outputs.VERSION }} - cat > release-notes.md << EOF - # Lighthouse ${VERSION} - Debian Packages - - This release includes reproducible Debian packages for Lighthouse ${VERSION}. - - ## Available Packages - - - **x86_64**: \`lighthouse_${VERSION#v}_x86_64.deb\` - - **aarch64**: \`lighthouse_${VERSION#v}_aarch64.deb\` - - ## Installation - - ### Ubuntu/Debian (x86_64): - \`\`\`bash - wget https://github.com/${{ github.repository }}/releases/download/${VERSION}/lighthouse_${VERSION#v}_x86_64.deb - sudo dpkg -i lighthouse_${VERSION#v}_x86_64.deb - sudo apt-get install -f # Fix any dependency issues - \`\`\` - - ### Ubuntu/Debian (ARM64): - \`\`\`bash - wget https://github.com/${{ github.repository }}/releases/download/${VERSION}/lighthouse_${VERSION#v}_aarch64.deb - sudo dpkg -i lighthouse_${VERSION#v}_aarch64.deb - sudo apt-get install -f # Fix any dependency issues - \`\`\` - - ## Verification - - All packages include SHA256 and SHA512 checksums for verification: - - \`\`\`bash - # Verify SHA256 - sha256sum -c lighthouse_${VERSION#v}_x86_64.deb.sha256 - - # Verify SHA512 - sha512sum -c lighthouse_${VERSION#v}_x86_64.deb.sha512 - \`\`\` - - ## System Service - - After installation, Lighthouse can be managed as a systemd service: - - \`\`\`bash - # Enable and start the service - sudo systemctl enable lighthouse - sudo systemctl start lighthouse - - # Check status - sudo systemctl status lighthouse - - # View logs - sudo journalctl -u lighthouse -f - \`\`\` - - ## Reproducible Builds - - These packages are built using reproducible build techniques. You can verify the build process by checking out the source code at tag \`${VERSION}\` and running: - - \`\`\`bash - make deb-cargo RUST_TARGET=x86_64-unknown-linux-gnu - make test-deb-reproducible RUST_TARGET=x86_64-unknown-linux-gnu - \`\`\` - - ## Package Details - - - **Maintainer**: Sigma Prime - - **Dependencies**: Automatically managed by dpkg/apt - - **Service Integration**: Includes systemd service unit - - **Build Method**: Reproducible builds with cargo-deb - - For more information, see the [Lighthouse documentation](https://lighthouse-book.sigmaprime.io/). - EOF - - - name: Create Release - uses: softprops/action-gh-release@v2 - with: - tag_name: ${{ needs.extract-version.outputs.VERSION }} - name: "Lighthouse ${{ needs.extract-version.outputs.VERSION }} - Debian Packages" - body_path: release-notes.md - files: release-assets/* - draft: false - prerelease: ${{ contains(needs.extract-version.outputs.VERSION, 'beta') || contains(needs.extract-version.outputs.VERSION, 'alpha') || contains(needs.extract-version.outputs.VERSION, 'rc') }} - generate_release_notes: true - - dry-run-summary: - name: dry run summary - runs-on: ubuntu-latest - needs: [extract-version, build-deb, test-reproducibility] - if: ${{ github.event.inputs.dry_run == 'true' }} - steps: - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts - - - name: Summarize dry run - run: | - VERSION=${{ needs.extract-version.outputs.VERSION }} - echo "## ๐Ÿงช Debian Package Release Dry Run Summary" - echo "" - echo "โœ… Successfully completed dry run for version ${VERSION}" - echo "" - echo "### Built Packages:" - find artifacts -name "*.deb" -exec basename {} \; | sort - echo "" - echo "### Package Sizes:" - find artifacts -name "*.deb" -exec ls -lh {} \; | awk '{print $9 ": " $5}' - echo "" - echo "### Checksums Generated:" - find artifacts -name "*.sha256" -exec basename {} \; | sort - find artifacts -name "*.sha512" -exec basename {} \; | sort - echo "" - echo "### What would happen in a real release:" - echo "- Packages would be uploaded to GitHub Releases" - echo "- Release notes would be automatically generated" - echo "- Users could install via:" - echo " \`wget + dpkg -i lighthouse_${VERSION#v}_x86_64.deb\`" - echo "" - echo "### Reproducibility Test Results:" - echo "โœ… All reproducibility tests passed" - echo "" - echo "### Next Steps:" - echo "To perform a real release, push a git tag:" - echo "\`git tag ${VERSION} && git push origin ${VERSION}\`" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index de4fd294093..4010d9273b7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,7 +34,9 @@ jobs: x86_64-unknown-linux-gnu, x86_64-apple-darwin, aarch64-apple-darwin, - x86_64-windows] + x86_64-windows, + x86_64-unknown-linux-gnu-deb, + aarch64-unknown-linux-gnu-deb] include: - arch: aarch64-unknown-linux-gnu runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }} @@ -51,6 +53,17 @@ jobs: - arch: x86_64-windows runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "windows", "release"]') || 'windows-2019' }} profile: maxperf + # Debian package builds + - arch: x86_64-unknown-linux-gnu-deb + runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }} + profile: maxperf + rust_target: x86_64-unknown-linux-gnu + gcc_package: gcc + - arch: aarch64-unknown-linux-gnu-deb + runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }} + profile: maxperf + rust_target: aarch64-unknown-linux-gnu + gcc_package: gcc-aarch64-linux-gnu runs-on: ${{ matrix.runner }} needs: extract-version @@ -75,7 +88,25 @@ jobs: run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV # ============================== - # Builds + # Debian build dependencies + # ============================== + + - name: Install Debian build dependencies + if: endsWith(matrix.arch, '-deb') + run: | + sudo apt-get update + sudo apt-get install -y libclang-dev cmake ${{ matrix.gcc_package }} + + - name: Install Rust target for Debian builds + if: endsWith(matrix.arch, '-deb') + run: rustup target add ${{ matrix.rust_target }} + + - name: Install cargo-deb + if: endsWith(matrix.arch, '-deb') + run: cargo install cargo-deb + + # ============================== + # Builds - Binaries # ============================== - name: Build Lighthouse for aarch64-unknown-linux-gnu @@ -91,7 +122,7 @@ jobs: env CROSS_PROFILE=${{ matrix.profile }} make build-x86_64 - name: Move cross-compiled binary - if: contains(matrix.arch, 'unknown-linux-gnu') + if: contains(matrix.arch, 'unknown-linux-gnu') && !endsWith(matrix.arch, '-deb') run: mv target/${{ matrix.arch }}/${{ matrix.profile }}/lighthouse ~/.cargo/bin/lighthouse - name: Build Lighthouse for x86_64-apple-darwin @@ -106,8 +137,44 @@ jobs: if: matrix.arch == 'x86_64-windows' run: cargo install --path lighthouse --force --locked --features portable,gnosis --profile ${{ matrix.profile }} + # ============================== + # Builds - Debian Packages + # ============================== + + - name: Build reproducible Debian package + if: endsWith(matrix.arch, '-deb') + run: | + make deb-cargo RUST_TARGET=${{ matrix.rust_target }} PROFILE=${{ matrix.profile }} + + - name: Find and prepare Debian package + if: endsWith(matrix.arch, '-deb') + run: | + VERSION=${{ needs.extract-version.outputs.VERSION }} + DEB_FILE=$(find target/${{ matrix.rust_target }}/debian -name "*.deb" | head -1) + if [ -n "$DEB_FILE" ]; then + # Create standardized filename to match existing pattern + ARCH_SHORT=$(echo "${{ matrix.rust_target }}" | cut -d'-' -f1) + NEW_NAME="lighthouse-${{ needs.extract-version.outputs.VERSION }}-${ARCH_SHORT}.deb" + cp "$DEB_FILE" "$NEW_NAME" + echo "DEB_PACKAGE=$NEW_NAME" >> $GITHUB_ENV + + # Generate checksums + sha256sum "$NEW_NAME" > "$NEW_NAME.sha256" + sha512sum "$NEW_NAME" > "$NEW_NAME.sha512" + + echo "Debian package built: $NEW_NAME" + echo "Size: $(du -h "$NEW_NAME" | cut -f1)" + else + echo "โŒ No .deb package found" + exit 1 + fi + + # ================================== + # Binary Artifacts (existing) + # ================================== + - name: Configure GPG and create artifacts - if: startsWith(matrix.arch, 'x86_64-windows') != true + if: startsWith(matrix.arch, 'x86_64-windows') != true && !endsWith(matrix.arch, '-deb') env: GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} @@ -140,12 +207,26 @@ jobs: gpg --passphrase "$env:GPG_PASSPHRASE" --batch --pinentry-mode loopback -ab lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz move *tar.gz* .. + # ============================== + # Debian Package Artifacts + # ============================== + + - name: Sign Debian package + if: endsWith(matrix.arch, '-deb') + env: + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} + GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + run: | + export GPG_TTY=$(tty) + echo "$GPG_SIGNING_KEY" | gpg --batch --import + echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab "$DEB_PACKAGE" + # ======================================================================= - # Upload artifacts - # This is required to share artifacts between different jobs + # Upload artifacts - Binaries (existing) # ======================================================================= - name: Upload artifact + if: !endsWith(matrix.arch, '-deb') uses: actions/upload-artifact@v4 with: name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz @@ -153,12 +234,43 @@ jobs: compression-level: 0 - name: Upload signature + if: !endsWith(matrix.arch, '-deb') uses: actions/upload-artifact@v4 with: name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc path: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc compression-level: 0 + # ======================================================================= + # Upload artifacts - Debian packages + # ======================================================================= + + - name: Upload Debian package + if: endsWith(matrix.arch, '-deb') + uses: actions/upload-artifact@v4 + with: + name: ${{ env.DEB_PACKAGE }} + path: ${{ env.DEB_PACKAGE }} + compression-level: 0 + + - name: Upload Debian package signature + if: endsWith(matrix.arch, '-deb') + uses: actions/upload-artifact@v4 + with: + name: ${{ env.DEB_PACKAGE }}.asc + path: ${{ env.DEB_PACKAGE }}.asc + compression-level: 0 + + - name: Upload Debian package checksums + if: endsWith(matrix.arch, '-deb') + uses: actions/upload-artifact@v4 + with: + name: ${{ env.DEB_PACKAGE }}-checksums + path: | + ${{ env.DEB_PACKAGE }}.sha256 + ${{ env.DEB_PACKAGE }}.sha512 + compression-level: 0 + draft-release: name: Draft Release needs: [build, extract-version] @@ -249,11 +361,34 @@ jobs: | Linux logo | x86_64 | [lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc) | | Raspberrypi logo | aarch64 | [lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz.asc) | | Windows logo | x86_64 | [lighthouse-${{ env.VERSION }}-x86_64-windows.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-windows.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-windows.tar.gz.asc) | + + ## Debian Packages + + For Debian/Ubuntu users, we provide pre-built `.deb` packages with systemd integration: + + | System | Architecture | Package | PGP Signature | Checksums | + |:---:|:---:|:---:|:---:|:---:| + | Debian logo | x86_64 | [lighthouse-${{ env.VERSION }}-x86_64.deb](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64.deb) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64.deb.asc) | [SHA256](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64.deb.sha256) / [SHA512](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64.deb.sha512) | + | Debian logo | aarch64 | [lighthouse-${{ env.VERSION }}-aarch64.deb](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64.deb) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64.deb.asc) | [SHA256](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64.deb.sha256) / [SHA512](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64.deb.sha512) | + + ### Installation: + \`\`\`bash + # Download and install (x86_64) + wget https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64.deb + sudo dpkg -i lighthouse-${{ env.VERSION }}-x86_64.deb + sudo apt-get install -f # Fix dependencies if needed + + # Enable and start service + sudo systemctl enable lighthouse + sudo systemctl start lighthouse + \`\`\` + | | | | | + |:---:|:---:|:---:|:---:| | **System** | **Option** | - | **Resource** | | Docker logo | Docker | [${{ env.VERSION }}](https://hub.docker.com/r/${{ env.IMAGE_NAME }}/tags?page=1&ordering=last_updated&name=${{ env.VERSION }}) | [${{ env.IMAGE_NAME }}](https://hub.docker.com/r/${{ env.IMAGE_NAME }}) | ENDBODY ) - assets=(./lighthouse-*.tar.gz*/lighthouse-*.tar.gz*) + assets=(./lighthouse-*.tar.gz*/lighthouse-*.tar.gz* ./lighthouse-*.deb*/lighthouse-*.deb*) tag_name="${{ env.VERSION }}" echo "$body" | gh release create --draft -F "-" "$tag_name" "${assets[@]}" From 211063ae0ec5cf9d8144db199185bb19451dfe59 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Wed, 18 Jun 2025 13:53:35 +0000 Subject: [PATCH 08/33] refinements and following packaging best practices --- Makefile | 100 +++++---------------- lighthouse/Cargo.toml | 11 +-- lighthouse/{ => debian}/lighthouse.service | 6 +- lighthouse/debian/postinst | 25 ++++++ lighthouse/debian/postrm | 17 ++++ lighthouse/debian/prerm | 12 +++ scripts/test-deb-reproducible.sh | 93 +++++++++++++++++++ 7 files changed, 173 insertions(+), 91 deletions(-) rename lighthouse/{ => debian}/lighthouse.service (84%) create mode 100644 lighthouse/debian/postinst create mode 100644 lighthouse/debian/postrm create mode 100644 lighthouse/debian/prerm create mode 100755 scripts/test-deb-reproducible.sh diff --git a/Makefile b/Makefile index 96ccdc80538..564f58f9fc3 100644 --- a/Makefile +++ b/Makefile @@ -342,27 +342,9 @@ clean: make -C $(EF_TESTS) clean make -C $(STATE_TRANSITION_VECTORS) clean -.PHONY: install-cargo-deb deb-cargo deb-cargo-x86_64 deb-cargo-aarch64 deb-cargo-all test-deb-reproducible install-deb-local remove-deb-local clean-deb help-deb -## Install cargo-deb if not present -install-cargo-deb: - @if ! command -v cargo-deb &> /dev/null; then \ - echo "Installing cargo-deb..."; \ - cargo install cargo-deb; \ - else \ - echo "cargo-deb already installed"; \ - fi - -## Build .deb package using cargo-deb with reproducible settings -deb-cargo: install-cargo-deb build-reproducible +.PHONY: deb-cargo +deb-cargo: build-reproducible ## Build .deb package using cargo-deb with reproducible settings @echo "Building .deb package with cargo-deb..." - @if [ ! -f "lighthouse/lighthouse.service" ]; then \ - echo "โŒ lighthouse.service not found in lighthouse/ directory"; \ - exit 1; \ - fi - @if [ ! -f "README.md" ]; then \ - echo "โŒ README.md not found in current directory"; \ - exit 1; \ - fi cd lighthouse && \ SOURCE_DATE_EPOCH=$(SOURCE_DATE) \ @@ -374,65 +356,26 @@ deb-cargo: install-cargo-deb build-reproducible @echo "โœ… Package built successfully!" @find target/$(RUST_TARGET)/debian -name "*.deb" -exec ls -la {} \; -## Build .deb for specific architectures -deb-cargo-x86_64: + +.PHONY: deb-cargo-x86_64 +deb-cargo-x86_64: ## Build .deb for specific architectures $(MAKE) deb-cargo RUST_TARGET=x86_64-unknown-linux-gnu +.PHONY: deb-cargo-aarch64 deb-cargo-aarch64: $(MAKE) deb-cargo RUST_TARGET=aarch64-unknown-linux-gnu +.PHONY: deb-cargo-all deb-cargo-all: deb-cargo-x86_64 deb-cargo-aarch64 -## Test reproducibility of cargo-deb packages -test-deb-reproducible: - @echo "Testing cargo-deb package reproducibility..." - @if ! command -v diffoscope &> /dev/null; then \ - echo "Installing diffoscope..."; \ - sudo apt-get update; \ - sudo apt-get install -y diffoscope binutils-multiarch; \ - fi - - @echo "Building first package..." - @rm -f lighthouse_*.deb lighthouse-deb-*.deb - @$(MAKE) clean || true - @$(MAKE) deb-cargo - @FIRST_PACKAGE=$$(find target/$(RUST_TARGET)/debian -name "*.deb" | head -1); \ - if [ -n "$$FIRST_PACKAGE" ]; then \ - cp "$$FIRST_PACKAGE" ./lighthouse-deb-build-1.deb; \ - else \ - echo "โŒ First package not found"; exit 1; \ - fi - - @echo "Building second package..." - @$(MAKE) clean || true - @$(MAKE) deb-cargo - @SECOND_PACKAGE=$$(find target/$(RUST_TARGET)/debian -name "*.deb" | head -1); \ - if [ -n "$$SECOND_PACKAGE" ]; then \ - cp "$$SECOND_PACKAGE" ./lighthouse-deb-build-2.deb; \ - else \ - echo "โŒ Second package not found"; exit 1; \ - fi - - @echo "Comparing packages..." - @echo "=== Package sizes ===" - @ls -la lighthouse-deb-build-*.deb - @echo "=== SHA256 checksums ===" - @sha256sum lighthouse-deb-build-*.deb - - @if cmp -s lighthouse-deb-build-1.deb lighthouse-deb-build-2.deb; then \ - echo "โœ… SUCCESS: cargo-deb packages are identical!"; \ - echo "โœ… Reproducible build PASSED"; \ - else \ - echo "โŒ FAILED: cargo-deb packages differ"; \ - echo "Running detailed analysis with diffoscope..."; \ - diffoscope --text lighthouse-deb-build-1.deb lighthouse-deb-build-2.deb > cargo-deb-diff.txt || true; \ - echo "Differences saved to cargo-deb-diff.txt"; \ - echo "โŒ Reproducible build FAILED"; \ - exit 1; \ - fi -## Install .deb package locally for testing -install-deb-local: +.PHONY: test-deb-reproducible +test-deb-reproducible: ## Test reproducibility of cargo-deb packages + @./scripts/test-deb-reproducible.sh $(RUST_TARGET) + + +.PHONY: install-deb-local +install-deb-local: ## Install .deb package locally for testing @PACKAGE=$$(find target/$(RUST_TARGET)/debian -name "*.deb" | head -1); \ if [ -n "$$PACKAGE" ]; then \ echo "Installing lighthouse package: $$PACKAGE"; \ @@ -449,22 +392,23 @@ install-deb-local: echo "โŒ No .deb package found. Run 'make deb-cargo' first."; \ fi -## Remove installed lighthouse package -remove-deb-local: + +.PHONY: remove-deb-local +remove-deb-local: ## Remove installed lighthouse package @echo "Removing lighthouse package..." sudo dpkg -r lighthouse || true sudo systemctl daemon-reload || true -## Clean up debian packaging artifacts -clean-deb: +.PHONY: clean-deb +clean-deb: ## Clean up debian packaging artifacts @echo "Cleaning up debian packaging artifacts..." rm -f lighthouse_*.deb lighthouse-deb-*.deb *-diff.txt rm -rf target/*/debian/ -## Show help for debian packaging -help-deb: + +.PHONY: help-deb +help-deb: ## Show help for debian packaging @echo "Clean Debian Packaging Targets:" - @echo " install-cargo-deb - Install cargo-deb tool" @echo " deb-cargo - Build .deb package with cargo-deb" @echo " deb-cargo-x86_64 - Build x86_64 .deb package" @echo " deb-cargo-aarch64 - Build aarch64 .deb package" diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index 956a0d202fb..5bc40fe1f0c 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -95,31 +95,24 @@ name = "lighthouse_tests" path = "tests/main.rs" [package.metadata.deb] -# Package metadata maintainer = "Sigma Prime " extended-description = """\ Lighthouse is a Rust implementation of the Ethereum beacon chain, \ built by Sigma Prime. It implements the official Ethereum 2.0 specification.""" -# Dependencies depends = "$auto, systemd" section = "net" priority = "optional" +maintainer-scripts = "debian/" # This tells cargo-deb where to find scripts # System integration systemd-units = { enable = false, start = false } # Assets to include in the package assets = [ - # Binary - use target/release/ prefix (cargo-deb will replace with actual path) ["target/release/lighthouse", "usr/bin/", "755"], - - # Systemd service (in lighthouse directory) - ["lighthouse.service", "lib/systemd/system/", "644"], - - # Documentation (in parent directory) + ["debian/lighthouse.service", "lib/systemd/system/", "644"], ["../README.md", "usr/share/doc/lighthouse/", "644"], ] -# Use default features - let Makefile handle the building default-features = false diff --git a/lighthouse/lighthouse.service b/lighthouse/debian/lighthouse.service similarity index 84% rename from lighthouse/lighthouse.service rename to lighthouse/debian/lighthouse.service index dfc2b4f9f52..4df28d4d6f0 100644 --- a/lighthouse/lighthouse.service +++ b/lighthouse/debian/lighthouse.service @@ -1,14 +1,12 @@ [Unit] -Description=Lighthouse Ethereum Beacon Chain Client -Documentation=https://lighthouse-book.sigmaprime.io/ +Description=Lighthouse Ethereum Beacon Node After=network.target Wants=network.target [Service] Type=simple -DynamicUser=true User=lighthouse -Group=eth +Group=lighthouse Restart=always RestartSec=5 TimeoutStopSec=180 diff --git a/lighthouse/debian/postinst b/lighthouse/debian/postinst new file mode 100644 index 00000000000..00c089ad912 --- /dev/null +++ b/lighthouse/debian/postinst @@ -0,0 +1,25 @@ +#!/bin/sh +set -e + +# Create lighthouse user if it doesn't exist +if ! getent passwd lighthouse >/dev/null 2>&1; then + adduser --system --group --home /var/lib/lighthouse \ + --shell /bin/false --gecos "Lighthouse Beacon Node" lighthouse +fi + +# Create data directory +mkdir -p /var/lib/lighthouse +chown lighthouse:lighthouse /var/lib/lighthouse +chmod 750 /var/lib/lighthouse + +# Create log directory +mkdir -p /var/log/lighthouse +chown lighthouse:lighthouse /var/log/lighthouse +chmod 750 /var/log/lighthouse + +# Reload systemd +systemctl daemon-reload || true + +#DEBHELPER# + +exit 0 diff --git a/lighthouse/debian/postrm b/lighthouse/debian/postrm new file mode 100644 index 00000000000..88de8652c3e --- /dev/null +++ b/lighthouse/debian/postrm @@ -0,0 +1,17 @@ +#!/bin/sh +set -e + +if [ "$1" = "purge" ]; then + # Remove user (optional - some prefer to keep it) + if getent passwd lighthouse >/dev/null 2>&1; then + deluser lighthouse || true + fi + + # Remove data directory (be careful!) + # rm -rf /var/lib/lighthouse || true + # rm -rf /var/log/lighthouse || true +fi + +#DEBHELPER# + +exit 0 diff --git a/lighthouse/debian/prerm b/lighthouse/debian/prerm new file mode 100644 index 00000000000..f8e1bcfd14a --- /dev/null +++ b/lighthouse/debian/prerm @@ -0,0 +1,12 @@ +#!/bin/sh +set -e + +if [ "$1" = "remove" ]; then + # Stop service if running + systemctl stop lighthouse || true + systemctl disable lighthouse || true +fi + +#DEBHELPER# + +exit 0 diff --git a/scripts/test-deb-reproducible.sh b/scripts/test-deb-reproducible.sh new file mode 100755 index 00000000000..b9b42555c7d --- /dev/null +++ b/scripts/test-deb-reproducible.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# +# Test reproducibility of cargo-deb packages +# +# Usage: ./scripts/test-deb-reproducible.sh [RUST_TARGET] +# +# This script builds the same Debian package twice with clean builds in between, +# then compares them to ensure reproducible builds are working correctly. + +set -euo pipefail + +# Default target if not provided +RUST_TARGET="${1:-x86_64-unknown-linux-gnu}" + +echo "๐Ÿ”„ Testing cargo-deb package reproducibility for ${RUST_TARGET}..." + +# Check if diffoscope is available, install if needed +if ! command -v diffoscope &> /dev/null; then + echo "๐Ÿ“ฆ Installing diffoscope..." + sudo apt-get update + sudo apt-get install -y diffoscope binutils-multiarch +fi + +# Clean up any existing test artifacts +echo "๐Ÿงน Cleaning up previous test artifacts..." +rm -f lighthouse_*.deb lighthouse-deb-*.deb *-diff.txt + +# Build first package +echo "๐Ÿ”จ Building first package..." +make clean || true +make deb-cargo RUST_TARGET="${RUST_TARGET}" + +FIRST_PACKAGE=$(find target/"${RUST_TARGET}"/debian -name "*.deb" | head -1) +if [ -n "$FIRST_PACKAGE" ]; then + cp "$FIRST_PACKAGE" ./lighthouse-deb-build-1.deb + echo "โœ… First package built: $(basename "$FIRST_PACKAGE")" +else + echo "โŒ First package not found" + exit 1 +fi + +# Build second package +echo "๐Ÿ”จ Building second package..." +make clean || true +make deb-cargo RUST_TARGET="${RUST_TARGET}" + +SECOND_PACKAGE=$(find target/"${RUST_TARGET}"/debian -name "*.deb" | head -1) +if [ -n "$SECOND_PACKAGE" ]; then + cp "$SECOND_PACKAGE" ./lighthouse-deb-build-2.deb + echo "โœ… Second package built: $(basename "$SECOND_PACKAGE")" +else + echo "โŒ Second package not found" + exit 1 +fi + +# Compare packages +echo "๐Ÿ“Š Comparing packages..." +echo "" +echo "=== Package sizes ===" +ls -lah lighthouse-deb-build-*.deb + +echo "" +echo "=== SHA256 checksums ===" +sha256sum lighthouse-deb-build-*.deb + +echo "" +echo "=== Binary comparison ===" +if cmp -s lighthouse-deb-build-1.deb lighthouse-deb-build-2.deb; then + echo "โœ… SUCCESS: cargo-deb packages are identical!" + echo "โœ… Reproducible build PASSED for ${RUST_TARGET}" + echo "" + echo "๐Ÿงน Cleaning up test artifacts..." + rm -f lighthouse-deb-build-*.deb + exit 0 +else + echo "โŒ FAILED: cargo-deb packages differ" + echo "๐Ÿ” Running detailed analysis with diffoscope..." + + # Generate detailed diff report + DIFF_FILE="cargo-deb-diff-${RUST_TARGET}.txt" + if diffoscope --text lighthouse-deb-build-1.deb lighthouse-deb-build-2.deb > "$DIFF_FILE" 2>/dev/null; then + echo "๐Ÿ“„ Differences saved to ${DIFF_FILE}" + echo "๐Ÿ“„ Summary of first few differences:" + head -20 "$DIFF_FILE" || true + else + echo "โš ๏ธ diffoscope encountered issues, but differences were detected" + fi + + echo "" + echo "โŒ Reproducible build FAILED for ${RUST_TARGET}" + echo "๐Ÿ’ก Tip: Check the diff file for details on what differs between builds" + exit 1 +fi From d93242f508f2d57c6b28abdb670666612251842d Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Tue, 24 Jun 2025 20:42:01 +0000 Subject: [PATCH 09/33] refine the reproducible build and test arm github runner --- .github/workflows/reproducible-build.yml | 236 ++++++++++++---------- testing/state_transition_vectors/Makefile | 2 +- 2 files changed, 133 insertions(+), 105 deletions(-) diff --git a/.github/workflows/reproducible-build.yml b/.github/workflows/reproducible-build.yml index 843e9246966..abe37510243 100644 --- a/.github/workflows/reproducible-build.yml +++ b/.github/workflows/reproducible-build.yml @@ -19,142 +19,169 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 with: - target: x86_64-unknown-linux-gnu + driver: docker # Use docker driver instead of docker-container for reproducibility - - name: Install build dependencies + - name: Build first reproducible Docker image run: | - sudo apt-get update - sudo apt-get install -y libclang-dev cmake + echo "=== Building first Docker image (x86_64) ===" + docker build -f Dockerfile.reproducible \ + --build-arg RUST_TARGET="x86_64-unknown-linux-gnu" \ + --build-arg RUST_IMAGE="rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1" \ + -t build-lighthouse-1 . + + echo "=== Extracting binary from first build ===" + docker create --name extract-lighthouse-1 build-lighthouse-1 + docker cp extract-lighthouse-1:/lighthouse ./lighthouse-build-1 + docker rm extract-lighthouse-1 + + echo "=== First build info ===" + ls -la lighthouse-build-1 + sha256sum lighthouse-build-1 + file lighthouse-build-1 + + - name: Clean Docker state completely + run: | + echo "=== Cleaning Docker state ===" + # Remove the first image + docker rmi build-lighthouse-1 || true - - name: Install cargo-cache - run: cargo install cargo-cache + # Remove all build cache (important for reproducibility testing) + docker buildx prune -f || true + docker system prune -f || true - - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: true - key: reproducible-build-x86_64 + # Clear any remaining containers + docker container prune -f || true - - name: Build Lighthouse (first build) - run: | - make build-reproducible \ - RUST_TARGET=x86_64-unknown-linux-gnu - cp target/x86_64-unknown-linux-gnu/release/lighthouse \ - lighthouse-build-1 - sha256sum lighthouse-build-1 > lighthouse-build-1.sha256 + echo "=== Docker state cleaned ===" + docker images + docker ps -a - - name: Clean build artifacts and cache + - name: Build second reproducible Docker image run: | - make clean - cargo cache -a - rm -rf target/ - - - name: Build Lighthouse (second build) + echo "=== Building second Docker image (x86_64) ===" + docker build -f Dockerfile.reproducible \ + --build-arg RUST_TARGET="x86_64-unknown-linux-gnu" \ + --build-arg RUST_IMAGE="rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1" \ + -t build-lighthouse-2 . + + echo "=== Extracting binary from second build ===" + docker create --name extract-lighthouse-2 build-lighthouse-2 + docker cp extract-lighthouse-2:/lighthouse ./lighthouse-build-2 + docker rm extract-lighthouse-2 + + echo "=== Second build info ===" + ls -la lighthouse-build-2 + sha256sum lighthouse-build-2 + file lighthouse-build-2 + + - name: Compare Docker-built binaries run: | - make build-reproducible \ - RUST_TARGET=x86_64-unknown-linux-gnu - cp target/x86_64-unknown-linux-gnu/release/lighthouse \ - lighthouse-build-2 - sha256sum lighthouse-build-2 > lighthouse-build-2.sha256 + echo "=== Comparing Docker-built binaries (x86_64) ===" + echo "Build 1 info:" + ls -la lighthouse-build-1 + echo "Build 2 info:" + ls -la lighthouse-build-2 - - name: Compare binaries - run: | - echo "=== Build 1 SHA256 ===" - cat lighthouse-build-1.sha256 - echo "=== Build 2 SHA256 ===" - cat lighthouse-build-2.sha256 - echo "=== Binary Comparison ===" + echo "=== SHA256 checksums ===" + sha256sum lighthouse-build-* + + echo "=== Binary comparison ===" if cmp lighthouse-build-1 lighthouse-build-2; then - echo "โœ… Binaries are identical - reproducible build PASSED" + echo "โœ… SUCCESS: Docker-built binaries are identical!" + echo "โœ… Reproducible Docker build PASSED for x86_64" else - echo "โŒ Binaries differ - reproducible build FAILED" + echo "โŒ FAILED: Docker-built binaries differ" + echo "First 10 differences:" + cmp -l lighthouse-build-1 lighthouse-build-2 | head -10 exit 1 fi - - name: Upload build artifacts (on failure) - if: failure() - uses: actions/upload-artifact@v4 - with: - name: failed-reproducible-builds-x86_64 - path: | - lighthouse-build-1 - lighthouse-build-2 - lighthouse-build-1.sha256 - lighthouse-build-2.sha256 - build-aarch64: name: test reproducible builds (aarch64) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04-arm steps: - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 with: - target: aarch64-unknown-linux-gnu + driver: docker - - name: Install build dependencies and cross-compilation tools + - name: Build first reproducible Docker image run: | - sudo apt-get update - sudo apt-get install -y libclang-dev cmake gcc-aarch64-linux-gnu - - - name: Install cargo-cache - run: cargo install cargo-cache + echo "=== Building first Docker image (aarch64) ===" + docker build -f Dockerfile.reproducible \ + --platform linux/arm64 \ + --build-arg RUST_TARGET="aarch64-unknown-linux-gnu" \ + --build-arg RUST_IMAGE="rust:1.86-bullseye@sha256:36053eabadeb701e3e0406610a2ce72ccfa10b7828963cd08cffdcf660518b27" \ + -t build-lighthouse-1-arm64 . + + echo "=== Extracting binary from first build ===" + docker create --name extract-lighthouse-1-arm64 build-lighthouse-1-arm64 + docker cp extract-lighthouse-1-arm64:/lighthouse ./lighthouse-build-1-arm64 + docker rm extract-lighthouse-1-arm64 + + echo "=== First build info ===" + ls -la lighthouse-build-1-arm64 + sha256sum lighthouse-build-1-arm64 + file lighthouse-build-1-arm64 + + - name: Clean Docker state completely + run: | + echo "=== Cleaning Docker state ===" + docker rmi build-lighthouse-1-arm64 || true + docker buildx prune -f || true + docker system prune -f || true + docker container prune -f || true - - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: true - key: reproducible-build-aarch64 + echo "=== Docker state cleaned ===" + docker images + docker ps -a - - name: Build Lighthouse (first build) + - name: Build second reproducible Docker image run: | - make build-reproducible \ - RUST_TARGET=aarch64-unknown-linux-gnu - cp target/aarch64-unknown-linux-gnu/release/lighthouse \ - lighthouse-build-1-arm64 - sha256sum lighthouse-build-1-arm64 > \ - lighthouse-build-1-arm64.sha256 - - - name: Clean build artifacts and cache + echo "=== Building second Docker image (aarch64) ===" + docker build -f Dockerfile.reproducible \ + --platform linux/arm64 \ + --build-arg RUST_TARGET="aarch64-unknown-linux-gnu" \ + --build-arg RUST_IMAGE="rust:1.86-bullseye@sha256:36053eabadeb701e3e0406610a2ce72ccfa10b7828963cd08cffdcf660518b27" \ + -t build-lighthouse-2-arm64 . + + echo "=== Extracting binary from second build ===" + docker create --name extract-lighthouse-2-arm64 build-lighthouse-2-arm64 + docker cp extract-lighthouse-2-arm64:/lighthouse ./lighthouse-build-2-arm64 + docker rm extract-lighthouse-2-arm64 + + echo "=== Second build info ===" + ls -la lighthouse-build-2-arm64 + sha256sum lighthouse-build-2-arm64 + file lighthouse-build-2-arm64 + + - name: Compare Docker-built binaries run: | - make clean - cargo cache -a - rm -rf target/ + echo "=== Comparing Docker-built binaries (aarch64) ===" + echo "Build 1 info:" + ls -la lighthouse-build-1-arm64 + echo "Build 2 info:" + ls -la lighthouse-build-2-arm64 - - name: Build Lighthouse (second build) - run: | - make build-reproducible \ - RUST_TARGET=aarch64-unknown-linux-gnu - cp target/aarch64-unknown-linux-gnu/release/lighthouse \ - lighthouse-build-2-arm64 - sha256sum lighthouse-build-2-arm64 > \ - lighthouse-build-2-arm64.sha256 - - - name: Compare binaries - run: | - echo "=== Build 1 SHA256 (ARM64) ===" - cat lighthouse-build-1-arm64.sha256 - echo "=== Build 2 SHA256 (ARM64) ===" - cat lighthouse-build-2-arm64.sha256 - echo "=== Binary Comparison ===" + echo "=== SHA256 checksums ===" + sha256sum lighthouse-build-*-arm64 + + echo "=== Binary comparison ===" if cmp lighthouse-build-1-arm64 lighthouse-build-2-arm64; then - echo "โœ… ARM64 binaries are identical - reproducible build PASSED" + echo "โœ… SUCCESS: Docker-built binaries are identical!" + echo "โœ… Reproducible Docker build PASSED for aarch64" else - echo "โŒ ARM64 binaries differ - reproducible build FAILED" + echo "โŒ FAILED: Docker-built binaries differ" + echo "First 10 differences:" + cmp -l lighthouse-build-1-arm64 lighthouse-build-2-arm64 | head -10 exit 1 fi - - name: Upload build artifacts (on failure) - if: failure() - uses: actions/upload-artifact@v4 - with: - name: failed-reproducible-builds-aarch64 - path: | - lighthouse-build-1-arm64 - lighthouse-build-2-arm64 - lighthouse-build-1-arm64.sha256 - lighthouse-build-2-arm64.sha256 - summary: name: reproducible build summary runs-on: ubuntu-latest @@ -181,8 +208,9 @@ jobs: if [[ "${{ needs.build-x86_64.result }}" == "success" ]] \ && [[ "${{ needs.build-aarch64.result }}" == "success" ]]; then echo "๐ŸŽ‰ **Overall**: All reproducible builds are working correctly!" + echo "Docker containers are reproducible." else echo "โš ๏ธ **Overall**: Some reproducible builds failed" - echo "Check the logs above" + echo "Check the logs above for details" exit 1 fi diff --git a/testing/state_transition_vectors/Makefile b/testing/state_transition_vectors/Makefile index 437aa50b00a..c90810ad398 100644 --- a/testing/state_transition_vectors/Makefile +++ b/testing/state_transition_vectors/Makefile @@ -5,4 +5,4 @@ test: cargo test --release --features "$(TEST_FEATURES)" clean: - rm -r vectors/ + rm -rf vectors/ From febc1387bbaab7eb7b1d4e395dd8d239ccdfb121 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Mon, 30 Jun 2025 12:50:50 +0000 Subject: [PATCH 10/33] make release-reproducible verify reproducibility --- .github/workflows/release-reproducible.yml | 195 ++++++++++++++++----- 1 file changed, 153 insertions(+), 42 deletions(-) diff --git a/.github/workflows/release-reproducible.yml b/.github/workflows/release-reproducible.yml index 10f668869ad..438dca5f4b5 100644 --- a/.github/workflows/release-reproducible.yml +++ b/.github/workflows/release-reproducible.yml @@ -30,9 +30,8 @@ jobs: outputs: VERSION: ${{ steps.extract_version.outputs.VERSION }} - build-reproducible: - name: build and push reproducible images - runs-on: ubuntu-latest + verify-and-build: + name: verify reproducibility and build needs: extract-version strategy: matrix: @@ -43,16 +42,81 @@ jobs: rust_image: >- rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1 platform: linux/amd64 + runner: ubuntu-latest - arch: arm64 rust_target: aarch64-unknown-linux-gnu rust_image: >- rust:1.86-bullseye@sha256:36053eabadeb701e3e0406610a2ce72ccfa10b7828963cd08cffdcf660518b27 platform: linux/arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + driver: docker + + - name: Verify reproducible builds (${{ matrix.arch }}) + run: | + echo "๐Ÿ”„ Verifying reproducible builds for ${{ matrix.arch }}..." + + # Build first image + echo "=== Building first verification image ===" + docker build -f Dockerfile.reproducible \ + --platform ${{ matrix.platform }} \ + --build-arg RUST_TARGET="${{ matrix.rust_target }}" \ + --build-arg RUST_IMAGE="${{ matrix.rust_image }}" \ + -t lighthouse-verify-1-${{ matrix.arch }} . + + # Extract binary from first build + docker create --name extract-1-${{ matrix.arch }} lighthouse-verify-1-${{ matrix.arch }} + docker cp extract-1-${{ matrix.arch }}:/lighthouse ./lighthouse-1-${{ matrix.arch }} + docker rm extract-1-${{ matrix.arch }} + + # Clean state for second build + docker buildx prune -f + docker system prune -f + + # Build second image + echo "=== Building second verification image ===" + docker build -f Dockerfile.reproducible \ + --platform ${{ matrix.platform }} \ + --build-arg RUST_TARGET="${{ matrix.rust_target }}" \ + --build-arg RUST_IMAGE="${{ matrix.rust_image }}" \ + -t lighthouse-verify-2-${{ matrix.arch }} . + + # Extract binary from second build + docker create --name extract-2-${{ matrix.arch }} lighthouse-verify-2-${{ matrix.arch }} + docker cp extract-2-${{ matrix.arch }}:/lighthouse ./lighthouse-2-${{ matrix.arch }} + docker rm extract-2-${{ matrix.arch }} + + # Compare binaries + echo "=== Comparing binaries ===" + echo "Build 1 SHA256: $(sha256sum lighthouse-1-${{ matrix.arch }})" + echo "Build 2 SHA256: $(sha256sum lighthouse-2-${{ matrix.arch }})" + + if cmp lighthouse-1-${{ matrix.arch }} lighthouse-2-${{ matrix.arch }}; then + echo "โœ… Reproducible build verified for ${{ matrix.arch }}" + else + echo "โŒ Reproducible build FAILED for ${{ matrix.arch }}" + echo "๐Ÿšจ BLOCKING RELEASE: Builds are not reproducible!" + echo "First 10 differences:" + cmp -l lighthouse-1-${{ matrix.arch }} lighthouse-2-${{ matrix.arch }} | head -10 + exit 1 + fi + + # Clean up verification artifacts but keep one image for publishing + rm -f lighthouse-*-${{ matrix.arch }} + docker rmi lighthouse-verify-1-${{ matrix.arch }} || true + + # Re-tag the second image for publishing (we verified it's identical to first) + VERSION=${{ needs.extract-version.outputs.VERSION }} + FINAL_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}" + docker tag lighthouse-verify-2-${{ matrix.arch }} "$FINAL_TAG" + + echo "โœ… Image ready for publishing: $FINAL_TAG" - name: Log in to Docker Hub if: ${{ github.event.inputs.dry_run != 'true' }} @@ -61,30 +125,34 @@ jobs: username: ${{ env.DOCKER_USERNAME }} password: ${{ env.DOCKER_PASSWORD }} - - name: Build reproducible image (${{ matrix.arch }}) - uses: docker/build-push-action@v6 - env: - IMAGE_BASE: ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }} - VERSION: ${{ needs.extract-version.outputs.VERSION }} - ARCH: ${{ matrix.arch }} - DOCKER_BUILD_RECORD_UPLOAD: false + - name: Push verified image (${{ matrix.arch }}) + if: ${{ github.event.inputs.dry_run != 'true' }} + run: | + VERSION=${{ needs.extract-version.outputs.VERSION }} + IMAGE_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}" + + echo "๐Ÿ“ค Pushing verified reproducible image: $IMAGE_TAG" + docker push "$IMAGE_TAG" + echo "โœ… Successfully pushed $IMAGE_TAG" + + - name: Clean up local images + run: | + docker rmi lighthouse-verify-2-${{ matrix.arch }} || true + VERSION=${{ needs.extract-version.outputs.VERSION }} + docker rmi "${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}" || true + + - name: Upload verification artifacts (on failure) + if: failure() + uses: actions/upload-artifact@v4 with: - context: . - file: ./Dockerfile.reproducible - platforms: ${{ matrix.platform }} - push: ${{ github.event.inputs.dry_run != 'true' }} - tags: ${{ env.IMAGE_BASE }}:${{ env.VERSION }}-${{ env.ARCH }} - build-args: | - RUST_TARGET=${{ matrix.rust_target }} - RUST_IMAGE=${{ matrix.rust_image }} - cache-from: type=gha,scope=${{ matrix.arch }} - cache-to: type=gha,mode=max,scope=${{ matrix.arch }} - provenance: false + name: verification-failure-${{ matrix.arch }} + path: | + lighthouse-*-${{ matrix.arch }} create-manifest: name: create multi-arch manifest runs-on: ubuntu-latest - needs: [extract-version, build-reproducible] + needs: [extract-version, verify-and-build] if: ${{ github.event.inputs.dry_run != 'true' }} steps: - name: Log in to Docker Hub @@ -97,6 +165,9 @@ jobs: run: | IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }} VERSION=${{ needs.extract-version.outputs.VERSION }} + + echo "๐Ÿ”— Creating multi-arch manifest for $IMAGE_NAME:$VERSION" + # Create manifest for version tag docker manifest create \ ${IMAGE_NAME}:${VERSION} \ @@ -113,30 +184,70 @@ jobs: docker manifest push ${IMAGE_NAME}:latest - dry-run-summary: - name: dry run summary + echo "โœ… Multi-arch manifests published:" + echo " - ${IMAGE_NAME}:${VERSION}" + echo " - ${IMAGE_NAME}:latest" + + release-summary: + name: release summary runs-on: ubuntu-latest - needs: [build-reproducible, extract-version] - if: ${{ github.event.inputs.dry_run == 'true' }} + needs: [extract-version, verify-and-build, create-manifest] + if: always() steps: - - name: Summarize dry run + - name: Report release results run: | - IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }} VERSION=${{ needs.extract-version.outputs.VERSION }} - echo "## ๐Ÿงช Reproducible Build Dry Run Summary" - echo "" - echo "โœ… Successfully completed dry run for version ${VERSION}" + IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }} + + echo "## ๐Ÿš€ Reproducible Release Summary for ${VERSION}" echo "" - echo "### What would happen in a real release:" - echo "- Multi-arch reproducible Docker images would be built" - echo "- Images would be pushed to Docker Hub as:" - echo " - \`${IMAGE_NAME}:${VERSION}\`" - echo " - \`${IMAGE_NAME}:latest\`" + + if [[ "${{ needs.verify-and-build.result }}" == "success" ]]; then + echo "โœ… **Reproducibility Verification & Build**: SUCCESS" + echo "- All architectures produce identical binaries" + echo "- Images built and ready for publishing" + else + echo "โŒ **Reproducibility Verification & Build**: FAILED" + echo "- Builds are not reproducible OR build failed" + echo "- Release was blocked" + fi + echo "" - echo "### Architectures built:" - echo "- linux/amd64 (x86_64-unknown-linux-gnu)" - echo "- linux/arm64 (aarch64-unknown-linux-gnu)" + if [[ "${{ github.event.inputs.dry_run }}" == "true" ]]; then + echo "๐Ÿงช **Mode**: DRY RUN" + echo "- Images were built and verified but NOT pushed" + echo "- Ready for real release" + elif [[ "${{ needs.create-manifest.result }}" == "success" ]]; then + echo "โœ… **Publication**: SUCCESS" + echo "- Images published to Docker Hub" + echo "- Multi-arch manifests created" + echo "" + echo "### ๐Ÿ“ฆ Published Images" + echo "- \`${IMAGE_NAME}:${VERSION}\`" + echo "- \`${IMAGE_NAME}:latest\`" + echo "" + echo "### ๐Ÿ—๏ธ Architectures" + echo "- linux/amd64 (\`${IMAGE_NAME}:${VERSION}-amd64\`)" + echo "- linux/arm64 (\`${IMAGE_NAME}:${VERSION}-arm64\`)" + else + echo "โŒ **Publication**: FAILED" + echo "- Images were verified but failed to publish" + fi + echo "" - echo "### Next Steps" - echo "To perform a real release, push a git tag" - echo "(e.g., \`git tag v4.6.0 && git push origin v4.6.0\`)" + if [[ "${{ needs.verify-and-build.result }}" == "success" ]] && [[ "${{ needs.create-manifest.result }}" == "success" ]] && [[ "${{ github.event.inputs.dry_run }}" != "true" ]]; then + echo "๐ŸŽ‰ **Overall**: Secure release completed successfully!" + echo "" + echo "### ๐Ÿ”’ Security Guarantees" + echo "- โœ… Reproducible builds verified" + echo "- โœ… Identical binaries across architectures" + echo "- โœ… No build artifacts tampering" + echo "- โœ… Deterministic build process" + elif [[ "${{ github.event.inputs.dry_run }}" == "true" ]]; then + echo "๐Ÿงช **Overall**: Dry run completed successfully!" + echo "- Reproducibility verified โœ…" + echo "- Ready for real release โœ…" + else + echo "๐Ÿšจ **Overall**: Release failed or incomplete" + echo "- Check logs above for details" + fi From 19f2a0f823a280b8385b62f7bf05af1d381e34f3 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Mon, 30 Jun 2025 13:27:35 +0000 Subject: [PATCH 11/33] add label trigger for reproducible builds --- .github/workflows/reproducible-build.yml | 33 ++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reproducible-build.yml b/.github/workflows/reproducible-build.yml index abe37510243..6c7e0ca19a7 100644 --- a/.github/workflows/reproducible-build.yml +++ b/.github/workflows/reproducible-build.yml @@ -3,8 +3,9 @@ name: reproducible-build on: workflow_dispatch: {} schedule: - - cron: "0 1 */2 * *" + - cron: "0 1 */2 * *" # Every 2 days to catch issues early pull_request: + types: [opened, synchronize, labeled] paths: - "Makefile" - "Dockerfile.reproducible" @@ -16,6 +17,11 @@ jobs: build-x86_64: name: test reproducible builds (x86_64) runs-on: ubuntu-latest + # Run on schedule, workflow_dispatch, or PRs with the 'test-reproducible' label + if: > + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'test-reproducible') steps: - uses: actions/checkout@v4 @@ -102,6 +108,11 @@ jobs: build-aarch64: name: test reproducible builds (aarch64) runs-on: ubuntu-24.04-arm + # Run on schedule, workflow_dispatch, or PRs with the 'test-reproducible' label + if: > + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'test-reproducible') steps: - uses: actions/checkout@v4 @@ -192,14 +203,29 @@ jobs: run: | echo "## ๐Ÿ”„ Reproducible Build Test Results" echo "" + + # Show trigger reason + if [[ "${{ github.event_name }}" == "schedule" ]]; then + echo "**Trigger**: Scheduled check (every 2 days)" + elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "**Trigger**: Manual run" + elif [[ "${{ github.event_name }}" == "pull_request" ]]; then + echo "**Trigger**: PR with 'test-reproducible' label" + fi + echo "" + if [[ "${{ needs.build-x86_64.result }}" == "success" ]]; then echo "โœ… **x86_64**: Reproducible builds PASSED" + elif [[ "${{ needs.build-x86_64.result }}" == "skipped" ]]; then + echo "โญ๏ธ **x86_64**: Skipped (no label or wrong trigger)" else echo "โŒ **x86_64**: Reproducible builds FAILED" fi if [[ "${{ needs.build-aarch64.result }}" == "success" ]]; then echo "โœ… **aarch64**: Reproducible builds PASSED" + elif [[ "${{ needs.build-aarch64.result }}" == "skipped" ]]; then + echo "โญ๏ธ **aarch64**: Skipped (no label or wrong trigger)" else echo "โŒ **aarch64**: Reproducible builds FAILED" fi @@ -209,8 +235,11 @@ jobs: && [[ "${{ needs.build-aarch64.result }}" == "success" ]]; then echo "๐ŸŽ‰ **Overall**: All reproducible builds are working correctly!" echo "Docker containers are reproducible." + elif [[ "${{ needs.build-x86_64.result }}" == "skipped" ]] \ + && [[ "${{ needs.build-aarch64.result }}" == "skipped" ]]; then + echo "โญ๏ธ **Overall**: Tests were skipped (add 'test-reproducible' label to run on PRs)" else - echo "โš ๏ธ **Overall**: Some reproducible builds failed" + echo "โš ๏ธ **Overall**: Some reproducible builds failed" echo "Check the logs above for details" exit 1 fi From c772026e012d023adab0c4fb9d3dadda3dc62382 Mon Sep 17 00:00:00 2001 From: Moe Mahhouk Date: Mon, 6 Oct 2025 12:30:32 +0200 Subject: [PATCH 12/33] Update lighthouse.service simplified the lighthouse systemd service --- lighthouse/debian/lighthouse.service | 30 +++++++--------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/lighthouse/debian/lighthouse.service b/lighthouse/debian/lighthouse.service index 4df28d4d6f0..185d202de57 100644 --- a/lighthouse/debian/lighthouse.service +++ b/lighthouse/debian/lighthouse.service @@ -4,29 +4,13 @@ After=network.target Wants=network.target [Service] -Type=simple -User=lighthouse -Group=lighthouse -Restart=always -RestartSec=5 -TimeoutStopSec=180 -Environment="RUST_LOG=info" -ExecStart=/usr/bin/lighthouse \ - bn \ - --checkpoint-sync-url https://sync-mainnet.beaconcha.in \ - --execution-endpoint http://localhost:8551 \ - --execution-jwt /etc/jwt.hex \ - --suggested-fee-recipient 0x0 \ - --always-prepare-payload \ - --prepare-payload-lookahead 8000 \ - --disable-deposit-contract-sync \ - --http \ - --port 9000 \ - --http-port 3500 \ - --metrics \ - --metrics-address 127.0.0.1 \ - --metrics-port 5054 \ - --datadir /mnt/data/lighthouse +Type=exec +DynamicUser=yes +StateDirectory=lighthouse +ExecStart=/usr/bin/lighthouse bn \ + --execution-endpoint http://localhost:8551 \ + --execution-jwt-secret-key 0000000000000000000000000000000000000000000000000000000000000000 + --datadir %S/lighthouse [Install] WantedBy=default.target From bddc4f7a68c0413f1a12812cee8a2e2d42433a5b Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Mon, 3 Nov 2025 15:46:31 +0000 Subject: [PATCH 13/33] update rust version for github actions --- .github/workflows/release-reproducible.yml | 4 ++-- .github/workflows/reproducible-build.yml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release-reproducible.yml b/.github/workflows/release-reproducible.yml index 438dca5f4b5..5021161588e 100644 --- a/.github/workflows/release-reproducible.yml +++ b/.github/workflows/release-reproducible.yml @@ -40,13 +40,13 @@ jobs: - arch: amd64 rust_target: x86_64-unknown-linux-gnu rust_image: >- - rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1 + rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9e315fd2cb5100e87a7187a9816 platform: linux/amd64 runner: ubuntu-latest - arch: arm64 rust_target: aarch64-unknown-linux-gnu rust_image: >- - rust:1.86-bullseye@sha256:36053eabadeb701e3e0406610a2ce72ccfa10b7828963cd08cffdcf660518b27 + rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94 platform: linux/arm64 runner: ubuntu-24.04-arm runs-on: ${{ matrix.runner }} diff --git a/.github/workflows/reproducible-build.yml b/.github/workflows/reproducible-build.yml index 6c7e0ca19a7..dd999772660 100644 --- a/.github/workflows/reproducible-build.yml +++ b/.github/workflows/reproducible-build.yml @@ -35,7 +35,7 @@ jobs: echo "=== Building first Docker image (x86_64) ===" docker build -f Dockerfile.reproducible \ --build-arg RUST_TARGET="x86_64-unknown-linux-gnu" \ - --build-arg RUST_IMAGE="rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1" \ + --build-arg RUST_IMAGE="rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9e315fd2cb5100e87a7187a9816" \ -t build-lighthouse-1 . echo "=== Extracting binary from first build ===" @@ -70,7 +70,7 @@ jobs: echo "=== Building second Docker image (x86_64) ===" docker build -f Dockerfile.reproducible \ --build-arg RUST_TARGET="x86_64-unknown-linux-gnu" \ - --build-arg RUST_IMAGE="rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1" \ + --build-arg RUST_IMAGE="rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9e315fd2cb5100e87a7187a9816" \ -t build-lighthouse-2 . echo "=== Extracting binary from second build ===" @@ -127,7 +127,7 @@ jobs: docker build -f Dockerfile.reproducible \ --platform linux/arm64 \ --build-arg RUST_TARGET="aarch64-unknown-linux-gnu" \ - --build-arg RUST_IMAGE="rust:1.86-bullseye@sha256:36053eabadeb701e3e0406610a2ce72ccfa10b7828963cd08cffdcf660518b27" \ + --build-arg RUST_IMAGE="rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94" \ -t build-lighthouse-1-arm64 . echo "=== Extracting binary from first build ===" @@ -158,7 +158,7 @@ jobs: docker build -f Dockerfile.reproducible \ --platform linux/arm64 \ --build-arg RUST_TARGET="aarch64-unknown-linux-gnu" \ - --build-arg RUST_IMAGE="rust:1.86-bullseye@sha256:36053eabadeb701e3e0406610a2ce72ccfa10b7828963cd08cffdcf660518b27" \ + --build-arg RUST_IMAGE="rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94" \ -t build-lighthouse-2-arm64 . echo "=== Extracting binary from second build ===" From 5e21162dc244baf86a847b666bbb7263d606771c Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Wed, 5 Nov 2025 16:04:23 +0000 Subject: [PATCH 14/33] addressing PR feedback --- ...producible.yml => docker-reproducible.yml} | 66 +++++++++--------- .github/workflows/reproducible-build.yml | 67 +++---------------- Dockerfile.reproducible | 2 +- Makefile | 5 +- 4 files changed, 43 insertions(+), 97 deletions(-) rename .github/workflows/{release-reproducible.yml => docker-reproducible.yml} (81%) diff --git a/.github/workflows/release-reproducible.yml b/.github/workflows/docker-reproducible.yml similarity index 81% rename from .github/workflows/release-reproducible.yml rename to .github/workflows/docker-reproducible.yml index 5021161588e..4d0c8138971 100644 --- a/.github/workflows/release-reproducible.yml +++ b/.github/workflows/docker-reproducible.yml @@ -1,4 +1,4 @@ -name: release-reproducible +name: docker-reproducible on: push: @@ -21,7 +21,7 @@ env: jobs: extract-version: name: extract version - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Extract version run: >- @@ -42,13 +42,13 @@ jobs: rust_image: >- rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9e315fd2cb5100e87a7187a9816 platform: linux/amd64 - runner: ubuntu-latest + runner: ubuntu-22.04 - arch: arm64 rust_target: aarch64-unknown-linux-gnu rust_image: >- rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94 platform: linux/arm64 - runner: ubuntu-24.04-arm + runner: ubuntu-22.04-arm runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 @@ -60,7 +60,7 @@ jobs: - name: Verify reproducible builds (${{ matrix.arch }}) run: | - echo "๐Ÿ”„ Verifying reproducible builds for ${{ matrix.arch }}..." + echo "Verifying reproducible builds for ${{ matrix.arch }}..." # Build first image echo "=== Building first verification image ===" @@ -98,10 +98,10 @@ jobs: echo "Build 2 SHA256: $(sha256sum lighthouse-2-${{ matrix.arch }})" if cmp lighthouse-1-${{ matrix.arch }} lighthouse-2-${{ matrix.arch }}; then - echo "โœ… Reproducible build verified for ${{ matrix.arch }}" + echo "Reproducible build verified for ${{ matrix.arch }}" else - echo "โŒ Reproducible build FAILED for ${{ matrix.arch }}" - echo "๐Ÿšจ BLOCKING RELEASE: Builds are not reproducible!" + echo "Reproducible build FAILED for ${{ matrix.arch }}" + echo "BLOCKING RELEASE: Builds are not reproducible!" echo "First 10 differences:" cmp -l lighthouse-1-${{ matrix.arch }} lighthouse-2-${{ matrix.arch }} | head -10 exit 1 @@ -116,7 +116,7 @@ jobs: FINAL_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}" docker tag lighthouse-verify-2-${{ matrix.arch }} "$FINAL_TAG" - echo "โœ… Image ready for publishing: $FINAL_TAG" + echo "Image ready for publishing: $FINAL_TAG" - name: Log in to Docker Hub if: ${{ github.event.inputs.dry_run != 'true' }} @@ -131,9 +131,9 @@ jobs: VERSION=${{ needs.extract-version.outputs.VERSION }} IMAGE_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}" - echo "๐Ÿ“ค Pushing verified reproducible image: $IMAGE_TAG" + echo "Pushing verified reproducible image: $IMAGE_TAG" docker push "$IMAGE_TAG" - echo "โœ… Successfully pushed $IMAGE_TAG" + echo "Successfully pushed $IMAGE_TAG" - name: Clean up local images run: | @@ -151,7 +151,7 @@ jobs: create-manifest: name: create multi-arch manifest - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: [extract-version, verify-and-build] if: ${{ github.event.inputs.dry_run != 'true' }} steps: @@ -166,7 +166,7 @@ jobs: IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }} VERSION=${{ needs.extract-version.outputs.VERSION }} - echo "๐Ÿ”— Creating multi-arch manifest for $IMAGE_NAME:$VERSION" + echo "Creating multi-arch manifest for $IMAGE_NAME:$VERSION" # Create manifest for version tag docker manifest create \ @@ -184,13 +184,13 @@ jobs: docker manifest push ${IMAGE_NAME}:latest - echo "โœ… Multi-arch manifests published:" + echo "Multi-arch manifests published:" echo " - ${IMAGE_NAME}:${VERSION}" echo " - ${IMAGE_NAME}:latest" release-summary: name: release summary - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: [extract-version, verify-and-build, create-manifest] if: always() steps: @@ -199,55 +199,55 @@ jobs: VERSION=${{ needs.extract-version.outputs.VERSION }} IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }} - echo "## ๐Ÿš€ Reproducible Release Summary for ${VERSION}" + echo "## Reproducible Release Summary for ${VERSION}" echo "" if [[ "${{ needs.verify-and-build.result }}" == "success" ]]; then - echo "โœ… **Reproducibility Verification & Build**: SUCCESS" + echo "**Reproducibility Verification & Build**: SUCCESS" echo "- All architectures produce identical binaries" echo "- Images built and ready for publishing" else - echo "โŒ **Reproducibility Verification & Build**: FAILED" + echo "**Reproducibility Verification & Build**: FAILED" echo "- Builds are not reproducible OR build failed" echo "- Release was blocked" fi echo "" if [[ "${{ github.event.inputs.dry_run }}" == "true" ]]; then - echo "๐Ÿงช **Mode**: DRY RUN" + echo "**Mode**: DRY RUN" echo "- Images were built and verified but NOT pushed" echo "- Ready for real release" elif [[ "${{ needs.create-manifest.result }}" == "success" ]]; then - echo "โœ… **Publication**: SUCCESS" + echo "**Publication**: SUCCESS" echo "- Images published to Docker Hub" echo "- Multi-arch manifests created" echo "" - echo "### ๐Ÿ“ฆ Published Images" + echo "### Published Images" echo "- \`${IMAGE_NAME}:${VERSION}\`" echo "- \`${IMAGE_NAME}:latest\`" echo "" - echo "### ๐Ÿ—๏ธ Architectures" + echo "### Architectures" echo "- linux/amd64 (\`${IMAGE_NAME}:${VERSION}-amd64\`)" echo "- linux/arm64 (\`${IMAGE_NAME}:${VERSION}-arm64\`)" else - echo "โŒ **Publication**: FAILED" + echo "**Publication**: FAILED" echo "- Images were verified but failed to publish" fi echo "" if [[ "${{ needs.verify-and-build.result }}" == "success" ]] && [[ "${{ needs.create-manifest.result }}" == "success" ]] && [[ "${{ github.event.inputs.dry_run }}" != "true" ]]; then - echo "๐ŸŽ‰ **Overall**: Secure release completed successfully!" + echo "**Overall**: Secure release completed successfully!" echo "" - echo "### ๐Ÿ”’ Security Guarantees" - echo "- โœ… Reproducible builds verified" - echo "- โœ… Identical binaries across architectures" - echo "- โœ… No build artifacts tampering" - echo "- โœ… Deterministic build process" + echo "### Security Guarantees" + echo "- Reproducible builds verified" + echo "- Identical binaries across architectures" + echo "- No build artifacts tampering" + echo "- Deterministic build process" elif [[ "${{ github.event.inputs.dry_run }}" == "true" ]]; then - echo "๐Ÿงช **Overall**: Dry run completed successfully!" - echo "- Reproducibility verified โœ…" - echo "- Ready for real release โœ…" + echo " **Overall**: Dry run completed successfully!" + echo "- Reproducibility verified" + echo "- Ready for real release" else - echo "๐Ÿšจ **Overall**: Release failed or incomplete" + echo "**Overall**: Release failed or incomplete" echo "- Check logs above for details" fi diff --git a/.github/workflows/reproducible-build.yml b/.github/workflows/reproducible-build.yml index dd999772660..c17d0ef7ad0 100644 --- a/.github/workflows/reproducible-build.yml +++ b/.github/workflows/reproducible-build.yml @@ -16,7 +16,7 @@ on: jobs: build-x86_64: name: test reproducible builds (x86_64) - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 # Run on schedule, workflow_dispatch, or PRs with the 'test-reproducible' label if: > github.event_name == 'schedule' || @@ -96,10 +96,10 @@ jobs: echo "=== Binary comparison ===" if cmp lighthouse-build-1 lighthouse-build-2; then - echo "โœ… SUCCESS: Docker-built binaries are identical!" - echo "โœ… Reproducible Docker build PASSED for x86_64" + echo "SUCCESS: Docker-built binaries are identical!" + echo "Reproducible Docker build PASSED for x86_64" else - echo "โŒ FAILED: Docker-built binaries differ" + echo "FAILED: Docker-built binaries differ" echo "First 10 differences:" cmp -l lighthouse-build-1 lighthouse-build-2 | head -10 exit 1 @@ -107,7 +107,7 @@ jobs: build-aarch64: name: test reproducible builds (aarch64) - runs-on: ubuntu-24.04-arm + runs-on: ubuntu-22.04-arm # Run on schedule, workflow_dispatch, or PRs with the 'test-reproducible' label if: > github.event_name == 'schedule' || @@ -184,62 +184,11 @@ jobs: echo "=== Binary comparison ===" if cmp lighthouse-build-1-arm64 lighthouse-build-2-arm64; then - echo "โœ… SUCCESS: Docker-built binaries are identical!" - echo "โœ… Reproducible Docker build PASSED for aarch64" + echo "SUCCESS: Docker-built binaries are identical!" + echo "Reproducible Docker build PASSED for aarch64" else - echo "โŒ FAILED: Docker-built binaries differ" + echo "FAILED: Docker-built binaries differ" echo "First 10 differences:" cmp -l lighthouse-build-1-arm64 lighthouse-build-2-arm64 | head -10 exit 1 fi - - summary: - name: reproducible build summary - runs-on: ubuntu-latest - needs: [build-x86_64, build-aarch64] - if: always() - steps: - - name: Report results - run: | - echo "## ๐Ÿ”„ Reproducible Build Test Results" - echo "" - - # Show trigger reason - if [[ "${{ github.event_name }}" == "schedule" ]]; then - echo "**Trigger**: Scheduled check (every 2 days)" - elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then - echo "**Trigger**: Manual run" - elif [[ "${{ github.event_name }}" == "pull_request" ]]; then - echo "**Trigger**: PR with 'test-reproducible' label" - fi - echo "" - - if [[ "${{ needs.build-x86_64.result }}" == "success" ]]; then - echo "โœ… **x86_64**: Reproducible builds PASSED" - elif [[ "${{ needs.build-x86_64.result }}" == "skipped" ]]; then - echo "โญ๏ธ **x86_64**: Skipped (no label or wrong trigger)" - else - echo "โŒ **x86_64**: Reproducible builds FAILED" - fi - - if [[ "${{ needs.build-aarch64.result }}" == "success" ]]; then - echo "โœ… **aarch64**: Reproducible builds PASSED" - elif [[ "${{ needs.build-aarch64.result }}" == "skipped" ]]; then - echo "โญ๏ธ **aarch64**: Skipped (no label or wrong trigger)" - else - echo "โŒ **aarch64**: Reproducible builds FAILED" - fi - - echo "" - if [[ "${{ needs.build-x86_64.result }}" == "success" ]] \ - && [[ "${{ needs.build-aarch64.result }}" == "success" ]]; then - echo "๐ŸŽ‰ **Overall**: All reproducible builds are working correctly!" - echo "Docker containers are reproducible." - elif [[ "${{ needs.build-x86_64.result }}" == "skipped" ]] \ - && [[ "${{ needs.build-aarch64.result }}" == "skipped" ]]; then - echo "โญ๏ธ **Overall**: Tests were skipped (add 'test-reproducible' label to run on PRs)" - else - echo "โš ๏ธ **Overall**: Some reproducible builds failed" - echo "Check the logs above for details" - exit 1 - fi diff --git a/Dockerfile.reproducible b/Dockerfile.reproducible index 4268669091e..6ca8bd1dc6a 100644 --- a/Dockerfile.reproducible +++ b/Dockerfile.reproducible @@ -21,5 +21,5 @@ RUN mv /app/target/${RUST_TARGET}/release/lighthouse /lighthouse FROM gcr.io/distroless/cc-debian12:nonroot-6755e21ccd99ddead6edc8106ba03888cbeed41a COPY --from=builder /lighthouse /lighthouse -EXPOSE 30303 30303/udp 9001 8545 8546 +EXPOSE 8551 9000 ENTRYPOINT [ "/lighthouse" ] diff --git a/Makefile b/Makefile index c798352462f..b478367dd04 100644 --- a/Makefile +++ b/Makefile @@ -99,9 +99,6 @@ LOCALE_VAL = C # Set UTC timezone for consistent time handling across builds TZ_VAL = UTC -# Default features for lighthouse -FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx,slasher-redb,jemalloc - # Default profile PROFILE ?= release @@ -119,7 +116,7 @@ build-reproducible: ## Build the lighthouse binary into `target` directory with CARGO_INCREMENTAL=${CARGO_INCREMENTAL_VAL} \ LC_ALL=${LOCALE_VAL} \ TZ=${TZ_VAL} \ - cargo build --bin lighthouse --features "$(FEATURES)" --profile "$(PROFILE)" --locked --target $(RUST_TARGET) + cargo build --bin lighthouse --features "$(CROSS_FEATURES)" --profile "$(PROFILE)" --locked --target $(RUST_TARGET) .PHONY: build-reproducible-x86_64 build-reproducible-x86_64: ## Build reproducible x86_64 Docker image From dbedf063a3dafe5a23496e42bebea002157d2dd1 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Thu, 6 Nov 2025 11:48:29 +0000 Subject: [PATCH 15/33] remove the release summary from docker-reproducible.yml --- .github/workflows/docker-reproducible.yml | 64 ----------------------- 1 file changed, 64 deletions(-) diff --git a/.github/workflows/docker-reproducible.yml b/.github/workflows/docker-reproducible.yml index 4d0c8138971..8b09848ae9a 100644 --- a/.github/workflows/docker-reproducible.yml +++ b/.github/workflows/docker-reproducible.yml @@ -187,67 +187,3 @@ jobs: echo "Multi-arch manifests published:" echo " - ${IMAGE_NAME}:${VERSION}" echo " - ${IMAGE_NAME}:latest" - - release-summary: - name: release summary - runs-on: ubuntu-22.04 - needs: [extract-version, verify-and-build, create-manifest] - if: always() - steps: - - name: Report release results - run: | - VERSION=${{ needs.extract-version.outputs.VERSION }} - IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }} - - echo "## Reproducible Release Summary for ${VERSION}" - echo "" - - if [[ "${{ needs.verify-and-build.result }}" == "success" ]]; then - echo "**Reproducibility Verification & Build**: SUCCESS" - echo "- All architectures produce identical binaries" - echo "- Images built and ready for publishing" - else - echo "**Reproducibility Verification & Build**: FAILED" - echo "- Builds are not reproducible OR build failed" - echo "- Release was blocked" - fi - - echo "" - if [[ "${{ github.event.inputs.dry_run }}" == "true" ]]; then - echo "**Mode**: DRY RUN" - echo "- Images were built and verified but NOT pushed" - echo "- Ready for real release" - elif [[ "${{ needs.create-manifest.result }}" == "success" ]]; then - echo "**Publication**: SUCCESS" - echo "- Images published to Docker Hub" - echo "- Multi-arch manifests created" - echo "" - echo "### Published Images" - echo "- \`${IMAGE_NAME}:${VERSION}\`" - echo "- \`${IMAGE_NAME}:latest\`" - echo "" - echo "### Architectures" - echo "- linux/amd64 (\`${IMAGE_NAME}:${VERSION}-amd64\`)" - echo "- linux/arm64 (\`${IMAGE_NAME}:${VERSION}-arm64\`)" - else - echo "**Publication**: FAILED" - echo "- Images were verified but failed to publish" - fi - - echo "" - if [[ "${{ needs.verify-and-build.result }}" == "success" ]] && [[ "${{ needs.create-manifest.result }}" == "success" ]] && [[ "${{ github.event.inputs.dry_run }}" != "true" ]]; then - echo "**Overall**: Secure release completed successfully!" - echo "" - echo "### Security Guarantees" - echo "- Reproducible builds verified" - echo "- Identical binaries across architectures" - echo "- No build artifacts tampering" - echo "- Deterministic build process" - elif [[ "${{ github.event.inputs.dry_run }}" == "true" ]]; then - echo " **Overall**: Dry run completed successfully!" - echo "- Reproducibility verified" - echo "- Ready for real release" - else - echo "**Overall**: Release failed or incomplete" - echo "- Check logs above for details" - fi From f4c1e46f93f480b8030f83c210efe1607b437b12 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Thu, 6 Nov 2025 11:48:57 +0000 Subject: [PATCH 16/33] removed exposed ports from the Dockerfile.reproducible --- Dockerfile.reproducible | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile.reproducible b/Dockerfile.reproducible index 6ca8bd1dc6a..092b87b534a 100644 --- a/Dockerfile.reproducible +++ b/Dockerfile.reproducible @@ -21,5 +21,4 @@ RUN mv /app/target/${RUST_TARGET}/release/lighthouse /lighthouse FROM gcr.io/distroless/cc-debian12:nonroot-6755e21ccd99ddead6edc8106ba03888cbeed41a COPY --from=builder /lighthouse /lighthouse -EXPOSE 8551 9000 ENTRYPOINT [ "/lighthouse" ] From 2bcc78494f4ecc56b111765dce163489d9d94c61 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Thu, 6 Nov 2025 16:18:07 +0000 Subject: [PATCH 17/33] add stable/unstable pushes to the docker-reproducible workflow --- .github/workflows/docker-reproducible.yml | 44 ++++++++++++++++------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docker-reproducible.yml b/.github/workflows/docker-reproducible.yml index 8b09848ae9a..f9891e626bf 100644 --- a/.github/workflows/docker-reproducible.yml +++ b/.github/workflows/docker-reproducible.yml @@ -2,6 +2,9 @@ name: docker-reproducible on: push: + branches: + - unstable + - stable tags: - v* workflow_dispatch: @@ -24,11 +27,25 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Extract version - run: >- - echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT + run: | + if [[ "${{ github.ref }}" == refs/tags/* ]]; then + # It's a tag (e.g., v1.2.3) + VERSION="${GITHUB_REF#refs/tags/}" + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + echo "IS_TAG=true" >> $GITHUB_OUTPUT + elif [[ "${{ github.ref }}" == refs/heads/stable ]]; then + # stable branch -> latest + echo "VERSION=latest" >> $GITHUB_OUTPUT + echo "IS_TAG=false" >> $GITHUB_OUTPUT + elif [[ "${{ github.ref }}" == refs/heads/unstable ]]; then + # unstable branch -> latest-unstable + echo "VERSION=latest-unstable" >> $GITHUB_OUTPUT + echo "IS_TAG=false" >> $GITHUB_OUTPUT + fi id: extract_version outputs: VERSION: ${{ steps.extract_version.outputs.VERSION }} + IS_TAG: ${{ steps.extract_version.outputs.IS_TAG }} verify-and-build: name: verify reproducibility and build @@ -165,10 +182,11 @@ jobs: run: | IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }} VERSION=${{ needs.extract-version.outputs.VERSION }} + IS_TAG=${{ needs.extract-version.outputs.IS_TAG }} echo "Creating multi-arch manifest for $IMAGE_NAME:$VERSION" - # Create manifest for version tag + # Create manifest for the version tag docker manifest create \ ${IMAGE_NAME}:${VERSION} \ ${IMAGE_NAME}:${VERSION}-amd64 \ @@ -176,14 +194,16 @@ jobs: docker manifest push ${IMAGE_NAME}:${VERSION} - # Create manifest for latest tag - docker manifest create \ - ${IMAGE_NAME}:latest \ - ${IMAGE_NAME}:${VERSION}-amd64 \ - ${IMAGE_NAME}:${VERSION}-arm64 + echo "Published: ${IMAGE_NAME}:${VERSION}" - docker manifest push ${IMAGE_NAME}:latest + # For tagged releases, also update 'latest' + if [[ "$IS_TAG" == "true" ]]; then + echo "Creating 'latest' manifest for tagged release" + docker manifest create \ + ${IMAGE_NAME}:latest \ + ${IMAGE_NAME}:${VERSION}-amd64 \ + ${IMAGE_NAME}:${VERSION}-arm64 - echo "Multi-arch manifests published:" - echo " - ${IMAGE_NAME}:${VERSION}" - echo " - ${IMAGE_NAME}:latest" + docker manifest push ${IMAGE_NAME}:latest + echo "Published: ${IMAGE_NAME}:latest" + fi From e838bfd6e62741f7c3368ff9f1cff2735ce68822 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Thu, 6 Nov 2025 16:18:47 +0000 Subject: [PATCH 18/33] remove reproducible-builds workflow --- .github/workflows/reproducible-build.yml | 194 ----------------------- 1 file changed, 194 deletions(-) delete mode 100644 .github/workflows/reproducible-build.yml diff --git a/.github/workflows/reproducible-build.yml b/.github/workflows/reproducible-build.yml deleted file mode 100644 index c17d0ef7ad0..00000000000 --- a/.github/workflows/reproducible-build.yml +++ /dev/null @@ -1,194 +0,0 @@ -name: reproducible-build - -on: - workflow_dispatch: {} - schedule: - - cron: "0 1 */2 * *" # Every 2 days to catch issues early - pull_request: - types: [opened, synchronize, labeled] - paths: - - "Makefile" - - "Dockerfile.reproducible" - - ".github/workflows/reproducible-build.yml" - - "Cargo.toml" - - "Cargo.lock" - -jobs: - build-x86_64: - name: test reproducible builds (x86_64) - runs-on: ubuntu-22.04 - # Run on schedule, workflow_dispatch, or PRs with the 'test-reproducible' label - if: > - github.event_name == 'schedule' || - github.event_name == 'workflow_dispatch' || - contains(github.event.pull_request.labels.*.name, 'test-reproducible') - steps: - - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - driver: docker # Use docker driver instead of docker-container for reproducibility - - - name: Build first reproducible Docker image - run: | - echo "=== Building first Docker image (x86_64) ===" - docker build -f Dockerfile.reproducible \ - --build-arg RUST_TARGET="x86_64-unknown-linux-gnu" \ - --build-arg RUST_IMAGE="rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9e315fd2cb5100e87a7187a9816" \ - -t build-lighthouse-1 . - - echo "=== Extracting binary from first build ===" - docker create --name extract-lighthouse-1 build-lighthouse-1 - docker cp extract-lighthouse-1:/lighthouse ./lighthouse-build-1 - docker rm extract-lighthouse-1 - - echo "=== First build info ===" - ls -la lighthouse-build-1 - sha256sum lighthouse-build-1 - file lighthouse-build-1 - - - name: Clean Docker state completely - run: | - echo "=== Cleaning Docker state ===" - # Remove the first image - docker rmi build-lighthouse-1 || true - - # Remove all build cache (important for reproducibility testing) - docker buildx prune -f || true - docker system prune -f || true - - # Clear any remaining containers - docker container prune -f || true - - echo "=== Docker state cleaned ===" - docker images - docker ps -a - - - name: Build second reproducible Docker image - run: | - echo "=== Building second Docker image (x86_64) ===" - docker build -f Dockerfile.reproducible \ - --build-arg RUST_TARGET="x86_64-unknown-linux-gnu" \ - --build-arg RUST_IMAGE="rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9e315fd2cb5100e87a7187a9816" \ - -t build-lighthouse-2 . - - echo "=== Extracting binary from second build ===" - docker create --name extract-lighthouse-2 build-lighthouse-2 - docker cp extract-lighthouse-2:/lighthouse ./lighthouse-build-2 - docker rm extract-lighthouse-2 - - echo "=== Second build info ===" - ls -la lighthouse-build-2 - sha256sum lighthouse-build-2 - file lighthouse-build-2 - - - name: Compare Docker-built binaries - run: | - echo "=== Comparing Docker-built binaries (x86_64) ===" - echo "Build 1 info:" - ls -la lighthouse-build-1 - echo "Build 2 info:" - ls -la lighthouse-build-2 - - echo "=== SHA256 checksums ===" - sha256sum lighthouse-build-* - - echo "=== Binary comparison ===" - if cmp lighthouse-build-1 lighthouse-build-2; then - echo "SUCCESS: Docker-built binaries are identical!" - echo "Reproducible Docker build PASSED for x86_64" - else - echo "FAILED: Docker-built binaries differ" - echo "First 10 differences:" - cmp -l lighthouse-build-1 lighthouse-build-2 | head -10 - exit 1 - fi - - build-aarch64: - name: test reproducible builds (aarch64) - runs-on: ubuntu-22.04-arm - # Run on schedule, workflow_dispatch, or PRs with the 'test-reproducible' label - if: > - github.event_name == 'schedule' || - github.event_name == 'workflow_dispatch' || - contains(github.event.pull_request.labels.*.name, 'test-reproducible') - steps: - - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - driver: docker - - - name: Build first reproducible Docker image - run: | - echo "=== Building first Docker image (aarch64) ===" - docker build -f Dockerfile.reproducible \ - --platform linux/arm64 \ - --build-arg RUST_TARGET="aarch64-unknown-linux-gnu" \ - --build-arg RUST_IMAGE="rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94" \ - -t build-lighthouse-1-arm64 . - - echo "=== Extracting binary from first build ===" - docker create --name extract-lighthouse-1-arm64 build-lighthouse-1-arm64 - docker cp extract-lighthouse-1-arm64:/lighthouse ./lighthouse-build-1-arm64 - docker rm extract-lighthouse-1-arm64 - - echo "=== First build info ===" - ls -la lighthouse-build-1-arm64 - sha256sum lighthouse-build-1-arm64 - file lighthouse-build-1-arm64 - - - name: Clean Docker state completely - run: | - echo "=== Cleaning Docker state ===" - docker rmi build-lighthouse-1-arm64 || true - docker buildx prune -f || true - docker system prune -f || true - docker container prune -f || true - - echo "=== Docker state cleaned ===" - docker images - docker ps -a - - - name: Build second reproducible Docker image - run: | - echo "=== Building second Docker image (aarch64) ===" - docker build -f Dockerfile.reproducible \ - --platform linux/arm64 \ - --build-arg RUST_TARGET="aarch64-unknown-linux-gnu" \ - --build-arg RUST_IMAGE="rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94" \ - -t build-lighthouse-2-arm64 . - - echo "=== Extracting binary from second build ===" - docker create --name extract-lighthouse-2-arm64 build-lighthouse-2-arm64 - docker cp extract-lighthouse-2-arm64:/lighthouse ./lighthouse-build-2-arm64 - docker rm extract-lighthouse-2-arm64 - - echo "=== Second build info ===" - ls -la lighthouse-build-2-arm64 - sha256sum lighthouse-build-2-arm64 - file lighthouse-build-2-arm64 - - - name: Compare Docker-built binaries - run: | - echo "=== Comparing Docker-built binaries (aarch64) ===" - echo "Build 1 info:" - ls -la lighthouse-build-1-arm64 - echo "Build 2 info:" - ls -la lighthouse-build-2-arm64 - - echo "=== SHA256 checksums ===" - sha256sum lighthouse-build-*-arm64 - - echo "=== Binary comparison ===" - if cmp lighthouse-build-1-arm64 lighthouse-build-2-arm64; then - echo "SUCCESS: Docker-built binaries are identical!" - echo "Reproducible Docker build PASSED for aarch64" - else - echo "FAILED: Docker-built binaries differ" - echo "First 10 differences:" - cmp -l lighthouse-build-1-arm64 lighthouse-build-2-arm64 | head -10 - exit 1 - fi From 19768d6e7d3430075640dca499ca23adee285c59 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Thu, 6 Nov 2025 16:53:46 +0000 Subject: [PATCH 19/33] remove unnecessary is_tag checks --- .github/workflows/docker-reproducible.yml | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/.github/workflows/docker-reproducible.yml b/.github/workflows/docker-reproducible.yml index f9891e626bf..db724766cb0 100644 --- a/.github/workflows/docker-reproducible.yml +++ b/.github/workflows/docker-reproducible.yml @@ -31,21 +31,17 @@ jobs: if [[ "${{ github.ref }}" == refs/tags/* ]]; then # It's a tag (e.g., v1.2.3) VERSION="${GITHUB_REF#refs/tags/}" - echo "VERSION=$VERSION" >> $GITHUB_OUTPUT - echo "IS_TAG=true" >> $GITHUB_OUTPUT elif [[ "${{ github.ref }}" == refs/heads/stable ]]; then # stable branch -> latest - echo "VERSION=latest" >> $GITHUB_OUTPUT - echo "IS_TAG=false" >> $GITHUB_OUTPUT + VERSION="latest" elif [[ "${{ github.ref }}" == refs/heads/unstable ]]; then # unstable branch -> latest-unstable - echo "VERSION=latest-unstable" >> $GITHUB_OUTPUT - echo "IS_TAG=false" >> $GITHUB_OUTPUT + VERSION="latest-unstable" fi + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT id: extract_version outputs: VERSION: ${{ steps.extract_version.outputs.VERSION }} - IS_TAG: ${{ steps.extract_version.outputs.IS_TAG }} verify-and-build: name: verify reproducibility and build @@ -182,7 +178,6 @@ jobs: run: | IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }} VERSION=${{ needs.extract-version.outputs.VERSION }} - IS_TAG=${{ needs.extract-version.outputs.IS_TAG }} echo "Creating multi-arch manifest for $IMAGE_NAME:$VERSION" @@ -195,15 +190,3 @@ jobs: docker manifest push ${IMAGE_NAME}:${VERSION} echo "Published: ${IMAGE_NAME}:${VERSION}" - - # For tagged releases, also update 'latest' - if [[ "$IS_TAG" == "true" ]]; then - echo "Creating 'latest' manifest for tagged release" - docker manifest create \ - ${IMAGE_NAME}:latest \ - ${IMAGE_NAME}:${VERSION}-amd64 \ - ${IMAGE_NAME}:${VERSION}-arm64 - - docker manifest push ${IMAGE_NAME}:latest - echo "Published: ${IMAGE_NAME}:latest" - fi From 3431d0349096e04f963c9ef65641d083193c3739 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Thu, 6 Nov 2025 17:17:28 +0000 Subject: [PATCH 20/33] manual workflow trigger is just for testing purposes only --- .github/workflows/docker-reproducible.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker-reproducible.yml b/.github/workflows/docker-reproducible.yml index db724766cb0..ff046918636 100644 --- a/.github/workflows/docker-reproducible.yml +++ b/.github/workflows/docker-reproducible.yml @@ -7,13 +7,7 @@ on: - stable tags: - v* - workflow_dispatch: - inputs: - dry_run: - description: >- - Enable dry run mode (builds images but skips push to registry) - type: boolean - default: false + workflow_dispatch: # allows manual triggering for testing purposes and skips publishing an image env: DOCKER_REPRODUCIBLE_IMAGE_NAME: >- @@ -37,6 +31,9 @@ jobs: elif [[ "${{ github.ref }}" == refs/heads/unstable ]]; then # unstable branch -> latest-unstable VERSION="latest-unstable" + else + # For manual triggers from other branches and will not publish any image + VERSION="test-build" fi echo "VERSION=$VERSION" >> $GITHUB_OUTPUT id: extract_version @@ -132,14 +129,14 @@ jobs: echo "Image ready for publishing: $FINAL_TAG" - name: Log in to Docker Hub - if: ${{ github.event.inputs.dry_run != 'true' }} + if: ${{ github.event_name != 'workflow_dispatch' }} uses: docker/login-action@v3 with: username: ${{ env.DOCKER_USERNAME }} password: ${{ env.DOCKER_PASSWORD }} - name: Push verified image (${{ matrix.arch }}) - if: ${{ github.event.inputs.dry_run != 'true' }} + if: ${{ github.event_name != 'workflow_dispatch' }} run: | VERSION=${{ needs.extract-version.outputs.VERSION }} IMAGE_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}" @@ -166,7 +163,7 @@ jobs: name: create multi-arch manifest runs-on: ubuntu-22.04 needs: [extract-version, verify-and-build] - if: ${{ github.event.inputs.dry_run != 'true' }} + if: ${{ github.event_name != 'workflow_dispatch' }} steps: - name: Log in to Docker Hub uses: docker/login-action@v3 From 12c057416da116a7809c29db2f1b7c57443e1a47 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Thu, 6 Nov 2025 19:12:39 +0000 Subject: [PATCH 21/33] Add reproducibility build for jemalloc-sys --- Dockerfile.reproducible | 2 +- Makefile | 10 +++++++++- common/malloc_utils/Cargo.toml | 2 ++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Dockerfile.reproducible b/Dockerfile.reproducible index 092b87b534a..903515373f8 100644 --- a/Dockerfile.reproducible +++ b/Dockerfile.reproducible @@ -3,7 +3,7 @@ ARG RUST_IMAGE="rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9 FROM ${RUST_IMAGE} AS builder # Install specific version of the build dependencies -RUN apt-get update && apt-get install -y libclang-dev=1:11.0-51+nmu5 cmake=3.18.4-2+deb11u1 +RUN apt-get update && apt-get install -y libclang-dev=1:11.0-51+nmu5 cmake=3.18.4-2+deb11u1 libjemalloc-dev=5.2.1-3 ARG RUST_TARGET="x86_64-unknown-linux-gnu" diff --git a/Makefile b/Makefile index b478367dd04..697225d2282 100644 --- a/Makefile +++ b/Makefile @@ -102,6 +102,13 @@ TZ_VAL = UTC # Default profile PROFILE ?= release +# Features for reproducible builds +FEATURES_REPRODUCIBLE = $(CROSS_FEATURES),jemalloc-unprefixed + +# Derive the architecture-specific library path from RUST_TARGET +JEMALLOC_LIB_ARCH = $(word 1,$(subst -, ,$(RUST_TARGET))) +JEMALLOC_OVERRIDE = /usr/lib/$(JEMALLOC_LIB_ARCH)-linux-gnu/libjemalloc.a + # Default target architecture RUST_TARGET ?= x86_64-unknown-linux-gnu @@ -116,7 +123,8 @@ build-reproducible: ## Build the lighthouse binary into `target` directory with CARGO_INCREMENTAL=${CARGO_INCREMENTAL_VAL} \ LC_ALL=${LOCALE_VAL} \ TZ=${TZ_VAL} \ - cargo build --bin lighthouse --features "$(CROSS_FEATURES)" --profile "$(PROFILE)" --locked --target $(RUST_TARGET) + JEMALLOC_OVERRIDE=${JEMALLOC_OVERRIDE} \ + cargo build --bin lighthouse --features "$(FEATURES_REPRODUCIBLE)" --profile "$(PROFILE)" --locked --target $(RUST_TARGET) .PHONY: build-reproducible-x86_64 build-reproducible-x86_64: ## Build reproducible x86_64 Docker image diff --git a/common/malloc_utils/Cargo.toml b/common/malloc_utils/Cargo.toml index 39c7137d4cb..1052128852a 100644 --- a/common/malloc_utils/Cargo.toml +++ b/common/malloc_utils/Cargo.toml @@ -21,6 +21,8 @@ jemalloc-profiling = ["tikv-jemallocator/profiling"] # Force the use of system malloc (or glibc) rather than jemalloc. # This is a no-op on Windows where jemalloc is always disabled. sysmalloc = [] +# Enable jemalloc with unprefixed malloc (recommended for reproducible builds) +jemalloc-unprefixed = ["jemalloc", "tikv-jemallocator/unprefixed_malloc_on_supported_platforms"] [dependencies] libc = "0.2.79" From 5613f2ad72c6bbd8df1397937afc933b43dc0c31 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Mon, 10 Nov 2025 08:38:21 +0000 Subject: [PATCH 22/33] chore: remove unnecessary echo --- .github/workflows/docker-reproducible.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/docker-reproducible.yml b/.github/workflows/docker-reproducible.yml index ff046918636..f3479e9468d 100644 --- a/.github/workflows/docker-reproducible.yml +++ b/.github/workflows/docker-reproducible.yml @@ -70,10 +70,7 @@ jobs: - name: Verify reproducible builds (${{ matrix.arch }}) run: | - echo "Verifying reproducible builds for ${{ matrix.arch }}..." - # Build first image - echo "=== Building first verification image ===" docker build -f Dockerfile.reproducible \ --platform ${{ matrix.platform }} \ --build-arg RUST_TARGET="${{ matrix.rust_target }}" \ @@ -90,7 +87,6 @@ jobs: docker system prune -f # Build second image - echo "=== Building second verification image ===" docker build -f Dockerfile.reproducible \ --platform ${{ matrix.platform }} \ --build-arg RUST_TARGET="${{ matrix.rust_target }}" \ @@ -126,8 +122,6 @@ jobs: FINAL_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}" docker tag lighthouse-verify-2-${{ matrix.arch }} "$FINAL_TAG" - echo "Image ready for publishing: $FINAL_TAG" - - name: Log in to Docker Hub if: ${{ github.event_name != 'workflow_dispatch' }} uses: docker/login-action@v3 @@ -140,10 +134,7 @@ jobs: run: | VERSION=${{ needs.extract-version.outputs.VERSION }} IMAGE_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}" - - echo "Pushing verified reproducible image: $IMAGE_TAG" docker push "$IMAGE_TAG" - echo "Successfully pushed $IMAGE_TAG" - name: Clean up local images run: | @@ -176,8 +167,6 @@ jobs: IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }} VERSION=${{ needs.extract-version.outputs.VERSION }} - echo "Creating multi-arch manifest for $IMAGE_NAME:$VERSION" - # Create manifest for the version tag docker manifest create \ ${IMAGE_NAME}:${VERSION} \ @@ -185,5 +174,3 @@ jobs: ${IMAGE_NAME}:${VERSION}-arm64 docker manifest push ${IMAGE_NAME}:${VERSION} - - echo "Published: ${IMAGE_NAME}:${VERSION}" From 8740bcd96138a481fb6b1da9c6b8973f589b3fb8 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 28 Nov 2025 09:28:09 +1100 Subject: [PATCH 23/33] Remove redundant PROFILE in Makefile --- Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Makefile b/Makefile index 697225d2282..a6891b682f7 100644 --- a/Makefile +++ b/Makefile @@ -99,9 +99,6 @@ LOCALE_VAL = C # Set UTC timezone for consistent time handling across builds TZ_VAL = UTC -# Default profile -PROFILE ?= release - # Features for reproducible builds FEATURES_REPRODUCIBLE = $(CROSS_FEATURES),jemalloc-unprefixed From 9a5b9b030ccbdc5d3aef198b1393b21f72b7765e Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Tue, 17 Jun 2025 18:18:10 +0000 Subject: [PATCH 24/33] Add necessary setup for reproducible debian packaging --- Makefile | 140 ++++++++++++++++++++++++++++++++++ lighthouse/Cargo.toml | 30 ++++++++ lighthouse/lighthouse.service | 34 +++++++++ 3 files changed, 204 insertions(+) create mode 100644 lighthouse/lighthouse.service diff --git a/Makefile b/Makefile index a6891b682f7..1e46485905d 100644 --- a/Makefile +++ b/Makefile @@ -332,3 +332,143 @@ clean: cargo clean make -C $(EF_TESTS) clean make -C $(STATE_TRANSITION_VECTORS) clean + +.PHONY: install-cargo-deb deb-cargo deb-cargo-x86_64 deb-cargo-aarch64 deb-cargo-all test-deb-reproducible install-deb-local remove-deb-local clean-deb help-deb +## Install cargo-deb if not present +install-cargo-deb: + @if ! command -v cargo-deb &> /dev/null; then \ + echo "Installing cargo-deb..."; \ + cargo install cargo-deb; \ + else \ + echo "cargo-deb already installed"; \ + fi + +## Build .deb package using cargo-deb with reproducible settings +deb-cargo: install-cargo-deb build-reproducible + @echo "Building .deb package with cargo-deb..." + @if [ ! -f "lighthouse/lighthouse.service" ]; then \ + echo "โŒ lighthouse.service not found in lighthouse/ directory"; \ + exit 1; \ + fi + @if [ ! -f "README.md" ]; then \ + echo "โŒ README.md not found in current directory"; \ + exit 1; \ + fi + + cd lighthouse && \ + SOURCE_DATE_EPOCH=$(SOURCE_DATE) \ + CARGO_INCREMENTAL=$(CARGO_INCREMENTAL_VAL) \ + LC_ALL=$(LOCALE_VAL) \ + TZ=$(TZ_VAL) \ + cargo deb --target $(RUST_TARGET) --no-build + + @echo "โœ… Package built successfully!" + @find target/$(RUST_TARGET)/debian -name "*.deb" -exec ls -la {} \; + +## Build .deb for specific architectures +deb-cargo-x86_64: + $(MAKE) deb-cargo RUST_TARGET=x86_64-unknown-linux-gnu + +deb-cargo-aarch64: + $(MAKE) deb-cargo RUST_TARGET=aarch64-unknown-linux-gnu + +deb-cargo-all: deb-cargo-x86_64 deb-cargo-aarch64 + +## Test reproducibility of cargo-deb packages +test-deb-reproducible: + @echo "Testing cargo-deb package reproducibility..." + @if ! command -v diffoscope &> /dev/null; then \ + echo "Installing diffoscope..."; \ + sudo apt-get update; \ + sudo apt-get install -y diffoscope binutils-multiarch; \ + fi + + @echo "Building first package..." + @rm -f lighthouse_*.deb lighthouse-deb-*.deb + @$(MAKE) clean || true + @$(MAKE) deb-cargo + @FIRST_PACKAGE=$$(find target/$(RUST_TARGET)/debian -name "*.deb" | head -1); \ + if [ -n "$$FIRST_PACKAGE" ]; then \ + cp "$$FIRST_PACKAGE" ./lighthouse-deb-build-1.deb; \ + else \ + echo "โŒ First package not found"; exit 1; \ + fi + + @echo "Building second package..." + @$(MAKE) clean || true + @$(MAKE) deb-cargo + @SECOND_PACKAGE=$$(find target/$(RUST_TARGET)/debian -name "*.deb" | head -1); \ + if [ -n "$$SECOND_PACKAGE" ]; then \ + cp "$$SECOND_PACKAGE" ./lighthouse-deb-build-2.deb; \ + else \ + echo "โŒ Second package not found"; exit 1; \ + fi + + @echo "Comparing packages..." + @echo "=== Package sizes ===" + @ls -la lighthouse-deb-build-*.deb + @echo "=== SHA256 checksums ===" + @sha256sum lighthouse-deb-build-*.deb + + @if cmp -s lighthouse-deb-build-1.deb lighthouse-deb-build-2.deb; then \ + echo "โœ… SUCCESS: cargo-deb packages are identical!"; \ + echo "โœ… Reproducible build PASSED"; \ + else \ + echo "โŒ FAILED: cargo-deb packages differ"; \ + echo "Running detailed analysis with diffoscope..."; \ + diffoscope --text lighthouse-deb-build-1.deb lighthouse-deb-build-2.deb > cargo-deb-diff.txt || true; \ + echo "Differences saved to cargo-deb-diff.txt"; \ + echo "โŒ Reproducible build FAILED"; \ + exit 1; \ + fi + +## Install .deb package locally for testing +install-deb-local: + @PACKAGE=$$(find target/$(RUST_TARGET)/debian -name "*.deb" | head -1); \ + if [ -n "$$PACKAGE" ]; then \ + echo "Installing lighthouse package: $$PACKAGE"; \ + sudo dpkg -i "$$PACKAGE"; \ + echo "Fixing dependencies if needed..."; \ + sudo apt-get install -f; \ + echo "โœ… Package installed successfully!"; \ + echo ""; \ + echo "The lighthouse service is now available but not started."; \ + echo "Your systemd service file handles user creation declaratively."; \ + echo ""; \ + echo "To check service status: systemctl status lighthouse"; \ + else \ + echo "โŒ No .deb package found. Run 'make deb-cargo' first."; \ + fi + +## Remove installed lighthouse package +remove-deb-local: + @echo "Removing lighthouse package..." + sudo dpkg -r lighthouse || true + sudo systemctl daemon-reload || true + +## Clean up debian packaging artifacts +clean-deb: + @echo "Cleaning up debian packaging artifacts..." + rm -f lighthouse_*.deb lighthouse-deb-*.deb *-diff.txt + rm -rf target/*/debian/ + +## Show help for debian packaging +help-deb: + @echo "Clean Debian Packaging Targets:" + @echo " install-cargo-deb - Install cargo-deb tool" + @echo " deb-cargo - Build .deb package with cargo-deb" + @echo " deb-cargo-x86_64 - Build x86_64 .deb package" + @echo " deb-cargo-aarch64 - Build aarch64 .deb package" + @echo " deb-cargo-all - Build all architectures" + @echo " test-deb-reproducible - Test reproducibility" + @echo " install-deb-local - Install .deb package locally" + @echo " remove-deb-local - Remove installed package" + @echo " clean-deb - Clean up packaging artifacts" + @echo "" + @echo "Prerequisites:" + @echo " - lighthouse/lighthouse.service file" + @echo " - README.md file in current directory" + @echo "" + @echo "Quick start:" + @echo " make deb-cargo - Build .deb for current RUST_TARGET" + @echo " make install-deb-local - Test the package" diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index ebe00c9be59..8ebaa2b1519 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -96,3 +96,33 @@ zeroize = { workspace = true } [[test]] name = "lighthouse_tests" path = "tests/main.rs" + +[package.metadata.deb] +# Package metadata +maintainer = "Sigma Prime " +extended-description = """\ +Lighthouse is a Rust implementation of the Ethereum beacon chain, \ +built by Sigma Prime. It implements the official Ethereum 2.0 specification.""" + +# Dependencies +depends = "$auto, systemd" +section = "net" +priority = "optional" + +# System integration +systemd-units = { enable = false, start = false } + +# Assets to include in the package +assets = [ + # Binary - use target/release/ prefix (cargo-deb will replace with actual path) + ["target/release/lighthouse", "usr/bin/", "755"], + + # Systemd service (in lighthouse directory) + ["lighthouse.service", "lib/systemd/system/", "644"], + + # Documentation (in parent directory) + ["../README.md", "usr/share/doc/lighthouse/", "644"], +] + +# Use default features - let Makefile handle the building +default-features = false diff --git a/lighthouse/lighthouse.service b/lighthouse/lighthouse.service new file mode 100644 index 00000000000..dfc2b4f9f52 --- /dev/null +++ b/lighthouse/lighthouse.service @@ -0,0 +1,34 @@ +[Unit] +Description=Lighthouse Ethereum Beacon Chain Client +Documentation=https://lighthouse-book.sigmaprime.io/ +After=network.target +Wants=network.target + +[Service] +Type=simple +DynamicUser=true +User=lighthouse +Group=eth +Restart=always +RestartSec=5 +TimeoutStopSec=180 +Environment="RUST_LOG=info" +ExecStart=/usr/bin/lighthouse \ + bn \ + --checkpoint-sync-url https://sync-mainnet.beaconcha.in \ + --execution-endpoint http://localhost:8551 \ + --execution-jwt /etc/jwt.hex \ + --suggested-fee-recipient 0x0 \ + --always-prepare-payload \ + --prepare-payload-lookahead 8000 \ + --disable-deposit-contract-sync \ + --http \ + --port 9000 \ + --http-port 3500 \ + --metrics \ + --metrics-address 127.0.0.1 \ + --metrics-port 5054 \ + --datadir /mnt/data/lighthouse + +[Install] +WantedBy=default.target From 3a9a1d581e1a1f2bd244cffe2e6f212993c665c2 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Tue, 17 Jun 2025 18:37:50 +0000 Subject: [PATCH 25/33] add CI workflow to build and upload deb packages as assets --- .github/workflows/release-deb.yml | 312 ++++++++++++++++++++++++++++++ 1 file changed, 312 insertions(+) create mode 100644 .github/workflows/release-deb.yml diff --git a/.github/workflows/release-deb.yml b/.github/workflows/release-deb.yml new file mode 100644 index 00000000000..b32ac845f2c --- /dev/null +++ b/.github/workflows/release-deb.yml @@ -0,0 +1,312 @@ +name: release-deb + +on: + push: + tags: + - v* + workflow_dispatch: + inputs: + dry_run: + description: >- + Enable dry run mode (builds packages but skips release upload) + type: boolean + default: false + +jobs: + extract-version: + name: extract version + runs-on: ubuntu-latest + steps: + - name: Extract version + run: >- + echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT + id: extract_version + outputs: + VERSION: ${{ steps.extract_version.outputs.VERSION }} + + build-deb: + name: build reproducible deb packages + runs-on: ubuntu-latest + needs: extract-version + strategy: + matrix: + arch: [x86_64, aarch64] + include: + - arch: x86_64 + rust_target: x86_64-unknown-linux-gnu + gcc_package: gcc + - arch: aarch64 + rust_target: aarch64-unknown-linux-gnu + gcc_package: gcc-aarch64-linux-gnu + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + with: + target: ${{ matrix.rust_target }} + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y libclang-dev cmake ${{ matrix.gcc_package }} + + - name: Install cargo-deb + run: cargo install cargo-deb + + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + key: deb-build-${{ matrix.arch }} + + - name: Build reproducible .deb package + run: | + make deb-cargo RUST_TARGET=${{ matrix.rust_target }} + + - name: Find and rename .deb package + run: | + VERSION=${{ needs.extract-version.outputs.VERSION }} + DEB_FILE=$(find target/${{ matrix.rust_target }}/debian -name "*.deb" | head -1) + if [ -n "$DEB_FILE" ]; then + # Extract package info + DEB_NAME=$(dpkg-deb -f "$DEB_FILE" Package) + DEB_VERSION=$(dpkg-deb -f "$DEB_FILE" Version) + DEB_ARCH=$(dpkg-deb -f "$DEB_FILE" Architecture) + + # Create standardized filename + NEW_NAME="lighthouse_${VERSION#v}_${{ matrix.arch }}.deb" + cp "$DEB_FILE" "$NEW_NAME" + + echo "DEB_FILE=$NEW_NAME" >> $GITHUB_ENV + echo "DEB_SIZE=$(stat -f%z "$NEW_NAME" 2>/dev/null || stat -c%s "$NEW_NAME")" >> $GITHUB_ENV + + # Generate checksums + sha256sum "$NEW_NAME" > "$NEW_NAME.sha256" + sha512sum "$NEW_NAME" > "$NEW_NAME.sha512" + + echo "Package built: $NEW_NAME" + echo "Size: $(du -h "$NEW_NAME" | cut -f1)" + echo "SHA256: $(cat "$NEW_NAME.sha256")" + else + echo "โŒ No .deb package found" + exit 1 + fi + + - name: Test package installation (dry run) + run: | + echo "Testing package metadata and dependencies..." + dpkg-deb -I "$DEB_FILE" + echo "" + echo "Package contents:" + dpkg-deb -c "$DEB_FILE" + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: lighthouse-deb-${{ matrix.arch }} + path: | + lighthouse_*_${{ matrix.arch }}.deb + lighthouse_*_${{ matrix.arch }}.deb.sha256 + lighthouse_*_${{ matrix.arch }}.deb.sha512 + retention-days: 7 + + test-reproducibility: + name: test deb reproducibility + runs-on: ubuntu-latest + needs: extract-version + strategy: + matrix: + arch: [x86_64, aarch64] + include: + - arch: x86_64 + rust_target: x86_64-unknown-linux-gnu + gcc_package: gcc + - arch: aarch64 + rust_target: aarch64-unknown-linux-gnu + gcc_package: gcc-aarch64-linux-gnu + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + with: + target: ${{ matrix.rust_target }} + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y libclang-dev cmake ${{ matrix.gcc_package }} diffoscope + + - name: Install cargo-deb and cargo-cache + run: | + cargo install cargo-deb + cargo install cargo-cache + + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + key: deb-reproducible-${{ matrix.arch }} + + - name: Test reproducible deb build + run: | + make test-deb-reproducible RUST_TARGET=${{ matrix.rust_target }} + + - name: Upload reproducibility test artifacts (on failure) + if: failure() + uses: actions/upload-artifact@v4 + with: + name: failed-reproducible-deb-${{ matrix.arch }} + path: | + lighthouse-deb-build-*.deb + *-diff.txt + + create-release: + name: create github release + runs-on: ubuntu-latest + needs: [extract-version, build-deb, test-reproducibility] + if: ${{ github.event.inputs.dry_run != 'true' }} + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Prepare release assets + run: | + mkdir -p release-assets + find artifacts -name "*.deb" -exec cp {} release-assets/ \; + find artifacts -name "*.sha256" -exec cp {} release-assets/ \; + find artifacts -name "*.sha512" -exec cp {} release-assets/ \; + + echo "Release assets:" + ls -la release-assets/ + + - name: Generate release notes + run: | + VERSION=${{ needs.extract-version.outputs.VERSION }} + cat > release-notes.md << EOF + # Lighthouse ${VERSION} - Debian Packages + + This release includes reproducible Debian packages for Lighthouse ${VERSION}. + + ## Available Packages + + - **x86_64**: \`lighthouse_${VERSION#v}_x86_64.deb\` + - **aarch64**: \`lighthouse_${VERSION#v}_aarch64.deb\` + + ## Installation + + ### Ubuntu/Debian (x86_64): + \`\`\`bash + wget https://github.com/${{ github.repository }}/releases/download/${VERSION}/lighthouse_${VERSION#v}_x86_64.deb + sudo dpkg -i lighthouse_${VERSION#v}_x86_64.deb + sudo apt-get install -f # Fix any dependency issues + \`\`\` + + ### Ubuntu/Debian (ARM64): + \`\`\`bash + wget https://github.com/${{ github.repository }}/releases/download/${VERSION}/lighthouse_${VERSION#v}_aarch64.deb + sudo dpkg -i lighthouse_${VERSION#v}_aarch64.deb + sudo apt-get install -f # Fix any dependency issues + \`\`\` + + ## Verification + + All packages include SHA256 and SHA512 checksums for verification: + + \`\`\`bash + # Verify SHA256 + sha256sum -c lighthouse_${VERSION#v}_x86_64.deb.sha256 + + # Verify SHA512 + sha512sum -c lighthouse_${VERSION#v}_x86_64.deb.sha512 + \`\`\` + + ## System Service + + After installation, Lighthouse can be managed as a systemd service: + + \`\`\`bash + # Enable and start the service + sudo systemctl enable lighthouse + sudo systemctl start lighthouse + + # Check status + sudo systemctl status lighthouse + + # View logs + sudo journalctl -u lighthouse -f + \`\`\` + + ## Reproducible Builds + + These packages are built using reproducible build techniques. You can verify the build process by checking out the source code at tag \`${VERSION}\` and running: + + \`\`\`bash + make deb-cargo RUST_TARGET=x86_64-unknown-linux-gnu + make test-deb-reproducible RUST_TARGET=x86_64-unknown-linux-gnu + \`\`\` + + ## Package Details + + - **Maintainer**: Sigma Prime + - **Dependencies**: Automatically managed by dpkg/apt + - **Service Integration**: Includes systemd service unit + - **Build Method**: Reproducible builds with cargo-deb + + For more information, see the [Lighthouse documentation](https://lighthouse-book.sigmaprime.io/). + EOF + + - name: Create Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ needs.extract-version.outputs.VERSION }} + name: "Lighthouse ${{ needs.extract-version.outputs.VERSION }} - Debian Packages" + body_path: release-notes.md + files: release-assets/* + draft: false + prerelease: ${{ contains(needs.extract-version.outputs.VERSION, 'beta') || contains(needs.extract-version.outputs.VERSION, 'alpha') || contains(needs.extract-version.outputs.VERSION, 'rc') }} + generate_release_notes: true + + dry-run-summary: + name: dry run summary + runs-on: ubuntu-latest + needs: [extract-version, build-deb, test-reproducibility] + if: ${{ github.event.inputs.dry_run == 'true' }} + steps: + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Summarize dry run + run: | + VERSION=${{ needs.extract-version.outputs.VERSION }} + echo "## ๐Ÿงช Debian Package Release Dry Run Summary" + echo "" + echo "โœ… Successfully completed dry run for version ${VERSION}" + echo "" + echo "### Built Packages:" + find artifacts -name "*.deb" -exec basename {} \; | sort + echo "" + echo "### Package Sizes:" + find artifacts -name "*.deb" -exec ls -lh {} \; | awk '{print $9 ": " $5}' + echo "" + echo "### Checksums Generated:" + find artifacts -name "*.sha256" -exec basename {} \; | sort + find artifacts -name "*.sha512" -exec basename {} \; | sort + echo "" + echo "### What would happen in a real release:" + echo "- Packages would be uploaded to GitHub Releases" + echo "- Release notes would be automatically generated" + echo "- Users could install via:" + echo " \`wget + dpkg -i lighthouse_${VERSION#v}_x86_64.deb\`" + echo "" + echo "### Reproducibility Test Results:" + echo "โœ… All reproducibility tests passed" + echo "" + echo "### Next Steps:" + echo "To perform a real release, push a git tag:" + echo "\`git tag ${VERSION} && git push origin ${VERSION}\`" From a93f8bcf4cef38f06cd43b824ece43a55beddcb4 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Wed, 18 Jun 2025 08:26:09 +0000 Subject: [PATCH 26/33] merge the deb packaging workflow into release.yml to avoid conflicts --- .github/workflows/release-deb.yml | 312 ------------------------------ .github/workflows/release.yml | 131 ++++++++++++- 2 files changed, 124 insertions(+), 319 deletions(-) delete mode 100644 .github/workflows/release-deb.yml diff --git a/.github/workflows/release-deb.yml b/.github/workflows/release-deb.yml deleted file mode 100644 index b32ac845f2c..00000000000 --- a/.github/workflows/release-deb.yml +++ /dev/null @@ -1,312 +0,0 @@ -name: release-deb - -on: - push: - tags: - - v* - workflow_dispatch: - inputs: - dry_run: - description: >- - Enable dry run mode (builds packages but skips release upload) - type: boolean - default: false - -jobs: - extract-version: - name: extract version - runs-on: ubuntu-latest - steps: - - name: Extract version - run: >- - echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT - id: extract_version - outputs: - VERSION: ${{ steps.extract_version.outputs.VERSION }} - - build-deb: - name: build reproducible deb packages - runs-on: ubuntu-latest - needs: extract-version - strategy: - matrix: - arch: [x86_64, aarch64] - include: - - arch: x86_64 - rust_target: x86_64-unknown-linux-gnu - gcc_package: gcc - - arch: aarch64 - rust_target: aarch64-unknown-linux-gnu - gcc_package: gcc-aarch64-linux-gnu - steps: - - uses: actions/checkout@v4 - - - uses: dtolnay/rust-toolchain@stable - with: - target: ${{ matrix.rust_target }} - - - name: Install build dependencies - run: | - sudo apt-get update - sudo apt-get install -y libclang-dev cmake ${{ matrix.gcc_package }} - - - name: Install cargo-deb - run: cargo install cargo-deb - - - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: true - key: deb-build-${{ matrix.arch }} - - - name: Build reproducible .deb package - run: | - make deb-cargo RUST_TARGET=${{ matrix.rust_target }} - - - name: Find and rename .deb package - run: | - VERSION=${{ needs.extract-version.outputs.VERSION }} - DEB_FILE=$(find target/${{ matrix.rust_target }}/debian -name "*.deb" | head -1) - if [ -n "$DEB_FILE" ]; then - # Extract package info - DEB_NAME=$(dpkg-deb -f "$DEB_FILE" Package) - DEB_VERSION=$(dpkg-deb -f "$DEB_FILE" Version) - DEB_ARCH=$(dpkg-deb -f "$DEB_FILE" Architecture) - - # Create standardized filename - NEW_NAME="lighthouse_${VERSION#v}_${{ matrix.arch }}.deb" - cp "$DEB_FILE" "$NEW_NAME" - - echo "DEB_FILE=$NEW_NAME" >> $GITHUB_ENV - echo "DEB_SIZE=$(stat -f%z "$NEW_NAME" 2>/dev/null || stat -c%s "$NEW_NAME")" >> $GITHUB_ENV - - # Generate checksums - sha256sum "$NEW_NAME" > "$NEW_NAME.sha256" - sha512sum "$NEW_NAME" > "$NEW_NAME.sha512" - - echo "Package built: $NEW_NAME" - echo "Size: $(du -h "$NEW_NAME" | cut -f1)" - echo "SHA256: $(cat "$NEW_NAME.sha256")" - else - echo "โŒ No .deb package found" - exit 1 - fi - - - name: Test package installation (dry run) - run: | - echo "Testing package metadata and dependencies..." - dpkg-deb -I "$DEB_FILE" - echo "" - echo "Package contents:" - dpkg-deb -c "$DEB_FILE" - - - name: Upload build artifacts - uses: actions/upload-artifact@v4 - with: - name: lighthouse-deb-${{ matrix.arch }} - path: | - lighthouse_*_${{ matrix.arch }}.deb - lighthouse_*_${{ matrix.arch }}.deb.sha256 - lighthouse_*_${{ matrix.arch }}.deb.sha512 - retention-days: 7 - - test-reproducibility: - name: test deb reproducibility - runs-on: ubuntu-latest - needs: extract-version - strategy: - matrix: - arch: [x86_64, aarch64] - include: - - arch: x86_64 - rust_target: x86_64-unknown-linux-gnu - gcc_package: gcc - - arch: aarch64 - rust_target: aarch64-unknown-linux-gnu - gcc_package: gcc-aarch64-linux-gnu - steps: - - uses: actions/checkout@v4 - - - uses: dtolnay/rust-toolchain@stable - with: - target: ${{ matrix.rust_target }} - - - name: Install build dependencies - run: | - sudo apt-get update - sudo apt-get install -y libclang-dev cmake ${{ matrix.gcc_package }} diffoscope - - - name: Install cargo-deb and cargo-cache - run: | - cargo install cargo-deb - cargo install cargo-cache - - - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: true - key: deb-reproducible-${{ matrix.arch }} - - - name: Test reproducible deb build - run: | - make test-deb-reproducible RUST_TARGET=${{ matrix.rust_target }} - - - name: Upload reproducibility test artifacts (on failure) - if: failure() - uses: actions/upload-artifact@v4 - with: - name: failed-reproducible-deb-${{ matrix.arch }} - path: | - lighthouse-deb-build-*.deb - *-diff.txt - - create-release: - name: create github release - runs-on: ubuntu-latest - needs: [extract-version, build-deb, test-reproducibility] - if: ${{ github.event.inputs.dry_run != 'true' }} - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts - - - name: Prepare release assets - run: | - mkdir -p release-assets - find artifacts -name "*.deb" -exec cp {} release-assets/ \; - find artifacts -name "*.sha256" -exec cp {} release-assets/ \; - find artifacts -name "*.sha512" -exec cp {} release-assets/ \; - - echo "Release assets:" - ls -la release-assets/ - - - name: Generate release notes - run: | - VERSION=${{ needs.extract-version.outputs.VERSION }} - cat > release-notes.md << EOF - # Lighthouse ${VERSION} - Debian Packages - - This release includes reproducible Debian packages for Lighthouse ${VERSION}. - - ## Available Packages - - - **x86_64**: \`lighthouse_${VERSION#v}_x86_64.deb\` - - **aarch64**: \`lighthouse_${VERSION#v}_aarch64.deb\` - - ## Installation - - ### Ubuntu/Debian (x86_64): - \`\`\`bash - wget https://github.com/${{ github.repository }}/releases/download/${VERSION}/lighthouse_${VERSION#v}_x86_64.deb - sudo dpkg -i lighthouse_${VERSION#v}_x86_64.deb - sudo apt-get install -f # Fix any dependency issues - \`\`\` - - ### Ubuntu/Debian (ARM64): - \`\`\`bash - wget https://github.com/${{ github.repository }}/releases/download/${VERSION}/lighthouse_${VERSION#v}_aarch64.deb - sudo dpkg -i lighthouse_${VERSION#v}_aarch64.deb - sudo apt-get install -f # Fix any dependency issues - \`\`\` - - ## Verification - - All packages include SHA256 and SHA512 checksums for verification: - - \`\`\`bash - # Verify SHA256 - sha256sum -c lighthouse_${VERSION#v}_x86_64.deb.sha256 - - # Verify SHA512 - sha512sum -c lighthouse_${VERSION#v}_x86_64.deb.sha512 - \`\`\` - - ## System Service - - After installation, Lighthouse can be managed as a systemd service: - - \`\`\`bash - # Enable and start the service - sudo systemctl enable lighthouse - sudo systemctl start lighthouse - - # Check status - sudo systemctl status lighthouse - - # View logs - sudo journalctl -u lighthouse -f - \`\`\` - - ## Reproducible Builds - - These packages are built using reproducible build techniques. You can verify the build process by checking out the source code at tag \`${VERSION}\` and running: - - \`\`\`bash - make deb-cargo RUST_TARGET=x86_64-unknown-linux-gnu - make test-deb-reproducible RUST_TARGET=x86_64-unknown-linux-gnu - \`\`\` - - ## Package Details - - - **Maintainer**: Sigma Prime - - **Dependencies**: Automatically managed by dpkg/apt - - **Service Integration**: Includes systemd service unit - - **Build Method**: Reproducible builds with cargo-deb - - For more information, see the [Lighthouse documentation](https://lighthouse-book.sigmaprime.io/). - EOF - - - name: Create Release - uses: softprops/action-gh-release@v2 - with: - tag_name: ${{ needs.extract-version.outputs.VERSION }} - name: "Lighthouse ${{ needs.extract-version.outputs.VERSION }} - Debian Packages" - body_path: release-notes.md - files: release-assets/* - draft: false - prerelease: ${{ contains(needs.extract-version.outputs.VERSION, 'beta') || contains(needs.extract-version.outputs.VERSION, 'alpha') || contains(needs.extract-version.outputs.VERSION, 'rc') }} - generate_release_notes: true - - dry-run-summary: - name: dry run summary - runs-on: ubuntu-latest - needs: [extract-version, build-deb, test-reproducibility] - if: ${{ github.event.inputs.dry_run == 'true' }} - steps: - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts - - - name: Summarize dry run - run: | - VERSION=${{ needs.extract-version.outputs.VERSION }} - echo "## ๐Ÿงช Debian Package Release Dry Run Summary" - echo "" - echo "โœ… Successfully completed dry run for version ${VERSION}" - echo "" - echo "### Built Packages:" - find artifacts -name "*.deb" -exec basename {} \; | sort - echo "" - echo "### Package Sizes:" - find artifacts -name "*.deb" -exec ls -lh {} \; | awk '{print $9 ": " $5}' - echo "" - echo "### Checksums Generated:" - find artifacts -name "*.sha256" -exec basename {} \; | sort - find artifacts -name "*.sha512" -exec basename {} \; | sort - echo "" - echo "### What would happen in a real release:" - echo "- Packages would be uploaded to GitHub Releases" - echo "- Release notes would be automatically generated" - echo "- Users could install via:" - echo " \`wget + dpkg -i lighthouse_${VERSION#v}_x86_64.deb\`" - echo "" - echo "### Reproducibility Test Results:" - echo "โœ… All reproducibility tests passed" - echo "" - echo "### Next Steps:" - echo "To perform a real release, push a git tag:" - echo "\`git tag ${VERSION} && git push origin ${VERSION}\`" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f7b65f07c91..ca3804d0d53 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,7 +32,9 @@ jobs: matrix: arch: [aarch64-unknown-linux-gnu, x86_64-unknown-linux-gnu, - aarch64-apple-darwin] + aarch64-apple-darwin, + x86_64-unknown-linux-gnu-deb, + aarch64-unknown-linux-gnu-deb] include: - arch: aarch64-unknown-linux-gnu runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }} @@ -43,6 +45,17 @@ jobs: - arch: aarch64-apple-darwin runner: macos-14 profile: maxperf + # Debian package builds + - arch: x86_64-unknown-linux-gnu-deb + runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }} + profile: maxperf + rust_target: x86_64-unknown-linux-gnu + gcc_package: gcc + - arch: aarch64-unknown-linux-gnu-deb + runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }} + profile: maxperf + rust_target: aarch64-unknown-linux-gnu + gcc_package: gcc-aarch64-linux-gnu runs-on: ${{ matrix.runner }} needs: extract-version @@ -54,7 +67,7 @@ jobs: run: rustup update stable # ============================== - # Builds + # Builds - Binaries # ============================== - name: Build Lighthouse for aarch64-unknown-linux-gnu @@ -70,14 +83,51 @@ jobs: env CROSS_PROFILE=${{ matrix.profile }} make build-x86_64 - name: Move cross-compiled binary - if: contains(matrix.arch, 'unknown-linux-gnu') + if: contains(matrix.arch, 'unknown-linux-gnu') && !endsWith(matrix.arch, '-deb') run: mv target/${{ matrix.arch }}/${{ matrix.profile }}/lighthouse ~/.cargo/bin/lighthouse - name: Build Lighthouse for aarch64-apple-darwin if: matrix.arch == 'aarch64-apple-darwin' run: cargo install --path lighthouse --force --locked --features portable,gnosis --profile ${{ matrix.profile }} + # ============================== + # Builds - Debian Packages + # ============================== + + - name: Build reproducible Debian package + if: endsWith(matrix.arch, '-deb') + run: | + make deb-cargo RUST_TARGET=${{ matrix.rust_target }} PROFILE=${{ matrix.profile }} + + - name: Find and prepare Debian package + if: endsWith(matrix.arch, '-deb') + run: | + VERSION=${{ needs.extract-version.outputs.VERSION }} + DEB_FILE=$(find target/${{ matrix.rust_target }}/debian -name "*.deb" | head -1) + if [ -n "$DEB_FILE" ]; then + # Create standardized filename to match existing pattern + ARCH_SHORT=$(echo "${{ matrix.rust_target }}" | cut -d'-' -f1) + NEW_NAME="lighthouse-${{ needs.extract-version.outputs.VERSION }}-${ARCH_SHORT}.deb" + cp "$DEB_FILE" "$NEW_NAME" + echo "DEB_PACKAGE=$NEW_NAME" >> $GITHUB_ENV + + # Generate checksums + sha256sum "$NEW_NAME" > "$NEW_NAME.sha256" + sha512sum "$NEW_NAME" > "$NEW_NAME.sha512" + + echo "Debian package built: $NEW_NAME" + echo "Size: $(du -h "$NEW_NAME" | cut -f1)" + else + echo "โŒ No .deb package found" + exit 1 + fi + + # ================================== + # Binary Artifacts (existing) + # ================================== + - name: Configure GPG and create artifacts + if: startsWith(matrix.arch, 'x86_64-windows') != true && !endsWith(matrix.arch, '-deb') env: GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} @@ -96,12 +146,26 @@ jobs: done mv *tar.gz* .. + # ============================== + # Debian Package Artifacts + # ============================== + + - name: Sign Debian package + if: endsWith(matrix.arch, '-deb') + env: + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} + GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + run: | + export GPG_TTY=$(tty) + echo "$GPG_SIGNING_KEY" | gpg --batch --import + echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab "$DEB_PACKAGE" + # ======================================================================= - # Upload artifacts - # This is required to share artifacts between different jobs + # Upload artifacts - Binaries (existing) # ======================================================================= - + - name: Upload artifact + if: !endsWith(matrix.arch, '-deb') uses: actions/upload-artifact@v4 with: name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz @@ -109,12 +173,43 @@ jobs: compression-level: 0 - name: Upload signature + if: !endsWith(matrix.arch, '-deb') uses: actions/upload-artifact@v4 with: name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc path: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc compression-level: 0 + # ======================================================================= + # Upload artifacts - Debian packages + # ======================================================================= + + - name: Upload Debian package + if: endsWith(matrix.arch, '-deb') + uses: actions/upload-artifact@v4 + with: + name: ${{ env.DEB_PACKAGE }} + path: ${{ env.DEB_PACKAGE }} + compression-level: 0 + + - name: Upload Debian package signature + if: endsWith(matrix.arch, '-deb') + uses: actions/upload-artifact@v4 + with: + name: ${{ env.DEB_PACKAGE }}.asc + path: ${{ env.DEB_PACKAGE }}.asc + compression-level: 0 + + - name: Upload Debian package checksums + if: endsWith(matrix.arch, '-deb') + uses: actions/upload-artifact@v4 + with: + name: ${{ env.DEB_PACKAGE }}-checksums + path: | + ${{ env.DEB_PACKAGE }}.sha256 + ${{ env.DEB_PACKAGE }}.sha512 + compression-level: 0 + draft-release: name: Draft Release needs: [build, extract-version] @@ -203,11 +298,33 @@ jobs: | Apple logo | aarch64 | [lighthouse-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz.asc) | | Linux logo | x86_64 | [lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc) | | Raspberrypi logo | aarch64 | [lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz.asc) | + ## Debian Packages + + For Debian/Ubuntu users, we provide pre-built `.deb` packages with systemd integration: + + | System | Architecture | Package | PGP Signature | Checksums | + |:---:|:---:|:---:|:---:|:---:| + | Debian logo | x86_64 | [lighthouse-${{ env.VERSION }}-x86_64.deb](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64.deb) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64.deb.asc) | [SHA256](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64.deb.sha256) / [SHA512](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64.deb.sha512) | + | Debian logo | aarch64 | [lighthouse-${{ env.VERSION }}-aarch64.deb](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64.deb) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64.deb.asc) | [SHA256](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64.deb.sha256) / [SHA512](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64.deb.sha512) | + + ### Installation: + \`\`\`bash + # Download and install (x86_64) + wget https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64.deb + sudo dpkg -i lighthouse-${{ env.VERSION }}-x86_64.deb + sudo apt-get install -f # Fix dependencies if needed + + # Enable and start service + sudo systemctl enable lighthouse + sudo systemctl start lighthouse + \`\`\` + | | | | | + |:---:|:---:|:---:|:---:| | **System** | **Option** | - | **Resource** | | Docker logo | Docker | [${{ env.VERSION }}](https://hub.docker.com/r/${{ env.IMAGE_NAME }}/tags?page=1&ordering=last_updated&name=${{ env.VERSION }}) | [${{ env.IMAGE_NAME }}](https://hub.docker.com/r/${{ env.IMAGE_NAME }}) | ENDBODY ) - assets=(./lighthouse-*.tar.gz*/lighthouse-*.tar.gz*) + assets=(./lighthouse-*.tar.gz*/lighthouse-*.tar.gz* ./lighthouse-*.deb*/lighthouse-*.deb*) tag_name="${{ env.VERSION }}" echo "$body" | gh release create --draft -F "-" "$tag_name" "${assets[@]}" From 246a159c32805f5973be0bd0bda4aa055bfee5da Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Wed, 18 Jun 2025 13:53:35 +0000 Subject: [PATCH 27/33] refinements and following packaging best practices --- Makefile | 100 +++++---------------- lighthouse/Cargo.toml | 11 +-- lighthouse/{ => debian}/lighthouse.service | 6 +- lighthouse/debian/postinst | 25 ++++++ lighthouse/debian/postrm | 17 ++++ lighthouse/debian/prerm | 12 +++ scripts/test-deb-reproducible.sh | 93 +++++++++++++++++++ 7 files changed, 173 insertions(+), 91 deletions(-) rename lighthouse/{ => debian}/lighthouse.service (84%) create mode 100644 lighthouse/debian/postinst create mode 100644 lighthouse/debian/postrm create mode 100644 lighthouse/debian/prerm create mode 100755 scripts/test-deb-reproducible.sh diff --git a/Makefile b/Makefile index 1e46485905d..32473acd43a 100644 --- a/Makefile +++ b/Makefile @@ -333,27 +333,9 @@ clean: make -C $(EF_TESTS) clean make -C $(STATE_TRANSITION_VECTORS) clean -.PHONY: install-cargo-deb deb-cargo deb-cargo-x86_64 deb-cargo-aarch64 deb-cargo-all test-deb-reproducible install-deb-local remove-deb-local clean-deb help-deb -## Install cargo-deb if not present -install-cargo-deb: - @if ! command -v cargo-deb &> /dev/null; then \ - echo "Installing cargo-deb..."; \ - cargo install cargo-deb; \ - else \ - echo "cargo-deb already installed"; \ - fi - -## Build .deb package using cargo-deb with reproducible settings -deb-cargo: install-cargo-deb build-reproducible +.PHONY: deb-cargo +deb-cargo: build-reproducible ## Build .deb package using cargo-deb with reproducible settings @echo "Building .deb package with cargo-deb..." - @if [ ! -f "lighthouse/lighthouse.service" ]; then \ - echo "โŒ lighthouse.service not found in lighthouse/ directory"; \ - exit 1; \ - fi - @if [ ! -f "README.md" ]; then \ - echo "โŒ README.md not found in current directory"; \ - exit 1; \ - fi cd lighthouse && \ SOURCE_DATE_EPOCH=$(SOURCE_DATE) \ @@ -365,65 +347,26 @@ deb-cargo: install-cargo-deb build-reproducible @echo "โœ… Package built successfully!" @find target/$(RUST_TARGET)/debian -name "*.deb" -exec ls -la {} \; -## Build .deb for specific architectures -deb-cargo-x86_64: + +.PHONY: deb-cargo-x86_64 +deb-cargo-x86_64: ## Build .deb for specific architectures $(MAKE) deb-cargo RUST_TARGET=x86_64-unknown-linux-gnu +.PHONY: deb-cargo-aarch64 deb-cargo-aarch64: $(MAKE) deb-cargo RUST_TARGET=aarch64-unknown-linux-gnu +.PHONY: deb-cargo-all deb-cargo-all: deb-cargo-x86_64 deb-cargo-aarch64 -## Test reproducibility of cargo-deb packages -test-deb-reproducible: - @echo "Testing cargo-deb package reproducibility..." - @if ! command -v diffoscope &> /dev/null; then \ - echo "Installing diffoscope..."; \ - sudo apt-get update; \ - sudo apt-get install -y diffoscope binutils-multiarch; \ - fi - - @echo "Building first package..." - @rm -f lighthouse_*.deb lighthouse-deb-*.deb - @$(MAKE) clean || true - @$(MAKE) deb-cargo - @FIRST_PACKAGE=$$(find target/$(RUST_TARGET)/debian -name "*.deb" | head -1); \ - if [ -n "$$FIRST_PACKAGE" ]; then \ - cp "$$FIRST_PACKAGE" ./lighthouse-deb-build-1.deb; \ - else \ - echo "โŒ First package not found"; exit 1; \ - fi - - @echo "Building second package..." - @$(MAKE) clean || true - @$(MAKE) deb-cargo - @SECOND_PACKAGE=$$(find target/$(RUST_TARGET)/debian -name "*.deb" | head -1); \ - if [ -n "$$SECOND_PACKAGE" ]; then \ - cp "$$SECOND_PACKAGE" ./lighthouse-deb-build-2.deb; \ - else \ - echo "โŒ Second package not found"; exit 1; \ - fi - - @echo "Comparing packages..." - @echo "=== Package sizes ===" - @ls -la lighthouse-deb-build-*.deb - @echo "=== SHA256 checksums ===" - @sha256sum lighthouse-deb-build-*.deb - - @if cmp -s lighthouse-deb-build-1.deb lighthouse-deb-build-2.deb; then \ - echo "โœ… SUCCESS: cargo-deb packages are identical!"; \ - echo "โœ… Reproducible build PASSED"; \ - else \ - echo "โŒ FAILED: cargo-deb packages differ"; \ - echo "Running detailed analysis with diffoscope..."; \ - diffoscope --text lighthouse-deb-build-1.deb lighthouse-deb-build-2.deb > cargo-deb-diff.txt || true; \ - echo "Differences saved to cargo-deb-diff.txt"; \ - echo "โŒ Reproducible build FAILED"; \ - exit 1; \ - fi -## Install .deb package locally for testing -install-deb-local: +.PHONY: test-deb-reproducible +test-deb-reproducible: ## Test reproducibility of cargo-deb packages + @./scripts/test-deb-reproducible.sh $(RUST_TARGET) + + +.PHONY: install-deb-local +install-deb-local: ## Install .deb package locally for testing @PACKAGE=$$(find target/$(RUST_TARGET)/debian -name "*.deb" | head -1); \ if [ -n "$$PACKAGE" ]; then \ echo "Installing lighthouse package: $$PACKAGE"; \ @@ -440,22 +383,23 @@ install-deb-local: echo "โŒ No .deb package found. Run 'make deb-cargo' first."; \ fi -## Remove installed lighthouse package -remove-deb-local: + +.PHONY: remove-deb-local +remove-deb-local: ## Remove installed lighthouse package @echo "Removing lighthouse package..." sudo dpkg -r lighthouse || true sudo systemctl daemon-reload || true -## Clean up debian packaging artifacts -clean-deb: +.PHONY: clean-deb +clean-deb: ## Clean up debian packaging artifacts @echo "Cleaning up debian packaging artifacts..." rm -f lighthouse_*.deb lighthouse-deb-*.deb *-diff.txt rm -rf target/*/debian/ -## Show help for debian packaging -help-deb: + +.PHONY: help-deb +help-deb: ## Show help for debian packaging @echo "Clean Debian Packaging Targets:" - @echo " install-cargo-deb - Install cargo-deb tool" @echo " deb-cargo - Build .deb package with cargo-deb" @echo " deb-cargo-x86_64 - Build x86_64 .deb package" @echo " deb-cargo-aarch64 - Build aarch64 .deb package" diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index 8ebaa2b1519..2669bd2e0b3 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -98,31 +98,24 @@ name = "lighthouse_tests" path = "tests/main.rs" [package.metadata.deb] -# Package metadata maintainer = "Sigma Prime " extended-description = """\ Lighthouse is a Rust implementation of the Ethereum beacon chain, \ built by Sigma Prime. It implements the official Ethereum 2.0 specification.""" -# Dependencies depends = "$auto, systemd" section = "net" priority = "optional" +maintainer-scripts = "debian/" # This tells cargo-deb where to find scripts # System integration systemd-units = { enable = false, start = false } # Assets to include in the package assets = [ - # Binary - use target/release/ prefix (cargo-deb will replace with actual path) ["target/release/lighthouse", "usr/bin/", "755"], - - # Systemd service (in lighthouse directory) - ["lighthouse.service", "lib/systemd/system/", "644"], - - # Documentation (in parent directory) + ["debian/lighthouse.service", "lib/systemd/system/", "644"], ["../README.md", "usr/share/doc/lighthouse/", "644"], ] -# Use default features - let Makefile handle the building default-features = false diff --git a/lighthouse/lighthouse.service b/lighthouse/debian/lighthouse.service similarity index 84% rename from lighthouse/lighthouse.service rename to lighthouse/debian/lighthouse.service index dfc2b4f9f52..4df28d4d6f0 100644 --- a/lighthouse/lighthouse.service +++ b/lighthouse/debian/lighthouse.service @@ -1,14 +1,12 @@ [Unit] -Description=Lighthouse Ethereum Beacon Chain Client -Documentation=https://lighthouse-book.sigmaprime.io/ +Description=Lighthouse Ethereum Beacon Node After=network.target Wants=network.target [Service] Type=simple -DynamicUser=true User=lighthouse -Group=eth +Group=lighthouse Restart=always RestartSec=5 TimeoutStopSec=180 diff --git a/lighthouse/debian/postinst b/lighthouse/debian/postinst new file mode 100644 index 00000000000..00c089ad912 --- /dev/null +++ b/lighthouse/debian/postinst @@ -0,0 +1,25 @@ +#!/bin/sh +set -e + +# Create lighthouse user if it doesn't exist +if ! getent passwd lighthouse >/dev/null 2>&1; then + adduser --system --group --home /var/lib/lighthouse \ + --shell /bin/false --gecos "Lighthouse Beacon Node" lighthouse +fi + +# Create data directory +mkdir -p /var/lib/lighthouse +chown lighthouse:lighthouse /var/lib/lighthouse +chmod 750 /var/lib/lighthouse + +# Create log directory +mkdir -p /var/log/lighthouse +chown lighthouse:lighthouse /var/log/lighthouse +chmod 750 /var/log/lighthouse + +# Reload systemd +systemctl daemon-reload || true + +#DEBHELPER# + +exit 0 diff --git a/lighthouse/debian/postrm b/lighthouse/debian/postrm new file mode 100644 index 00000000000..88de8652c3e --- /dev/null +++ b/lighthouse/debian/postrm @@ -0,0 +1,17 @@ +#!/bin/sh +set -e + +if [ "$1" = "purge" ]; then + # Remove user (optional - some prefer to keep it) + if getent passwd lighthouse >/dev/null 2>&1; then + deluser lighthouse || true + fi + + # Remove data directory (be careful!) + # rm -rf /var/lib/lighthouse || true + # rm -rf /var/log/lighthouse || true +fi + +#DEBHELPER# + +exit 0 diff --git a/lighthouse/debian/prerm b/lighthouse/debian/prerm new file mode 100644 index 00000000000..f8e1bcfd14a --- /dev/null +++ b/lighthouse/debian/prerm @@ -0,0 +1,12 @@ +#!/bin/sh +set -e + +if [ "$1" = "remove" ]; then + # Stop service if running + systemctl stop lighthouse || true + systemctl disable lighthouse || true +fi + +#DEBHELPER# + +exit 0 diff --git a/scripts/test-deb-reproducible.sh b/scripts/test-deb-reproducible.sh new file mode 100755 index 00000000000..b9b42555c7d --- /dev/null +++ b/scripts/test-deb-reproducible.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# +# Test reproducibility of cargo-deb packages +# +# Usage: ./scripts/test-deb-reproducible.sh [RUST_TARGET] +# +# This script builds the same Debian package twice with clean builds in between, +# then compares them to ensure reproducible builds are working correctly. + +set -euo pipefail + +# Default target if not provided +RUST_TARGET="${1:-x86_64-unknown-linux-gnu}" + +echo "๐Ÿ”„ Testing cargo-deb package reproducibility for ${RUST_TARGET}..." + +# Check if diffoscope is available, install if needed +if ! command -v diffoscope &> /dev/null; then + echo "๐Ÿ“ฆ Installing diffoscope..." + sudo apt-get update + sudo apt-get install -y diffoscope binutils-multiarch +fi + +# Clean up any existing test artifacts +echo "๐Ÿงน Cleaning up previous test artifacts..." +rm -f lighthouse_*.deb lighthouse-deb-*.deb *-diff.txt + +# Build first package +echo "๐Ÿ”จ Building first package..." +make clean || true +make deb-cargo RUST_TARGET="${RUST_TARGET}" + +FIRST_PACKAGE=$(find target/"${RUST_TARGET}"/debian -name "*.deb" | head -1) +if [ -n "$FIRST_PACKAGE" ]; then + cp "$FIRST_PACKAGE" ./lighthouse-deb-build-1.deb + echo "โœ… First package built: $(basename "$FIRST_PACKAGE")" +else + echo "โŒ First package not found" + exit 1 +fi + +# Build second package +echo "๐Ÿ”จ Building second package..." +make clean || true +make deb-cargo RUST_TARGET="${RUST_TARGET}" + +SECOND_PACKAGE=$(find target/"${RUST_TARGET}"/debian -name "*.deb" | head -1) +if [ -n "$SECOND_PACKAGE" ]; then + cp "$SECOND_PACKAGE" ./lighthouse-deb-build-2.deb + echo "โœ… Second package built: $(basename "$SECOND_PACKAGE")" +else + echo "โŒ Second package not found" + exit 1 +fi + +# Compare packages +echo "๐Ÿ“Š Comparing packages..." +echo "" +echo "=== Package sizes ===" +ls -lah lighthouse-deb-build-*.deb + +echo "" +echo "=== SHA256 checksums ===" +sha256sum lighthouse-deb-build-*.deb + +echo "" +echo "=== Binary comparison ===" +if cmp -s lighthouse-deb-build-1.deb lighthouse-deb-build-2.deb; then + echo "โœ… SUCCESS: cargo-deb packages are identical!" + echo "โœ… Reproducible build PASSED for ${RUST_TARGET}" + echo "" + echo "๐Ÿงน Cleaning up test artifacts..." + rm -f lighthouse-deb-build-*.deb + exit 0 +else + echo "โŒ FAILED: cargo-deb packages differ" + echo "๐Ÿ” Running detailed analysis with diffoscope..." + + # Generate detailed diff report + DIFF_FILE="cargo-deb-diff-${RUST_TARGET}.txt" + if diffoscope --text lighthouse-deb-build-1.deb lighthouse-deb-build-2.deb > "$DIFF_FILE" 2>/dev/null; then + echo "๐Ÿ“„ Differences saved to ${DIFF_FILE}" + echo "๐Ÿ“„ Summary of first few differences:" + head -20 "$DIFF_FILE" || true + else + echo "โš ๏ธ diffoscope encountered issues, but differences were detected" + fi + + echo "" + echo "โŒ Reproducible build FAILED for ${RUST_TARGET}" + echo "๐Ÿ’ก Tip: Check the diff file for details on what differs between builds" + exit 1 +fi From a3436b9ecbe96c14e2a8bad984c251517fd2dcaa Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Mon, 1 Dec 2025 15:43:07 +0000 Subject: [PATCH 28/33] remove merge conflict duplicates and fix lint issues --- .github/workflows/release.yml | 52 ++--------------------------------- 1 file changed, 3 insertions(+), 49 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 31e97a879d1..c7f2b21796c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -136,39 +136,7 @@ jobs: echo "Debian package built: $NEW_NAME" echo "Size: $(du -h "$NEW_NAME" | cut -f1)" else - echo "โŒ No .deb package found" - exit 1 - fi - - # ============================== - # Builds - Debian Packages - # ============================== - - - name: Build reproducible Debian package - if: endsWith(matrix.arch, '-deb') - run: | - make deb-cargo RUST_TARGET=${{ matrix.rust_target }} PROFILE=${{ matrix.profile }} - - - name: Find and prepare Debian package - if: endsWith(matrix.arch, '-deb') - run: | - VERSION=${{ needs.extract-version.outputs.VERSION }} - DEB_FILE=$(find target/${{ matrix.rust_target }}/debian -name "*.deb" | head -1) - if [ -n "$DEB_FILE" ]; then - # Create standardized filename to match existing pattern - ARCH_SHORT=$(echo "${{ matrix.rust_target }}" | cut -d'-' -f1) - NEW_NAME="lighthouse-${{ needs.extract-version.outputs.VERSION }}-${ARCH_SHORT}.deb" - cp "$DEB_FILE" "$NEW_NAME" - echo "DEB_PACKAGE=$NEW_NAME" >> $GITHUB_ENV - - # Generate checksums - sha256sum "$NEW_NAME" > "$NEW_NAME.sha256" - sha512sum "$NEW_NAME" > "$NEW_NAME.sha512" - - echo "Debian package built: $NEW_NAME" - echo "Size: $(du -h "$NEW_NAME" | cut -f1)" - else - echo "โŒ No .deb package found" + echo "No .deb package found" exit 1 fi @@ -200,20 +168,6 @@ jobs: # Debian Package Artifacts # ============================== - - name: Sign Debian package - if: endsWith(matrix.arch, '-deb') - env: - GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} - GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - run: | - export GPG_TTY=$(tty) - echo "$GPG_SIGNING_KEY" | gpg --batch --import - echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab "$DEB_PACKAGE" - - # ============================== - # Debian Package Artifacts - # ============================== - - name: Sign Debian package if: endsWith(matrix.arch, '-deb') env: @@ -229,7 +183,7 @@ jobs: # ======================================================================= - name: Upload artifact - if: !endsWith(matrix.arch, '-deb') + if: ${{ !endsWith(matrix.arch, '-deb') }} uses: actions/upload-artifact@v4 with: name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz @@ -237,7 +191,7 @@ jobs: compression-level: 0 - name: Upload signature - if: !endsWith(matrix.arch, '-deb') + if: ${{ !endsWith(matrix.arch, '-deb') }} uses: actions/upload-artifact@v4 with: name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc From e09a200d95aa55e349835d281c15299379d4d772 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Mon, 1 Dec 2025 15:55:13 +0000 Subject: [PATCH 29/33] remove testing script --- scripts/test-deb-reproducible.sh | 93 -------------------------------- 1 file changed, 93 deletions(-) delete mode 100755 scripts/test-deb-reproducible.sh diff --git a/scripts/test-deb-reproducible.sh b/scripts/test-deb-reproducible.sh deleted file mode 100755 index b9b42555c7d..00000000000 --- a/scripts/test-deb-reproducible.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash -# -# Test reproducibility of cargo-deb packages -# -# Usage: ./scripts/test-deb-reproducible.sh [RUST_TARGET] -# -# This script builds the same Debian package twice with clean builds in between, -# then compares them to ensure reproducible builds are working correctly. - -set -euo pipefail - -# Default target if not provided -RUST_TARGET="${1:-x86_64-unknown-linux-gnu}" - -echo "๐Ÿ”„ Testing cargo-deb package reproducibility for ${RUST_TARGET}..." - -# Check if diffoscope is available, install if needed -if ! command -v diffoscope &> /dev/null; then - echo "๐Ÿ“ฆ Installing diffoscope..." - sudo apt-get update - sudo apt-get install -y diffoscope binutils-multiarch -fi - -# Clean up any existing test artifacts -echo "๐Ÿงน Cleaning up previous test artifacts..." -rm -f lighthouse_*.deb lighthouse-deb-*.deb *-diff.txt - -# Build first package -echo "๐Ÿ”จ Building first package..." -make clean || true -make deb-cargo RUST_TARGET="${RUST_TARGET}" - -FIRST_PACKAGE=$(find target/"${RUST_TARGET}"/debian -name "*.deb" | head -1) -if [ -n "$FIRST_PACKAGE" ]; then - cp "$FIRST_PACKAGE" ./lighthouse-deb-build-1.deb - echo "โœ… First package built: $(basename "$FIRST_PACKAGE")" -else - echo "โŒ First package not found" - exit 1 -fi - -# Build second package -echo "๐Ÿ”จ Building second package..." -make clean || true -make deb-cargo RUST_TARGET="${RUST_TARGET}" - -SECOND_PACKAGE=$(find target/"${RUST_TARGET}"/debian -name "*.deb" | head -1) -if [ -n "$SECOND_PACKAGE" ]; then - cp "$SECOND_PACKAGE" ./lighthouse-deb-build-2.deb - echo "โœ… Second package built: $(basename "$SECOND_PACKAGE")" -else - echo "โŒ Second package not found" - exit 1 -fi - -# Compare packages -echo "๐Ÿ“Š Comparing packages..." -echo "" -echo "=== Package sizes ===" -ls -lah lighthouse-deb-build-*.deb - -echo "" -echo "=== SHA256 checksums ===" -sha256sum lighthouse-deb-build-*.deb - -echo "" -echo "=== Binary comparison ===" -if cmp -s lighthouse-deb-build-1.deb lighthouse-deb-build-2.deb; then - echo "โœ… SUCCESS: cargo-deb packages are identical!" - echo "โœ… Reproducible build PASSED for ${RUST_TARGET}" - echo "" - echo "๐Ÿงน Cleaning up test artifacts..." - rm -f lighthouse-deb-build-*.deb - exit 0 -else - echo "โŒ FAILED: cargo-deb packages differ" - echo "๐Ÿ” Running detailed analysis with diffoscope..." - - # Generate detailed diff report - DIFF_FILE="cargo-deb-diff-${RUST_TARGET}.txt" - if diffoscope --text lighthouse-deb-build-1.deb lighthouse-deb-build-2.deb > "$DIFF_FILE" 2>/dev/null; then - echo "๐Ÿ“„ Differences saved to ${DIFF_FILE}" - echo "๐Ÿ“„ Summary of first few differences:" - head -20 "$DIFF_FILE" || true - else - echo "โš ๏ธ diffoscope encountered issues, but differences were detected" - fi - - echo "" - echo "โŒ Reproducible build FAILED for ${RUST_TARGET}" - echo "๐Ÿ’ก Tip: Check the diff file for details on what differs between builds" - exit 1 -fi From e3dfb6f2eb4f05161a67a4f73f3a43c935e8b4ce Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Mon, 1 Dec 2025 15:59:21 +0000 Subject: [PATCH 30/33] lint cleanup --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c7f2b21796c..fb140147beb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -181,7 +181,7 @@ jobs: # ======================================================================= # Upload artifacts - Binaries (existing) # ======================================================================= - + - name: Upload artifact if: ${{ !endsWith(matrix.arch, '-deb') }} uses: actions/upload-artifact@v4 From b42041b872588c52aaef1649df6a63a64d6e5f67 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Mon, 1 Dec 2025 16:53:56 +0000 Subject: [PATCH 31/33] chore: remove unused makefile target --- Makefile | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 32473acd43a..73efe3f78c5 100644 --- a/Makefile +++ b/Makefile @@ -344,7 +344,7 @@ deb-cargo: build-reproducible ## Build .deb package using cargo-deb with reprodu TZ=$(TZ_VAL) \ cargo deb --target $(RUST_TARGET) --no-build - @echo "โœ… Package built successfully!" + @echo "Package built successfully!" @find target/$(RUST_TARGET)/debian -name "*.deb" -exec ls -la {} \; @@ -359,12 +359,6 @@ deb-cargo-aarch64: .PHONY: deb-cargo-all deb-cargo-all: deb-cargo-x86_64 deb-cargo-aarch64 - -.PHONY: test-deb-reproducible -test-deb-reproducible: ## Test reproducibility of cargo-deb packages - @./scripts/test-deb-reproducible.sh $(RUST_TARGET) - - .PHONY: install-deb-local install-deb-local: ## Install .deb package locally for testing @PACKAGE=$$(find target/$(RUST_TARGET)/debian -name "*.deb" | head -1); \ @@ -373,14 +367,14 @@ install-deb-local: ## Install .deb package locally for testing sudo dpkg -i "$$PACKAGE"; \ echo "Fixing dependencies if needed..."; \ sudo apt-get install -f; \ - echo "โœ… Package installed successfully!"; \ + echo "Package installed successfully!"; \ echo ""; \ echo "The lighthouse service is now available but not started."; \ echo "Your systemd service file handles user creation declaratively."; \ echo ""; \ echo "To check service status: systemctl status lighthouse"; \ else \ - echo "โŒ No .deb package found. Run 'make deb-cargo' first."; \ + echo "No .deb package found. Run 'make deb-cargo' first."; \ fi From 16fc7f5bc221b258627733320d56c6076bc597d8 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Tue, 2 Dec 2025 11:27:36 +0000 Subject: [PATCH 32/33] remove unnecessary scripts --- lighthouse/debian/postinst | 25 ------------------------- lighthouse/debian/postrm | 17 ----------------- 2 files changed, 42 deletions(-) delete mode 100644 lighthouse/debian/postinst delete mode 100644 lighthouse/debian/postrm diff --git a/lighthouse/debian/postinst b/lighthouse/debian/postinst deleted file mode 100644 index 00c089ad912..00000000000 --- a/lighthouse/debian/postinst +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh -set -e - -# Create lighthouse user if it doesn't exist -if ! getent passwd lighthouse >/dev/null 2>&1; then - adduser --system --group --home /var/lib/lighthouse \ - --shell /bin/false --gecos "Lighthouse Beacon Node" lighthouse -fi - -# Create data directory -mkdir -p /var/lib/lighthouse -chown lighthouse:lighthouse /var/lib/lighthouse -chmod 750 /var/lib/lighthouse - -# Create log directory -mkdir -p /var/log/lighthouse -chown lighthouse:lighthouse /var/log/lighthouse -chmod 750 /var/log/lighthouse - -# Reload systemd -systemctl daemon-reload || true - -#DEBHELPER# - -exit 0 diff --git a/lighthouse/debian/postrm b/lighthouse/debian/postrm deleted file mode 100644 index 88de8652c3e..00000000000 --- a/lighthouse/debian/postrm +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh -set -e - -if [ "$1" = "purge" ]; then - # Remove user (optional - some prefer to keep it) - if getent passwd lighthouse >/dev/null 2>&1; then - deluser lighthouse || true - fi - - # Remove data directory (be careful!) - # rm -rf /var/lib/lighthouse || true - # rm -rf /var/log/lighthouse || true -fi - -#DEBHELPER# - -exit 0 From f399fafc12711436913d094d63d0f0ea958ca786 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Tue, 2 Dec 2025 18:19:33 +0000 Subject: [PATCH 33/33] chore: extract release reproducible binaries in their own workflow --- .github/workflows/docker-reproducible.yml | 17 +++ .github/workflows/release-reproducible.yml | 126 ++++++++++++++++++ .github/workflows/release.yml | 147 +-------------------- Dockerfile.reproducible | 15 ++- Makefile | 4 +- 5 files changed, 162 insertions(+), 147 deletions(-) create mode 100644 .github/workflows/release-reproducible.yml diff --git a/.github/workflows/docker-reproducible.yml b/.github/workflows/docker-reproducible.yml index f3479e9468d..fc78af3fc02 100644 --- a/.github/workflows/docker-reproducible.yml +++ b/.github/workflows/docker-reproducible.yml @@ -136,6 +136,23 @@ jobs: IMAGE_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}" docker push "$IMAGE_TAG" + - name: Extract artifacts for release + run: | + # Extract binary and .deb from verified image + docker create --name extract-${{ matrix.arch }} lighthouse-verify-2-${{ matrix.arch }} + docker cp extract-${{ matrix.arch }}:/lighthouse ./lighthouse-${{ matrix.arch }} + docker cp extract-${{ matrix.arch }}:/lighthouse.deb ./lighthouse-${{ matrix.arch }}.deb + docker rm extract-${{ matrix.arch }} + + - name: Upload reproducible artifacts + uses: actions/upload-artifact@v4 + with: + name: reproducible-artifacts-${{ matrix.arch }} + path: | + lighthouse-${{ matrix.arch }} + lighthouse-${{ matrix.arch }}.deb + retention-days: 1 + - name: Clean up local images run: | docker rmi lighthouse-verify-2-${{ matrix.arch }} || true diff --git a/.github/workflows/release-reproducible.yml b/.github/workflows/release-reproducible.yml new file mode 100644 index 00000000000..7375c7cada9 --- /dev/null +++ b/.github/workflows/release-reproducible.yml @@ -0,0 +1,126 @@ +# This workflow signs and publishes reproducible artifacts +# It triggers when either Release Suite or docker-reproducible completes +# But only proceeds when BOTH have completed successfully for the same tag + +name: release-reproducible + +on: + workflow_run: + workflows: [Release Suite, docker-reproducible] + types: [completed] + +jobs: + check-both-workflows: + name: verify both workflows completed + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + outputs: + should_proceed: ${{ steps.check.outputs.should_proceed }} + version: ${{ steps.check.outputs.version }} + steps: + - name: Check if both workflows completed successfully + id: check + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Get the tag/branch from the triggering workflow + TAG="${{ github.event.workflow_run.head_branch }}" + + # Only proceed for version tags + if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then + echo "Not a version tag, skipping" + echo "should_proceed=false" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Checking workflows for tag: $TAG" + echo "version=$TAG" >> $GITHUB_OUTPUT + + # Check Release Suite status + RELEASE_SUCCESS=$(gh api /repos/${{ github.repository }}/actions/workflows/release.yml/runs \ + --jq ".workflow_runs[] | select(.head_branch == \"$TAG\" and .conclusion == \"success\") | .id" \ + | head -1) + + # Check docker-reproducible status + DOCKER_SUCCESS=$(gh api /repos/${{ github.repository }}/actions/workflows/docker-reproducible.yml/runs \ + --jq ".workflow_runs[] | select(.head_branch == \"$TAG\" and .conclusion == \"success\") | .id" \ + | head -1) + + if [[ -n "$RELEASE_SUCCESS" ]] && [[ -n "$DOCKER_SUCCESS" ]]; then + echo "Both workflows completed successfully for $TAG" + echo " - Release Suite: run $RELEASE_SUCCESS" + echo " - docker-reproducible: run $DOCKER_SUCCESS" + echo "should_proceed=true" >> $GITHUB_OUTPUT + else + echo "Waiting for both workflows to complete for $TAG" + [[ -z "$RELEASE_SUCCESS" ]] && echo " - Release Suite: not completed" + [[ -z "$DOCKER_SUCCESS" ]] && echo " - docker-reproducible: not completed" + echo "should_proceed=false" >> $GITHUB_OUTPUT + fi + + sign-and-publish: + name: sign and publish reproducible artifacts + needs: check-both-workflows + if: needs.check-both-workflows.outputs.should_proceed == 'true' + runs-on: ubuntu-latest + permissions: + contents: write + strategy: + matrix: + arch: [amd64, arm64] + include: + - arch: amd64 + rust_target: x86_64-unknown-linux-gnu + - arch: arm64 + rust_target: aarch64-unknown-linux-gnu + steps: + - name: Download reproducible artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: docker-reproducible.yml + name: reproducible-artifacts-${{ matrix.arch }} + github_token: ${{ secrets.GITHUB_TOKEN }} + branch: ${{ needs.check-both-workflows.outputs.version }} + + - name: Prepare artifacts for signing + run: | + VERSION=${{ needs.check-both-workflows.outputs.version }} + ARCH_SHORT=$(echo "${{ matrix.rust_target }}" | cut -d'-' -f1) + + # Rename binary and create tarball + mv lighthouse-${{ matrix.arch }} lighthouse-reproducible-${VERSION}-${{ matrix.rust_target }} + tar -czf lighthouse-reproducible-${VERSION}-${{ matrix.rust_target }}.tar.gz \ + lighthouse-reproducible-${VERSION}-${{ matrix.rust_target }} --remove-files + + # Rename Debian package + mv lighthouse-${{ matrix.arch }}.deb lighthouse-${VERSION}-${ARCH_SHORT}-reproducible.deb + + - name: Sign artifacts with GPG + env: + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} + GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + run: | + export GPG_TTY=$(tty) + echo -n "$GPG_SIGNING_KEY" | base64 --decode | gpg --batch --import + + VERSION=${{ needs.check-both-workflows.outputs.version }} + ARCH_SHORT=$(echo "${{ matrix.rust_target }}" | cut -d'-' -f1) + + # Sign binary tarball + echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab \ + lighthouse-reproducible-${VERSION}-${{ matrix.rust_target }}.tar.gz + + # Sign Debian package + echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab \ + lighthouse-${VERSION}-${ARCH_SHORT}-reproducible.deb + + - name: Upload reproducible artifacts to release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + VERSION=${{ needs.check-both-workflows.outputs.version }} + + # Upload all signed artifacts and their signatures + gh release upload ${VERSION} \ + lighthouse-reproducible-*.tar.gz* \ + lighthouse-*-reproducible.deb* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fb140147beb..f7b65f07c91 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,9 +32,7 @@ jobs: matrix: arch: [aarch64-unknown-linux-gnu, x86_64-unknown-linux-gnu, - aarch64-apple-darwin, - x86_64-unknown-linux-gnu-deb, - aarch64-unknown-linux-gnu-deb] + aarch64-apple-darwin] include: - arch: aarch64-unknown-linux-gnu runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }} @@ -45,17 +43,6 @@ jobs: - arch: aarch64-apple-darwin runner: macos-14 profile: maxperf - # Debian package builds - - arch: x86_64-unknown-linux-gnu-deb - runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }} - profile: maxperf - rust_target: x86_64-unknown-linux-gnu - gcc_package: gcc - - arch: aarch64-unknown-linux-gnu-deb - runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }} - profile: maxperf - rust_target: aarch64-unknown-linux-gnu - gcc_package: gcc-aarch64-linux-gnu runs-on: ${{ matrix.runner }} needs: extract-version @@ -67,25 +54,7 @@ jobs: run: rustup update stable # ============================== - # Debian build dependencies - # ============================== - - - name: Install Debian build dependencies - if: endsWith(matrix.arch, '-deb') - run: | - sudo apt-get update - sudo apt-get install -y libclang-dev cmake ${{ matrix.gcc_package }} - - - name: Install Rust target for Debian builds - if: endsWith(matrix.arch, '-deb') - run: rustup target add ${{ matrix.rust_target }} - - - name: Install cargo-deb - if: endsWith(matrix.arch, '-deb') - run: cargo install cargo-deb - - # ============================== - # Builds - Binaries + # Builds # ============================== - name: Build Lighthouse for aarch64-unknown-linux-gnu @@ -101,51 +70,14 @@ jobs: env CROSS_PROFILE=${{ matrix.profile }} make build-x86_64 - name: Move cross-compiled binary - if: contains(matrix.arch, 'unknown-linux-gnu') && !endsWith(matrix.arch, '-deb') + if: contains(matrix.arch, 'unknown-linux-gnu') run: mv target/${{ matrix.arch }}/${{ matrix.profile }}/lighthouse ~/.cargo/bin/lighthouse - name: Build Lighthouse for aarch64-apple-darwin if: matrix.arch == 'aarch64-apple-darwin' run: cargo install --path lighthouse --force --locked --features portable,gnosis --profile ${{ matrix.profile }} - # ============================== - # Builds - Debian Packages - # ============================== - - - name: Build reproducible Debian package - if: endsWith(matrix.arch, '-deb') - run: | - make deb-cargo RUST_TARGET=${{ matrix.rust_target }} PROFILE=${{ matrix.profile }} - - - name: Find and prepare Debian package - if: endsWith(matrix.arch, '-deb') - run: | - VERSION=${{ needs.extract-version.outputs.VERSION }} - DEB_FILE=$(find target/${{ matrix.rust_target }}/debian -name "*.deb" | head -1) - if [ -n "$DEB_FILE" ]; then - # Create standardized filename to match existing pattern - ARCH_SHORT=$(echo "${{ matrix.rust_target }}" | cut -d'-' -f1) - NEW_NAME="lighthouse-${{ needs.extract-version.outputs.VERSION }}-${ARCH_SHORT}.deb" - cp "$DEB_FILE" "$NEW_NAME" - echo "DEB_PACKAGE=$NEW_NAME" >> $GITHUB_ENV - - # Generate checksums - sha256sum "$NEW_NAME" > "$NEW_NAME.sha256" - sha512sum "$NEW_NAME" > "$NEW_NAME.sha512" - - echo "Debian package built: $NEW_NAME" - echo "Size: $(du -h "$NEW_NAME" | cut -f1)" - else - echo "No .deb package found" - exit 1 - fi - - # ================================== - # Binary Artifacts (existing) - # ================================== - - name: Configure GPG and create artifacts - if: startsWith(matrix.arch, 'x86_64-windows') != true && !endsWith(matrix.arch, '-deb') env: GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} @@ -164,26 +96,12 @@ jobs: done mv *tar.gz* .. - # ============================== - # Debian Package Artifacts - # ============================== - - - name: Sign Debian package - if: endsWith(matrix.arch, '-deb') - env: - GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} - GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - run: | - export GPG_TTY=$(tty) - echo "$GPG_SIGNING_KEY" | gpg --batch --import - echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab "$DEB_PACKAGE" - # ======================================================================= - # Upload artifacts - Binaries (existing) + # Upload artifacts + # This is required to share artifacts between different jobs # ======================================================================= - name: Upload artifact - if: ${{ !endsWith(matrix.arch, '-deb') }} uses: actions/upload-artifact@v4 with: name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz @@ -191,43 +109,12 @@ jobs: compression-level: 0 - name: Upload signature - if: ${{ !endsWith(matrix.arch, '-deb') }} uses: actions/upload-artifact@v4 with: name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc path: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc compression-level: 0 - # ======================================================================= - # Upload artifacts - Debian packages - # ======================================================================= - - - name: Upload Debian package - if: endsWith(matrix.arch, '-deb') - uses: actions/upload-artifact@v4 - with: - name: ${{ env.DEB_PACKAGE }} - path: ${{ env.DEB_PACKAGE }} - compression-level: 0 - - - name: Upload Debian package signature - if: endsWith(matrix.arch, '-deb') - uses: actions/upload-artifact@v4 - with: - name: ${{ env.DEB_PACKAGE }}.asc - path: ${{ env.DEB_PACKAGE }}.asc - compression-level: 0 - - - name: Upload Debian package checksums - if: endsWith(matrix.arch, '-deb') - uses: actions/upload-artifact@v4 - with: - name: ${{ env.DEB_PACKAGE }}-checksums - path: | - ${{ env.DEB_PACKAGE }}.sha256 - ${{ env.DEB_PACKAGE }}.sha512 - compression-level: 0 - draft-release: name: Draft Release needs: [build, extract-version] @@ -316,33 +203,11 @@ jobs: | Apple logo | aarch64 | [lighthouse-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz.asc) | | Linux logo | x86_64 | [lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc) | | Raspberrypi logo | aarch64 | [lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz.asc) | - ## Debian Packages - - For Debian/Ubuntu users, we provide pre-built `.deb` packages with systemd integration: - - | System | Architecture | Package | PGP Signature | Checksums | - |:---:|:---:|:---:|:---:|:---:| - | Debian logo | x86_64 | [lighthouse-${{ env.VERSION }}-x86_64.deb](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64.deb) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64.deb.asc) | [SHA256](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64.deb.sha256) / [SHA512](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64.deb.sha512) | - | Debian logo | aarch64 | [lighthouse-${{ env.VERSION }}-aarch64.deb](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64.deb) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64.deb.asc) | [SHA256](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64.deb.sha256) / [SHA512](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64.deb.sha512) | - - ### Installation: - \`\`\`bash - # Download and install (x86_64) - wget https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64.deb - sudo dpkg -i lighthouse-${{ env.VERSION }}-x86_64.deb - sudo apt-get install -f # Fix dependencies if needed - - # Enable and start service - sudo systemctl enable lighthouse - sudo systemctl start lighthouse - \`\`\` - | | | | | - |:---:|:---:|:---:|:---:| | **System** | **Option** | - | **Resource** | | Docker logo | Docker | [${{ env.VERSION }}](https://hub.docker.com/r/${{ env.IMAGE_NAME }}/tags?page=1&ordering=last_updated&name=${{ env.VERSION }}) | [${{ env.IMAGE_NAME }}](https://hub.docker.com/r/${{ env.IMAGE_NAME }}) | ENDBODY ) - assets=(./lighthouse-*.tar.gz*/lighthouse-*.tar.gz* ./lighthouse-*.deb*/lighthouse-*.deb*) + assets=(./lighthouse-*.tar.gz*/lighthouse-*.tar.gz*) tag_name="${{ env.VERSION }}" echo "$body" | gh release create --draft -F "-" "$tag_name" "${assets[@]}" diff --git a/Dockerfile.reproducible b/Dockerfile.reproducible index 903515373f8..7a210759461 100644 --- a/Dockerfile.reproducible +++ b/Dockerfile.reproducible @@ -11,11 +11,18 @@ ARG RUST_TARGET="x86_64-unknown-linux-gnu" COPY ./ /app WORKDIR /app -# Build the project with the reproducible settings -RUN make build-reproducible +# Build the project with reproducible settings and create Debian package +RUN make build-reproducible RUST_TARGET=${RUST_TARGET} && \ + make deb-cargo RUST_TARGET=${RUST_TARGET} -# Move the binary to a standard location -RUN mv /app/target/${RUST_TARGET}/release/lighthouse /lighthouse +# Move artifacts to standard locations for easier extraction +RUN mv /app/target/${RUST_TARGET}/release/lighthouse /lighthouse && \ + mv /app/target/${RUST_TARGET}/debian/*.deb /lighthouse.deb + +# Artifacts stage for extracting files in CI/CD workflows +FROM scratch AS artifacts +COPY --from=builder /lighthouse /lighthouse +COPY --from=builder /lighthouse.deb /lighthouse.deb # Create a minimal final image with just the binary FROM gcr.io/distroless/cc-debian12:nonroot-6755e21ccd99ddead6edc8106ba03888cbeed41a diff --git a/Makefile b/Makefile index 73efe3f78c5..fa628043220 100644 --- a/Makefile +++ b/Makefile @@ -336,13 +336,13 @@ clean: .PHONY: deb-cargo deb-cargo: build-reproducible ## Build .deb package using cargo-deb with reproducible settings @echo "Building .deb package with cargo-deb..." - + cargo install cargo-deb@3.6.0 --locked cd lighthouse && \ SOURCE_DATE_EPOCH=$(SOURCE_DATE) \ CARGO_INCREMENTAL=$(CARGO_INCREMENTAL_VAL) \ LC_ALL=$(LOCALE_VAL) \ TZ=$(TZ_VAL) \ - cargo deb --target $(RUST_TARGET) --no-build + cargo deb --target $(RUST_TARGET) --no-build --no-dbgsym --no-strip @echo "Package built successfully!" @find target/$(RUST_TARGET)/debian -name "*.deb" -exec ls -la {} \;