diff --git a/.github/workflows/postgresql.yml b/.github/workflows/postgresql.yml index f66e7ae..16bb70b 100644 --- a/.github/workflows/postgresql.yml +++ b/.github/workflows/postgresql.yml @@ -92,16 +92,73 @@ jobs: username: ${{ secrets._TEMP_DOCKERHUB_USER }} password: ${{ secrets._TEMP_DOCKERHUB_PASSWORD }} - - name: build and push amd64 image + - name: Build amd64 image locally run: | docker buildx build \ --build-arg GIT_COMMIT=${{ github.sha }} \ - --push \ + --load \ --platform linux/amd64 \ --no-cache-filter trimmed \ --no-cache-filter trimmed-all \ $TAG . + - name: Install slim toolkit + run: | + curl -sL https://raw.githubusercontent.com/slimtoolkit/slim/master/scripts/install-slim.sh | sudo -E bash - + + - name: Slim the image + run: | + # Extract image name from TAG (remove -t prefix) + IMAGE_NAME=$(echo "$TAG" | sed 's/-t //') + slim build --target $IMAGE_NAME \ + --tag ${IMAGE_NAME}-slim \ + --http-probe=false \ + --continue-after=15 \ + --expose=5432 \ + --expose=8008 \ + --expose=8081 \ + --include-path=/usr/lib/postgresql \ + --include-path=/usr/lib/x86_64-linux-gnu \ + --include-path=/usr/share/postgresql \ + --include-path=/usr/share/proj \ + --include-path=/usr/share/gdal \ + --include-path=/etc/alternatives \ + --preserve-path=/var/lib/postgresql \ + --preserve-path=/docker-entrypoint-initdb.d \ + --preserve-path=/or-entrypoint.sh \ + --preserve-path=/etc/postgresql \ + --preserve-path=/etc/ssl \ + --include-shell \ + --include-bin=/usr/bin/sort \ + --include-bin=/usr/bin/find \ + --include-bin=/usr/bin/xargs \ + --include-bin=/usr/bin/dirname \ + --include-bin=/usr/bin/basename \ + --include-bin=/usr/bin/head \ + --include-bin=/usr/bin/tail \ + --include-bin=/usr/bin/wc \ + --include-bin=/usr/bin/cut \ + --include-bin=/usr/bin/tr \ + --include-bin=/usr/bin/sed \ + --include-bin=/usr/bin/awk \ + --include-bin=/usr/bin/grep \ + --include-bin=/bin/cat \ + --include-bin=/bin/mv \ + --include-bin=/bin/mkdir \ + --include-bin=/bin/chmod \ + --include-bin=/bin/rm \ + --include-bin=/bin/cp \ + --include-bin=/bin/touch \ + --include-bin=/usr/bin/id \ + --include-bin=/usr/bin/env + # Replace original with slim version + docker tag ${IMAGE_NAME}-slim $IMAGE_NAME + + - name: Push amd64 image + run: | + IMAGE_NAME=$(echo "$TAG" | sed 's/-t //') + docker push $IMAGE_NAME + image_postgresql_arm64: needs: image_postgresql_amd64 runs-on: ubuntu-latest @@ -171,16 +228,73 @@ jobs: username: ${{ secrets._TEMP_DOCKERHUB_USER }} password: ${{ secrets._TEMP_DOCKERHUB_PASSWORD }} - - name: build and push arm64 image + - name: Build arm64 image locally run: | docker buildx build \ --build-arg GIT_COMMIT=${{ github.sha }} \ - --push \ - --platform linux/aarch64 \ + --load \ + --platform linux/arm64 \ --no-cache-filter trimmed \ --no-cache-filter trimmed-all \ $TAG . + - name: Install slim toolkit + run: | + curl -sL https://raw.githubusercontent.com/slimtoolkit/slim/master/scripts/install-slim.sh | sudo -E bash - + + - name: Slim the image + run: | + # Extract image name from TAG (remove -t prefix) + IMAGE_NAME=$(echo "$TAG" | sed 's/-t //') + slim build --target $IMAGE_NAME \ + --tag ${IMAGE_NAME}-slim \ + --http-probe=false \ + --continue-after=15 \ + --expose=5432 \ + --expose=8008 \ + --expose=8081 \ + --include-path=/usr/lib/postgresql \ + --include-path=/usr/lib/aarch64-linux-gnu \ + --include-path=/usr/share/postgresql \ + --include-path=/usr/share/proj \ + --include-path=/usr/share/gdal \ + --include-path=/etc/alternatives \ + --preserve-path=/var/lib/postgresql \ + --preserve-path=/docker-entrypoint-initdb.d \ + --preserve-path=/or-entrypoint.sh \ + --preserve-path=/etc/postgresql \ + --preserve-path=/etc/ssl \ + --include-shell \ + --include-bin=/usr/bin/sort \ + --include-bin=/usr/bin/find \ + --include-bin=/usr/bin/xargs \ + --include-bin=/usr/bin/dirname \ + --include-bin=/usr/bin/basename \ + --include-bin=/usr/bin/head \ + --include-bin=/usr/bin/tail \ + --include-bin=/usr/bin/wc \ + --include-bin=/usr/bin/cut \ + --include-bin=/usr/bin/tr \ + --include-bin=/usr/bin/sed \ + --include-bin=/usr/bin/awk \ + --include-bin=/usr/bin/grep \ + --include-bin=/bin/cat \ + --include-bin=/bin/mv \ + --include-bin=/bin/mkdir \ + --include-bin=/bin/chmod \ + --include-bin=/bin/rm \ + --include-bin=/bin/cp \ + --include-bin=/bin/touch \ + --include-bin=/usr/bin/id \ + --include-bin=/usr/bin/env + # Replace original with slim version + docker tag ${IMAGE_NAME}-slim $IMAGE_NAME + + - name: Push arm64 image + run: | + IMAGE_NAME=$(echo "$TAG" | sed 's/-t //') + docker push $IMAGE_NAME + create_manifest: needs: [image_postgresql_amd64, image_postgresql_arm64] runs-on: ubuntu-latest diff --git a/Dockerfile b/Dockerfile index fd95e43..756dcde 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,51 +1,79 @@ - ARG PG_MAJOR=17 +ARG PREV_PG_MAJOR=15 ARG TIMESCALE_VERSION=2.22 -FROM timescale/timescaledb-ha:pg17-ts${TIMESCALE_VERSION} AS trimmed -LABEL maintainer="support@openremote.io" +# Stage 1: Get PostgreSQL ${PREV_PG_MAJOR} binaries for upgrade support +FROM timescale/timescaledb-ha:pg${PG_MAJOR}-ts${TIMESCALE_VERSION}-all AS pg-all USER root -# install fd to find files to speed up chown and chgrp -RUN apt-get update && apt-get install -y fd-find && rm -rf /var/lib/apt/lists/* - -# Give postgres user the same UID and GID as the old alpine postgres image to simplify migration of existing DB -RUN usermod -u 70 postgres \ - && groupmod -g 70 postgres \ - && (fd / -group 1000 -exec chgrp -h postgres {} \; || true) \ - && (fd / -user 1000 -exec chown -h postgres {} \; || true) +ARG PREV_PG_MAJOR -# Set PGDATA to the same location as our old alpine image -RUN mkdir -p /var/lib/postgresql && mv /home/postgres/pgdata/* /var/lib/postgresql/ && chown -R postgres:postgres /var/lib/postgresql +# Strip debug symbols and remove unnecessary files from PG ${PREV_PG_MAJOR} in this stage +# For pg_upgrade we only need bin/ and lib/, plus minimal share files (NOT extensions) +RUN find /usr/lib/postgresql/${PREV_PG_MAJOR} -type f -name '*.so*' -exec strip --strip-unneeded {} \; 2>/dev/null || true \ + && find /usr/lib/postgresql/${PREV_PG_MAJOR} -type f -executable -exec strip --strip-unneeded {} \; 2>/dev/null || true \ + && rm -rf /usr/share/postgresql/${PREV_PG_MAJOR}/extension \ + /usr/share/postgresql/${PREV_PG_MAJOR}/man \ + /usr/share/postgresql/${PREV_PG_MAJOR}/doc \ + /usr/share/postgresql/${PREV_PG_MAJOR}/contrib -# Add custom entry point (see file header for details) -COPY or-entrypoint.sh / -RUN chmod +x /or-entrypoint.sh +# Stage 2: Prepare the main image with UID/GID changes and cleanup +FROM timescale/timescaledb-ha:pg${PG_MAJOR}-ts${TIMESCALE_VERSION} AS final +LABEL maintainer="support@openremote.io" -# Add custom initdb script(s) -COPY docker-entrypoint-initdb.d/ /docker-entrypoint-initdb.d/ -RUN chmod +x /docker-entrypoint-initdb.d/* +USER root +ARG PREV_PG_MAJOR -# Below is mostly copied from https://github.com/timescale/timescaledb-docker-ha/blob/master/Dockerfile (with OR specific entrypoint, -# workdir and OR env defaults) +# Copy only PG ${PREV_PG_MAJOR} bin directories for pg_upgrade (lib is needed for binaries to work) +COPY --from=pg-all /usr/lib/postgresql/${PREV_PG_MAJOR}/bin /usr/lib/postgresql/${PREV_PG_MAJOR}/bin +COPY --from=pg-all /usr/lib/postgresql/${PREV_PG_MAJOR}/lib /usr/lib/postgresql/${PREV_PG_MAJOR}/lib +# Copy minimal share files needed for pg_upgrade (excluding extensions which are ~500MB each) +COPY --from=pg-all /usr/share/postgresql/${PREV_PG_MAJOR} /usr/share/postgresql/${PREV_PG_MAJOR} -# Get the -all variant which contains multiple PostgreSQL versions -# According to TimescaleDB docs: "timescale/timescaledb-ha images have the files necessary to run previous versions" -FROM timescale/timescaledb-ha:pg17-ts${TIMESCALE_VERSION}-all AS trimmed-all +# Copy entrypoint scripts +COPY or-entrypoint.sh / +COPY docker-entrypoint-initdb.d/ /docker-entrypoint-initdb.d/ -## Create a smaller Docker image from the builder image -FROM scratch -COPY --from=trimmed / / +# Install fd-find, fix UID/GID, setup directories, strip binaries, and cleanup - all in one layer +RUN apt-get update && apt-get install -y --no-install-recommends fd-find \ + # Give postgres user the same UID and GID as the old alpine postgres image + && usermod -u 70 postgres \ + && groupmod -g 70 postgres \ + && (fdfind . / -group 1000 -exec chgrp -h postgres {} \; 2>/dev/null || true) \ + && (fdfind . / -user 1000 -exec chown -h postgres {} \; 2>/dev/null || true) \ + # Set PGDATA to the same location as our old alpine image + && mkdir -p /var/lib/postgresql \ + && mv /home/postgres/pgdata/* /var/lib/postgresql/ \ + && chown -R postgres:postgres /var/lib/postgresql \ + # Make scripts executable + && chmod +x /or-entrypoint.sh /docker-entrypoint-initdb.d/* \ + # Strip debug symbols from PostgreSQL binaries to reduce size + && find /usr/lib/postgresql -type f -name '*.so*' -exec strip --strip-unneeded {} \; 2>/dev/null || true \ + && find /usr/lib/postgresql -type f -executable -exec strip --strip-unneeded {} \; 2>/dev/null || true \ + # Remove fd-find and clean up + && apt-get purge -y fd-find \ + && apt-get autoremove -y --purge \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + /var/cache/apt/* \ + /var/log/* \ + /usr/share/doc/* \ + /usr/share/man/* \ + /usr/share/info/* \ + /usr/share/lintian/* \ + /usr/share/locale/* \ + /tmp/* \ + /var/tmp/* \ + /root/.cache \ + /home/postgres/.cache \ + /usr/local/lib/pgai \ + /usr/share/postgresql/*/man \ + /usr/share/postgresql/*/doc ARG PG_MAJOR - -## Copy only PostgreSQL 14 and 15 for upgrade support -COPY --from=trimmed-all /usr/lib/postgresql/14 /usr/lib/postgresql/14 -COPY --from=trimmed-all /usr/lib/postgresql/15 /usr/lib/postgresql/15 -COPY --from=trimmed-all /usr/share/postgresql/14 /usr/share/postgresql/14 -COPY --from=trimmed-all /usr/share/postgresql/15 /usr/share/postgresql/15 +ARG PREV_PG_MAJOR # Increment this to indicate that a re-index should be carried out on first startup with existing data; REINDEX can still be overidden # with OR_DISABLE_REINDEX=true @@ -80,6 +108,7 @@ ENV PGROOT=/var/lib/postgresql \ POSTGRES_USER=${POSTGRES_USER:-postgres} \ POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} \ PG_MAJOR=$PG_MAJOR \ + PREV_PG_MAJOR=$PREV_PG_MAJOR \ OR_REINDEX_COUNTER=${OR_REINDEX_COUNTER} \ OR_DISABLE_REINDEX=${OR_DISABLE_REINDEX:-false} \ POSTGRES_MAX_CONNECTIONS=${POSTGRES_MAX_CONNECTIONS:-50} \ diff --git a/README.md b/README.md index cb77985..0c190c1 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,36 @@ # Postgresql docker image -[![build multirach postgresql Docker image and push to it dockerhub](https://github.com/openremote/postgresql/actions/workflows/postgresql.yml/badge.svg)](https://github.com/openremote/postgresql/actions/workflows/postgresql.yml) +[![build multiarch postgresql Docker image and push to dockerhub](https://github.com/openremote/postgresql/actions/workflows/postgresql.yml/badge.svg)](https://github.com/openremote/postgresql/actions/workflows/postgresql.yml) - POSTGIS and TimescaleDB (inc. toolkit for hyperfunctions) image built for aarch64 support using `timescaledev/timescaledb-ha` base image with: +POSTGIS and TimescaleDB (inc. toolkit for hyperfunctions) image built for amd64 and arm64 using `timescale/timescaledb-ha` base image with: - OR specific ENV variables and a healthcheck added - Easy configuration of `max_connections` using `POSTGRES_MAX_CONNECTIONS` environment variable (set to `-1` to disable this setting) - PGDATA path set to match old Alpine image (for ease of DB migration) - POSTGRES user UID and GID changed to match old Alpine image (for ease of DB migration) - Auto upgrade of database with PG major version changes from previous PG major version; can be disabled using - OR_DISABLE_AUTO_UPGRADE=true. + `OR_DISABLE_AUTO_UPGRADE=true` - Auto upgrade of timescaleDB extension when a new version is available in the container; can be disabled using - OR_DISABLE_AUTO_UPGRADE=true. + `OR_DISABLE_AUTO_UPGRADE=true` - OR_DISABLE_REINDEX env variable with associated scripts to determine if a REINDEX of the entire DB should be carried - out at first startup with existing DB (checks whether or not $PGDATA/OR_REINDEX_COUNTER.$OR_REINDEX_COUNTER exists). + out at first startup with existing DB (checks whether or not `$PGDATA/OR_REINDEX_COUNTER.$OR_REINDEX_COUNTER` exists). This is used when a collation change has occurred (glibc version change, muslc <-> glibc) which can break the indexes; migration can either be manually handled or auto handled depending on OR_DISABLE_REINDEX env variable value. - NOTE THAT A REINDEX CAN TAKE A LONG TIME DEPENDING ON THE SIZE OF THE DB! And startup will be delayed until completed + NOTE THAT A REINDEX CAN TAKE A LONG TIME DEPENDING ON THE SIZE OF THE DB! And startup will be delayed until completed. This functionality is intended to simplify migration for basic users; advanced users with large DBs should take care of this themselves. +- **Slimmed images** using [slim toolkit](https://github.com/slimtoolkit/slim) to reduce image size by ~60% -`timescale/timescaledb-ha` image is ubuntu based and only currently supports amd64; they are working on ARM64 support in timescaledev/timescaledb-ha see: +## Local Development -https://github.com/timescale/timescaledb-docker-ha/pull/355 +To build and slim the image locally: -See this issue for POSTGIS base image aarch64 support discussion: +```bash +./build_and_slim.sh +``` -https://github.com/postgis/docker-postgis/issues/216 - -TODO: Switch over to timescale/timescaledb-ha once arm64 supported +This will: +1. Build the regular Docker image +2. Use slim toolkit to create an optimized version with reduced size ## Upgrading ***NOTE: If you change the version of container you use then make sure you have backed up your DB first as this container will try to auto upgrade your DB and/or TimescaleDB extension; this auto upgrade functionality can be disabled using `OR_DISABLE_AUTO_UPGRADE=true`*** diff --git a/build_and_slim.sh b/build_and_slim.sh new file mode 100755 index 0000000..1562f1c --- /dev/null +++ b/build_and_slim.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# PostgreSQL version tag +PG_MAJOR=17 +IMAGE_NAME="openremote/postgresql:pg${PG_MAJOR}" +SLIM_IMAGE_NAME="openremote/postgresql:pg${PG_MAJOR}-slim" + +# Build the regular Docker image first +echo "Building regular Docker image..." +docker build -t $IMAGE_NAME . + +# Use slimtoolkit to create an optimized version +echo "Creating optimized image with slimtoolkit..." +slim build --target $IMAGE_NAME \ + --tag $SLIM_IMAGE_NAME \ + --http-probe=false \ + --continue-after=10 \ + --expose=5432 \ + --expose=8008 \ + --expose=8081 \ + --include-path=/usr/lib/postgresql \ + --include-path=/usr/lib/aarch64-linux-gnu \ + --include-path=/usr/share/postgresql \ + --include-path=/usr/share/proj \ + --include-path=/usr/share/gdal \ + --include-path=/etc/alternatives \ + --preserve-path=/var/lib/postgresql \ + --preserve-path=/docker-entrypoint-initdb.d \ + --preserve-path=/or-entrypoint.sh \ + --preserve-path=/etc/postgresql \ + --preserve-path=/etc/ssl \ + --include-shell \ + --include-bin=/usr/bin/sort \ + --include-bin=/usr/bin/find \ + --include-bin=/usr/bin/xargs \ + --include-bin=/usr/bin/dirname \ + --include-bin=/usr/bin/basename \ + --include-bin=/usr/bin/head \ + --include-bin=/usr/bin/tail \ + --include-bin=/usr/bin/wc \ + --include-bin=/usr/bin/cut \ + --include-bin=/usr/bin/tr \ + --include-bin=/usr/bin/sed \ + --include-bin=/usr/bin/awk \ + --include-bin=/usr/bin/grep \ + --include-bin=/bin/cat \ + --include-bin=/bin/mv \ + --include-bin=/bin/mkdir \ + --include-bin=/bin/chmod \ + --include-bin=/bin/rm \ + --include-bin=/bin/cp \ + --include-bin=/bin/touch \ + --include-bin=/usr/bin/id \ + --include-bin=/usr/bin/env \ + --show-clogs diff --git a/or-entrypoint.sh b/or-entrypoint.sh index de041c5..913bf2d 100644 --- a/or-entrypoint.sh +++ b/or-entrypoint.sh @@ -77,6 +77,20 @@ if [ -n "$DATABASE_ALREADY_EXISTS" ]; then echo "---------------------------------------------------------------------------------" fi + # Check if the old DB version is supported for upgrade + if [ "$DB_VERSION" != "$PG_MAJOR" ] && [ "$OR_DISABLE_AUTO_UPGRADE" != "true" ]; then + # Only PREV_PG_MAJOR and PG_MAJOR are supported + if [ "$DB_VERSION" != "$PREV_PG_MAJOR" ] && [ "$DB_VERSION" != "$PG_MAJOR" ]; then + echo "********************************************************************************" + echo "ERROR: Database version ${DB_VERSION} is not supported for automatic upgrade!" + echo "This image only supports upgrading from PostgreSQL ${PREV_PG_MAJOR} to ${PG_MAJOR}." + echo "To upgrade from ${DB_VERSION}, you need to use an intermediate image version" + echo "that supports upgrading from ${DB_VERSION} first." + echo "********************************************************************************" + exit 12 + fi + fi + # STEP 1: Upgrade TimescaleDB on OLD PostgreSQL version (if needed) # This must happen BEFORE pg_upgrade so both old and new PG have the same TS version if [ "$DB_VERSION" != "$PG_MAJOR" ] && [ "$OR_DISABLE_AUTO_UPGRADE" != "true" ]; then