Feature/image size optimisation

.github/workflows/postgresql.yml

@@ -92,16 +92,73 @@ jobs:
           username: ${{ secrets._TEMP_DOCKERHUB_USER }}
           password: ${{ secrets._TEMP_DOCKERHUB_PASSWORD }}
 
-      - name: build and push amd64 image
+      - name: Build amd64 image locally
         run: |
           docker buildx build \
             --build-arg GIT_COMMIT=${{ github.sha }} \
-            --push \
+            --load \
             --platform linux/amd64 \
             --no-cache-filter trimmed \
             --no-cache-filter trimmed-all \
             $TAG .
 
+      - name: Install slim toolkit
+        run: |
+          curl -sL https://raw.githubusercontent.com/slimtoolkit/slim/master/scripts/install-slim.sh | sudo -E bash -
+
+      - name: Slim the image
+        run: |
+          # Extract image name from TAG (remove -t prefix)
+          IMAGE_NAME=$(echo "$TAG" | sed 's/-t //')
+          slim build --target $IMAGE_NAME \
+            --tag ${IMAGE_NAME}-slim \
+            --http-probe=false \
+            --continue-after=15 \
+            --expose=5432 \
+            --expose=8008 \
+            --expose=8081 \
+            --include-path=/usr/lib/postgresql \
+            --include-path=/usr/lib/x86_64-linux-gnu \
+            --include-path=/usr/share/postgresql \
+            --include-path=/usr/share/proj \
+            --include-path=/usr/share/gdal \
+            --include-path=/etc/alternatives \
+            --preserve-path=/var/lib/postgresql \
+            --preserve-path=/docker-entrypoint-initdb.d \
+            --preserve-path=/or-entrypoint.sh \
+            --preserve-path=/etc/postgresql \
+            --preserve-path=/etc/ssl \
+            --include-shell \
+            --include-bin=/usr/bin/sort \
+            --include-bin=/usr/bin/find \
+            --include-bin=/usr/bin/xargs \
+            --include-bin=/usr/bin/dirname \
+            --include-bin=/usr/bin/basename \
+            --include-bin=/usr/bin/head \
+            --include-bin=/usr/bin/tail \
+            --include-bin=/usr/bin/wc \
+            --include-bin=/usr/bin/cut \
+            --include-bin=/usr/bin/tr \
+            --include-bin=/usr/bin/sed \
+            --include-bin=/usr/bin/awk \
+            --include-bin=/usr/bin/grep \
+            --include-bin=/bin/cat \
+            --include-bin=/bin/mv \
+            --include-bin=/bin/mkdir \
+            --include-bin=/bin/chmod \
+            --include-bin=/bin/rm \
+            --include-bin=/bin/cp \
+            --include-bin=/bin/touch \
+            --include-bin=/usr/bin/id \
+            --include-bin=/usr/bin/env
+          # Replace original with slim version
+          docker tag ${IMAGE_NAME}-slim $IMAGE_NAME
+
+      - name: Push amd64 image
+        run: |
+          IMAGE_NAME=$(echo "$TAG" | sed 's/-t //')
+          docker push $IMAGE_NAME
+
   image_postgresql_arm64:
     needs: image_postgresql_amd64
     runs-on: ubuntu-latest
@@ -171,16 +228,73 @@ jobs:
           username: ${{ secrets._TEMP_DOCKERHUB_USER }}
           password: ${{ secrets._TEMP_DOCKERHUB_PASSWORD }}
 
-      - name: build and push arm64 image
+      - name: Build arm64 image locally
         run: |
           docker buildx build \
             --build-arg GIT_COMMIT=${{ github.sha }} \
-            --push \
-            --platform linux/aarch64 \
+            --load \
+            --platform linux/arm64 \
             --no-cache-filter trimmed \
             --no-cache-filter trimmed-all \
             $TAG .
 
+      - name: Install slim toolkit
+        run: |
+          curl -sL https://raw.githubusercontent.com/slimtoolkit/slim/master/scripts/install-slim.sh | sudo -E bash -
+
+      - name: Slim the image
+        run: |
+          # Extract image name from TAG (remove -t prefix)
+          IMAGE_NAME=$(echo "$TAG" | sed 's/-t //')
+          slim build --target $IMAGE_NAME \
+            --tag ${IMAGE_NAME}-slim \
+            --http-probe=false \
+            --continue-after=15 \
+            --expose=5432 \
+            --expose=8008 \
+            --expose=8081 \
+            --include-path=/usr/lib/postgresql \
+            --include-path=/usr/lib/aarch64-linux-gnu \
+            --include-path=/usr/share/postgresql \
+            --include-path=/usr/share/proj \
+            --include-path=/usr/share/gdal \
+            --include-path=/etc/alternatives \
+            --preserve-path=/var/lib/postgresql \
+            --preserve-path=/docker-entrypoint-initdb.d \
+            --preserve-path=/or-entrypoint.sh \
+            --preserve-path=/etc/postgresql \
+            --preserve-path=/etc/ssl \
+            --include-shell \
+            --include-bin=/usr/bin/sort \
+            --include-bin=/usr/bin/find \
+            --include-bin=/usr/bin/xargs \
+            --include-bin=/usr/bin/dirname \
+            --include-bin=/usr/bin/basename \
+            --include-bin=/usr/bin/head \
+            --include-bin=/usr/bin/tail \
+            --include-bin=/usr/bin/wc \
+            --include-bin=/usr/bin/cut \
+            --include-bin=/usr/bin/tr \
+            --include-bin=/usr/bin/sed \
+            --include-bin=/usr/bin/awk \
+            --include-bin=/usr/bin/grep \
+            --include-bin=/bin/cat \
+            --include-bin=/bin/mv \
+            --include-bin=/bin/mkdir \
+            --include-bin=/bin/chmod \
+            --include-bin=/bin/rm \
+            --include-bin=/bin/cp \
+            --include-bin=/bin/touch \
+            --include-bin=/usr/bin/id \
+            --include-bin=/usr/bin/env
+          # Replace original with slim version
+          docker tag ${IMAGE_NAME}-slim $IMAGE_NAME
+
+      - name: Push arm64 image
+        run: |
+          IMAGE_NAME=$(echo "$TAG" | sed 's/-t //')
+          docker push $IMAGE_NAME
+
   create_manifest:
     needs: [image_postgresql_amd64, image_postgresql_arm64]
     runs-on: ubuntu-latest

Dockerfile

@@ -1,51 +1,79 @@
-
 ARG PG_MAJOR=17
+ARG PREV_PG_MAJOR=15
 ARG TIMESCALE_VERSION=2.22
 
-FROM timescale/timescaledb-ha:pg17-ts${TIMESCALE_VERSION} AS trimmed
-LABEL maintainer="support@openremote.io"
+# Stage 1: Get PostgreSQL ${PREV_PG_MAJOR} binaries for upgrade support
+FROM timescale/timescaledb-ha:pg${PG_MAJOR}-ts${TIMESCALE_VERSION}-all AS pg-all
 
 USER root
 
-# install fd to find files to speed up chown and chgrp
-RUN apt-get update && apt-get install -y fd-find && rm -rf /var/lib/apt/lists/*
-
-# Give postgres user the same UID and GID as the old alpine postgres image to simplify migration of existing DB
-RUN usermod -u 70 postgres \
- && groupmod -g 70 postgres \
- && (fd / -group 1000 -exec chgrp -h postgres {} \; || true) \
- && (fd / -user 1000 -exec chown -h postgres {} \; || true)
+ARG PREV_PG_MAJOR
 
-# Set PGDATA to the same location as our old alpine image
-RUN mkdir -p /var/lib/postgresql && mv /home/postgres/pgdata/* /var/lib/postgresql/ && chown -R postgres:postgres /var/lib/postgresql
+# Strip debug symbols and remove unnecessary files from PG ${PREV_PG_MAJOR} in this stage
+# For pg_upgrade we only need bin/ and lib/, plus minimal share files (NOT extensions)
+RUN find /usr/lib/postgresql/${PREV_PG_MAJOR} -type f -name '*.so*' -exec strip --strip-unneeded {} \; 2>/dev/null || true \
+    && find /usr/lib/postgresql/${PREV_PG_MAJOR} -type f -executable -exec strip --strip-unneeded {} \; 2>/dev/null || true \
+    && rm -rf /usr/share/postgresql/${PREV_PG_MAJOR}/extension \
+              /usr/share/postgresql/${PREV_PG_MAJOR}/man \
+              /usr/share/postgresql/${PREV_PG_MAJOR}/doc \
+              /usr/share/postgresql/${PREV_PG_MAJOR}/contrib
 
-# Add custom entry point (see file header for details)
-COPY or-entrypoint.sh /
-RUN chmod +x /or-entrypoint.sh
+# Stage 2: Prepare the main image with UID/GID changes and cleanup
+FROM timescale/timescaledb-ha:pg${PG_MAJOR}-ts${TIMESCALE_VERSION} AS final
+LABEL maintainer="support@openremote.io"
 
-# Add custom initdb script(s)
-COPY docker-entrypoint-initdb.d/ /docker-entrypoint-initdb.d/
-RUN chmod +x /docker-entrypoint-initdb.d/*
+USER root
 
+ARG PREV_PG_MAJOR
 
-# Below is mostly copied from https://github.com/timescale/timescaledb-docker-ha/blob/master/Dockerfile (with OR specific entrypoint,
-# workdir and OR env defaults)
+# Copy only PG ${PREV_PG_MAJOR} bin directories for pg_upgrade (lib is needed for binaries to work)
+COPY --from=pg-all /usr/lib/postgresql/${PREV_PG_MAJOR}/bin /usr/lib/postgresql/${PREV_PG_MAJOR}/bin
+COPY --from=pg-all /usr/lib/postgresql/${PREV_PG_MAJOR}/lib /usr/lib/postgresql/${PREV_PG_MAJOR}/lib
+# Copy minimal share files needed for pg_upgrade (excluding extensions which are ~500MB each)
+COPY --from=pg-all /usr/share/postgresql/${PREV_PG_MAJOR} /usr/share/postgresql/${PREV_PG_MAJOR}
 
-# Get the -all variant which contains multiple PostgreSQL versions
-# According to TimescaleDB docs: "timescale/timescaledb-ha images have the files necessary to run previous versions"
-FROM timescale/timescaledb-ha:pg17-ts${TIMESCALE_VERSION}-all AS trimmed-all
+# Copy entrypoint scripts
+COPY or-entrypoint.sh /
+COPY docker-entrypoint-initdb.d/ /docker-entrypoint-initdb.d/
 
-## Create a smaller Docker image from the builder image
-FROM scratch
-COPY --from=trimmed / /
+# Install fd-find, fix UID/GID, setup directories, strip binaries, and cleanup - all in one layer
+RUN apt-get update && apt-get install -y --no-install-recommends fd-find \
+    # Give postgres user the same UID and GID as the old alpine postgres image
+    && usermod -u 70 postgres \
+    && groupmod -g 70 postgres \
+    && (fdfind . / -group 1000 -exec chgrp -h postgres {} \; 2>/dev/null || true) \
+    && (fdfind . / -user 1000 -exec chown -h postgres {} \; 2>/dev/null || true) \
+    # Set PGDATA to the same location as our old alpine image
+    && mkdir -p /var/lib/postgresql \
+    && mv /home/postgres/pgdata/* /var/lib/postgresql/ \
+    && chown -R postgres:postgres /var/lib/postgresql \
+    # Make scripts executable
+    && chmod +x /or-entrypoint.sh /docker-entrypoint-initdb.d/* \
+    # Strip debug symbols from PostgreSQL binaries to reduce size
+    && find /usr/lib/postgresql -type f -name '*.so*' -exec strip --strip-unneeded {} \; 2>/dev/null || true \
+    && find /usr/lib/postgresql -type f -executable -exec strip --strip-unneeded {} \; 2>/dev/null || true \
+    # Remove fd-find and clean up
+    && apt-get purge -y fd-find \
+    && apt-get autoremove -y --purge \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+              /var/cache/apt/* \
+              /var/log/* \
+              /usr/share/doc/* \
+              /usr/share/man/* \
+              /usr/share/info/* \
+              /usr/share/lintian/* \
+              /usr/share/locale/* \
+              /tmp/* \
+              /var/tmp/* \
+              /root/.cache \
+              /home/postgres/.cache \
+              /usr/local/lib/pgai \
+              /usr/share/postgresql/*/man \
+              /usr/share/postgresql/*/doc
 
 ARG PG_MAJOR
-
-## Copy only PostgreSQL 14 and 15 for upgrade support
-COPY --from=trimmed-all /usr/lib/postgresql/14 /usr/lib/postgresql/14
-COPY --from=trimmed-all /usr/lib/postgresql/15 /usr/lib/postgresql/15
-COPY --from=trimmed-all /usr/share/postgresql/14 /usr/share/postgresql/14
-COPY --from=trimmed-all /usr/share/postgresql/15 /usr/share/postgresql/15
+ARG PREV_PG_MAJOR
 
 # Increment this to indicate that a re-index should be carried out on first startup with existing data; REINDEX can still be overidden
 # with OR_DISABLE_REINDEX=true
@@ -80,6 +108,7 @@ ENV PGROOT=/var/lib/postgresql \
     POSTGRES_USER=${POSTGRES_USER:-postgres} \
     POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} \
     PG_MAJOR=$PG_MAJOR \
+    PREV_PG_MAJOR=$PREV_PG_MAJOR \
     OR_REINDEX_COUNTER=${OR_REINDEX_COUNTER} \
     OR_DISABLE_REINDEX=${OR_DISABLE_REINDEX:-false} \
     POSTGRES_MAX_CONNECTIONS=${POSTGRES_MAX_CONNECTIONS:-50} \

README.md

@@ -1,33 +1,36 @@
 # Postgresql docker image
-[![build multirach postgresql Docker image and push to it dockerhub](https://github.com/openremote/postgresql/actions/workflows/postgresql.yml/badge.svg)](https://github.com/openremote/postgresql/actions/workflows/postgresql.yml)
+[![build multiarch postgresql Docker image and push to dockerhub](https://github.com/openremote/postgresql/actions/workflows/postgresql.yml/badge.svg)](https://github.com/openremote/postgresql/actions/workflows/postgresql.yml)
 
- POSTGIS and TimescaleDB (inc. toolkit for hyperfunctions) image built for aarch64 support using `timescaledev/timescaledb-ha` base image with:
+POSTGIS and TimescaleDB (inc. toolkit for hyperfunctions) image built for amd64 and arm64 using `timescale/timescaledb-ha` base image with:
 
 - OR specific ENV variables and a healthcheck added
 - Easy configuration of `max_connections` using `POSTGRES_MAX_CONNECTIONS` environment variable (set to `-1` to disable this setting)
 - PGDATA path set to match old Alpine image (for ease of DB migration)
 - POSTGRES user UID and GID changed to match old Alpine image (for ease of DB migration)
 - Auto upgrade of database with PG major version changes from previous PG major version; can be disabled using
-  OR_DISABLE_AUTO_UPGRADE=true.
+  `OR_DISABLE_AUTO_UPGRADE=true`
 - Auto upgrade of timescaleDB extension when a new version is available in the container; can be disabled using
-  OR_DISABLE_AUTO_UPGRADE=true.
+  `OR_DISABLE_AUTO_UPGRADE=true`
 - OR_DISABLE_REINDEX env variable with associated scripts to determine if a REINDEX of the entire DB should be carried
-  out at first startup with existing DB (checks whether or not $PGDATA/OR_REINDEX_COUNTER.$OR_REINDEX_COUNTER exists).
+  out at first startup with existing DB (checks whether or not `$PGDATA/OR_REINDEX_COUNTER.$OR_REINDEX_COUNTER` exists).
   This is used when a collation change has occurred (glibc version change, muslc <-> glibc) which can break the indexes;
   migration can either be manually handled or auto handled depending on OR_DISABLE_REINDEX env variable value.
-  NOTE THAT A REINDEX CAN TAKE A LONG TIME DEPENDING ON THE SIZE OF THE DB! And startup will be delayed until completed
+  NOTE THAT A REINDEX CAN TAKE A LONG TIME DEPENDING ON THE SIZE OF THE DB! And startup will be delayed until completed.
   This functionality is intended to simplify migration for basic users; advanced users with large DBs should take care of this
   themselves.
+- **Slimmed images** using [slim toolkit](https://github.com/slimtoolkit/slim) to reduce image size by ~60%
 
-`timescale/timescaledb-ha` image is ubuntu based and only currently supports amd64; they are working on ARM64 support in timescaledev/timescaledb-ha see:
+## Local Development
 
-https://github.com/timescale/timescaledb-docker-ha/pull/355
+To build and slim the image locally:
 
-See this issue for POSTGIS base image aarch64 support discussion:
+```bash
+./build_and_slim.sh
+```
 
-https://github.com/postgis/docker-postgis/issues/216
-
-TODO: Switch over to timescale/timescaledb-ha once arm64 supported
+This will:
+1. Build the regular Docker image
+2. Use slim toolkit to create an optimized version with reduced size
 
 ## Upgrading
 ***NOTE: If you change the version of container you use then make sure you have backed up your DB first as this container will try to auto upgrade your DB and/or TimescaleDB extension; this auto upgrade functionality can be disabled using `OR_DISABLE_AUTO_UPGRADE=true`***

build_and_slim.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# PostgreSQL version tag
+PG_MAJOR=17
+IMAGE_NAME="openremote/postgresql:pg${PG_MAJOR}"
+SLIM_IMAGE_NAME="openremote/postgresql:pg${PG_MAJOR}-slim"
+
+# Build the regular Docker image first
+echo "Building regular Docker image..."
+docker build -t $IMAGE_NAME .
+
+# Use slimtoolkit to create an optimized version
+echo "Creating optimized image with slimtoolkit..."
+slim build --target $IMAGE_NAME \
+    --tag $SLIM_IMAGE_NAME \
+    --http-probe=false \
+    --continue-after=10 \
+    --expose=5432 \
+    --expose=8008 \
+    --expose=8081 \
+    --include-path=/usr/lib/postgresql \
+    --include-path=/usr/lib/aarch64-linux-gnu \
+    --include-path=/usr/share/postgresql \
+    --include-path=/usr/share/proj \
+    --include-path=/usr/share/gdal \
+    --include-path=/etc/alternatives \
+    --preserve-path=/var/lib/postgresql \
+    --preserve-path=/docker-entrypoint-initdb.d \
+    --preserve-path=/or-entrypoint.sh \
+    --preserve-path=/etc/postgresql \
+    --preserve-path=/etc/ssl \
+    --include-shell \
+    --include-bin=/usr/bin/sort \
+    --include-bin=/usr/bin/find \
+    --include-bin=/usr/bin/xargs \
+    --include-bin=/usr/bin/dirname \
+    --include-bin=/usr/bin/basename \
+    --include-bin=/usr/bin/head \
+    --include-bin=/usr/bin/tail \
+    --include-bin=/usr/bin/wc \
+    --include-bin=/usr/bin/cut \
+    --include-bin=/usr/bin/tr \
+    --include-bin=/usr/bin/sed \
+    --include-bin=/usr/bin/awk \
+    --include-bin=/usr/bin/grep \
+    --include-bin=/bin/cat \
+    --include-bin=/bin/mv \
+    --include-bin=/bin/mkdir \
+    --include-bin=/bin/chmod \
+    --include-bin=/bin/rm \
+    --include-bin=/bin/cp \
+    --include-bin=/bin/touch \
+    --include-bin=/usr/bin/id \
+    --include-bin=/usr/bin/env \
+    --show-clogs

or-entrypoint.sh

@@ -77,6 +77,20 @@ if [ -n "$DATABASE_ALREADY_EXISTS" ]; then
       echo "---------------------------------------------------------------------------------"
     fi
 
+    # Check if the old DB version is supported for upgrade
+    if [ "$DB_VERSION" != "$PG_MAJOR" ] && [ "$OR_DISABLE_AUTO_UPGRADE" != "true" ]; then
+      # Only PREV_PG_MAJOR and PG_MAJOR are supported
+      if [ "$DB_VERSION" != "$PREV_PG_MAJOR" ] && [ "$DB_VERSION" != "$PG_MAJOR" ]; then
+        echo "********************************************************************************"
+        echo "ERROR: Database version ${DB_VERSION} is not supported for automatic upgrade!"
+        echo "This image only supports upgrading from PostgreSQL ${PREV_PG_MAJOR} to ${PG_MAJOR}."
+        echo "To upgrade from ${DB_VERSION}, you need to use an intermediate image version"
+        echo "that supports upgrading from ${DB_VERSION} first."
+        echo "********************************************************************************"
+        exit 12
+      fi
+    fi
+
     # STEP 1: Upgrade TimescaleDB on OLD PostgreSQL version (if needed)
     # This must happen BEFORE pg_upgrade so both old and new PG have the same TS version
     if [ "$DB_VERSION" != "$PG_MAJOR" ] && [ "$OR_DISABLE_AUTO_UPGRADE" != "true" ]; then