feat: re-use source artifacts between ppa pushes, introduce docker images for faster CI builds

.github/workflows/bench.yml

@@ -12,30 +12,28 @@ concurrency:
   group: bench-${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   bench:
     runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:
+        # Keep these in sync with CI and docker images workflow configuration.
         nginx_version:
           - "1.26.3"
           - "1.28.3"
           - "1.29.8"
-
     env:
       NGINX_VERSION: ${{ matrix.nginx_version }}
-
     steps:
       - uses: actions/checkout@v6
 
-      - name: Show Docker versions
-        run: |
-          docker --version
-          docker compose version
-
-      - name: Build docker image
-        run: docker compose build
+      - name: Pull docker images
+        continue-on-error: true
+        run: docker compose pull dev 
 
       - name: Run benchmark suite
         run: docker compose run --rm dev make bench-quick
@@ -49,4 +47,4 @@ jobs:
             bench/results/latest/summary.txt
             bench/results/latest/*.json
             bench/results/latest/*.log
-          if-no-files-found: error
+          if-no-files-found: error

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: CI
+name: Test
 
 on:
   push:
@@ -23,24 +23,19 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        # Keep these in sync with bench and docker images workflow configuration.
         nginx_version:
           - "1.26.3"
           - "1.28.3"
           - "1.29.8"
-
     env:
       NGINX_VERSION: ${{ matrix.nginx_version }}
-
     steps:
       - uses: actions/checkout@v6
 
-      - name: Show Docker versions
-        run: |
-          docker --version
-          docker compose version
-
-      - name: Build docker image
-        run: docker compose build dev
+      - name: Pull docker images
+        continue-on-error: true
+        run: docker compose pull dev
 
       - name: Lint code
         run: |
@@ -52,12 +47,12 @@ jobs:
 
   debian-package:
     runs-on: ubuntu-24.04
-
     steps:
       - uses: actions/checkout@v6
 
-      - name: Build packaging image
-        run: docker compose build packaging
+      - name: Pull docker images
+        continue-on-error: true
+        run: docker compose pull packaging
 
       - name: Build Debian package and run smoke test
         run: docker compose run --rm packaging make debian-package-smoke

.github/workflows/publish-docker-images.yml

@@ -0,0 +1,94 @@
+name: Publish Docker Images
+
+on:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+concurrency:
+  group: container-images-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  packaging:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Show Docker versions
+        run: |
+          docker --version
+          docker compose version
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push packaging image
+        uses: docker/bake-action@v7
+        with:
+          files: docker-compose.yml
+          targets: packaging
+          push: true
+          set: |
+            packaging.platform=linux/amd64,linux/arm64
+            packaging.cache-from=type=gha,scope=packaging
+            packaging.cache-to=type=gha,mode=max,scope=packaging
+
+  dev:
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        # Keep these in sync with CI and bench workflow configuration.
+        nginx_version:
+          - "1.26.3"
+          - "1.28.3"
+          - "1.29.8"
+    env:
+      NGINX_VERSION: ${{ matrix.nginx_version }}
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Show Docker versions
+        run: |
+          docker --version
+          docker compose version
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push dev image
+        uses: docker/bake-action@v7
+        with:
+          files: docker-compose.yml
+          targets: dev
+          push: true
+          set: |
+            dev.platform=linux/amd64,linux/arm64
+            dev.cache-from=type=gha,scope=dev-${{ env.NGINX_VERSION }}
+            dev.cache-to=type=gha,mode=max,scope=dev-${{ env.NGINX_VERSION }}

.github/workflows/publish-launchpad-ppa.yml

@@ -17,8 +17,29 @@ permissions:
   contents: read
 
 jobs:
+  source-tarball:
+    runs-on: ubuntu-24.04
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Pull packaging image
+        continue-on-error: true
+        run: docker compose pull packaging
+
+      - name: Build shared source tarball
+        run: docker compose run --rm packaging make debian-orig-tarball
+
+      - name: Upload shared source tarball
+        uses: actions/upload-artifact@v7
+        with:
+          name: launchpad-orig-tarball
+          path: .pkg-build/libnginx-mod-http-cache-pilot_*.orig.tar.gz
+          if-no-files-found: error
+
   publish:
     runs-on: ubuntu-24.04
+    needs: source-tarball
     strategy:
       fail-fast: false
       matrix:
@@ -34,11 +55,15 @@ jobs:
     steps:
       - uses: actions/checkout@v6
 
-      - name: Show Docker versions
-        run: docker compose version
+      - name: Pull packaging image
+        continue-on-error: true
+        run: docker compose pull packaging
 
-      - name: Build packaging image
-        run: docker compose build packaging
+      - name: Download shared source tarball
+        uses: actions/download-artifact@v8
+        with:
+          name: launchpad-orig-tarball
+          path: .pkg-build
 
       - name: Build, sign, and upload source package
         env:

Dockerfile

@@ -1,7 +1,9 @@
 FROM debian:bookworm-slim
 
+LABEL org.opencontainers.image.source="https://github.com/wpelevator/ngx_cache_pilot"
+LABEL org.opencontainers.image.description="Debian development environment for ngx_cache_pilot"
+
 ARG NGINX_VERSION=1.25.5
-ARG TEST_NGINX_REPO=https://github.com/openresty/test-nginx.git
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV NGINX_VERSION=${NGINX_VERSION}
@@ -30,6 +32,9 @@ RUN mkdir -p /opt/nginx-src \
     && tar -xzf /tmp/nginx.tar.gz -C /opt/nginx-src \
     && rm /tmp/nginx.tar.gz
 
+RUN git clone --depth=1 --branch v0.32 https://github.com/openresty/test-nginx.git /opt/test-nginx \
+    && cpanm --notest /opt/test-nginx
+
 RUN cd "${NGINX_SRC_DIR}" \
     && ./configure \
         --prefix="${NGINX_BUILD_PREFIX}" \
@@ -40,9 +45,6 @@ RUN cd "${NGINX_SRC_DIR}" \
     && make -j"$(nproc)" \
     && make install
 
-RUN git clone --depth=1 "${TEST_NGINX_REPO}" /opt/test-nginx \
-    && cpanm --notest /opt/test-nginx
-
 WORKDIR /workspace
 
 CMD ["/bin/bash"]

Dockerfile.packaging

@@ -1,5 +1,8 @@
 FROM debian:bookworm-slim
 
+LABEL org.opencontainers.image.source="https://github.com/wpelevator/ngx_cache_pilot"
+LABEL org.opencontainers.image.description="Debian packaging environment for ngx_cache_pilot"
+
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update \

Makefile

@@ -10,7 +10,12 @@ DEBIAN_DISTRIBUTION ?=
 DEBIAN_VERSION_SUFFIX ?=
 LAUNCHPAD_PPA ?= ppa:wpelevator/packages
 
-.PHONY: help image shell packaging-shell nginx-build nginx-build-dynamic nginx-version format test bench bench-quick debian-package debian-package-smoke debian-package-clean debian-source-package debian-source-package-signed launchpad-ppa-upload
+DEBIAN_PACKAGE_VERSION = $(shell dpkg-parsechangelog -l "$(CURDIR)/debian/changelog" -SVersion)
+DEBIAN_UPSTREAM_VERSION = $(shell printf '%s\n' "$(DEBIAN_PACKAGE_VERSION)" | sed 's/-[^-]*$$//')
+DEBIAN_SOURCE_DIR = $(DEBIAN_BUILD_ROOT)/$(DEBIAN_SOURCE_PACKAGE)-$(DEBIAN_UPSTREAM_VERSION)
+DEBIAN_ORIG_TARBALL = $(DEBIAN_BUILD_ROOT)/$(DEBIAN_SOURCE_PACKAGE)_$(DEBIAN_UPSTREAM_VERSION).orig.tar.gz
+
+.PHONY: help image shell packaging-shell nginx-build nginx-build-dynamic nginx-version format test bench bench-quick debian-package debian-package-smoke debian-package-clean debian-source-tree debian-orig-tarball debian-source-package debian-source-package-signed launchpad-ppa-upload
 
 help:
 	@printf '%s\n' \
@@ -23,6 +28,7 @@ help:
 		'make test                Run the Test::Nginx suite' \
 		'make debian-package      Build Debian source and binary packages under .pkg-build/' \
 		'make debian-package-smoke Build Debian packages and run debian/tests/smoke' \
+		'make debian-orig-tarball Build Debian upstream orig tarball under .pkg-build/' \
 		'make debian-source-package Build unsigned Debian source package under .pkg-build/' \
 		'make debian-source-package-signed Build signed Debian source package under .pkg-build/' \
 		'make launchpad-ppa-upload Build, sign, and upload source package to Launchpad PPA' \
@@ -82,16 +88,9 @@ bench-quick: nginx-build
 debian-package-clean:
 	rm -rf "$(DEBIAN_BUILD_ROOT)"
 
-debian-source-package:
-	package_version="$$(dpkg-parsechangelog -l "$(CURDIR)/debian/changelog" -SVersion)"; \
-	source_version="$${package_version}$(DEBIAN_VERSION_SUFFIX)"; \
-	upstream_version="$$(printf '%s\n' "$$package_version" | sed 's/-[^-]*$$//')"; \
-	source_package="$(DEBIAN_SOURCE_PACKAGE)"; \
-	build_root="$(DEBIAN_BUILD_ROOT)"; \
-	source_dir="$$build_root/$${source_package}-$${upstream_version}"; \
-	orig_tarball="$$build_root/$${source_package}_$${upstream_version}.orig.tar.gz"; \
-	rm -rf "$$build_root"; \
-	mkdir -p "$$source_dir"; \
+debian-source-tree:
+	rm -rf "$(DEBIAN_SOURCE_DIR)"
+	mkdir -p "$(DEBIAN_SOURCE_DIR)"
 	tar \
 		--exclude-vcs \
 		--exclude=".pkg-build" \
@@ -107,27 +106,37 @@ debian-source-package:
 		--exclude="*.debian.tar.*" \
 		--exclude="*.dsc" \
 		--exclude="*.orig.tar.gz" \
-		-cf - . | tar -xf - -C "$$source_dir"; \
-	tar -C "$$build_root" -czf "$$orig_tarball" "$${source_package}-$${upstream_version}"; \
+		-cf - . | tar -xf - -C "$(DEBIAN_SOURCE_DIR)"
+
+debian-orig-tarball: debian-source-tree
+	source_date_epoch="$${SOURCE_DATE_EPOCH:-$$(dpkg-parsechangelog -l "$(CURDIR)/debian/changelog" -STimestamp)}"; \
+	tar -C "$(DEBIAN_BUILD_ROOT)" \
+		--sort=name \
+		--mtime="@$$source_date_epoch" \
+		--owner=0 \
+		--group=0 \
+		--numeric-owner \
+		-cf - "$(DEBIAN_SOURCE_PACKAGE)-$(DEBIAN_UPSTREAM_VERSION)" | gzip -n > "$(DEBIAN_ORIG_TARBALL)"
+
+debian-source-package: debian-source-tree
+	@test -f "$(DEBIAN_ORIG_TARBALL)" || $(MAKE) debian-orig-tarball
+	package_version="$$(dpkg-parsechangelog -l "$(CURDIR)/debian/changelog" -SVersion)"; \
+	source_version="$${package_version}$(DEBIAN_VERSION_SUFFIX)"; \
 	changelog_distribution="$$(dpkg-parsechangelog -l "$(CURDIR)/debian/changelog" -SDistribution)"; \
 	source_distribution="$${DEBIAN_DISTRIBUTION:-$$changelog_distribution}"; \
 	if [ "$$source_version" != "$$package_version" ] || [ "$$source_distribution" != "$$changelog_distribution" ]; then \
-		cd "$$source_dir" && \
+		cd "$(DEBIAN_SOURCE_DIR)" && \
 			DEBFULLNAME="$${DEBFULLNAME:-WPElevator Packaging Team}" \
 			DEBEMAIL="$${DEBEMAIL:-hi@wpelevator.com}" \
 			dch --newversion "$$source_version" \
 				--distribution "$$source_distribution" \
 				--force-distribution \
 				"Build for $$source_distribution."; \
 	fi; \
-	cd "$$source_dir" && dpkg-buildpackage -S -sa -us -uc
+	cd "$(DEBIAN_SOURCE_DIR)" && dpkg-buildpackage -S -sa -us -uc
 
 debian-package: debian-source-package
-	package_version="$$(dpkg-parsechangelog -l "$(CURDIR)/debian/changelog" -SVersion)"; \
-	upstream_version="$$(printf '%s\n' "$$package_version" | sed 's/-[^-]*$$//')"; \
-	source_package="$(DEBIAN_SOURCE_PACKAGE)"; \
-	source_dir="$(DEBIAN_BUILD_ROOT)/$${source_package}-$${upstream_version}"; \
-	cd "$$source_dir" && dpkg-buildpackage -b -us -uc
+	cd "$(DEBIAN_SOURCE_DIR)" && dpkg-buildpackage -b -us -uc
 
 debian-package-smoke: debian-package
 	apt-get update