Merge pull request #436 from NERSC/better-tagging #675
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Docker Bake Build and Push | |
| # This is mostly taken from | |
| # here: https://docs.docker.com/build/ci/github-actions/multi-platform/#with-bake | |
| on: | |
| push: | |
| branches: [ main ] | |
| tags: [ 'v*.*.*' ] | |
| paths-ignore: | |
| - 'docs/**' | |
| - '*.md' | |
| - 'README.md' | |
| - 'LICENSE' | |
| pull_request: | |
| branches: [ main ] | |
| paths-ignore: | |
| - 'docs/**' | |
| - '*.md' | |
| - 'README.md' | |
| - 'LICENSE' | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| REGISTRY: ghcr.io/nersc/interactem | |
| jobs: | |
| prepare: | |
| # Only run for: pushes, workflow_dispatch, PRs from same repo, or Dependabot | |
| if: > | |
| github.event_name != 'pull_request' || | |
| github.event.pull_request.head.repo.full_name == github.repository | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| outputs: | |
| version: ${{ steps.meta.outputs.version }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| services: ${{ steps.services.outputs.services }} | |
| all-images: ${{ steps.services.outputs.all-images }} | |
| meta-json: ${{ steps.meta.outputs.json }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: false | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.REGISTRY }}/interactem | |
| # FIX 2: robust tagging strategy | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=pr | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{major}}.{{minor}} | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=sha,format=short | |
| - name: Define services | |
| id: services | |
| run: | | |
| # Extract services from the prod group in docker-bake.hcl | |
| SERVICES=$(docker buildx bake -f docker-bake.hcl prod --print | jq -cr '[.group.prod.targets[]]') | |
| echo "Services found: ${SERVICES}" | |
| echo "services=${SERVICES}" >> $GITHUB_OUTPUT | |
| # Create combined list including base image for merge-manifests | |
| ALL_IMAGES=$(echo ${SERVICES} | jq -c '. + ["interactem"]') | |
| echo "All images (including base): ${ALL_IMAGES}" | |
| echo "all-images=${ALL_IMAGES}" >> $GITHUB_OUTPUT | |
| working-directory: ./docker | |
| build-base: | |
| needs: prepare | |
| strategy: | |
| matrix: | |
| include: | |
| - platform: amd64 | |
| runner: ubuntu-24.04 | |
| docker-platform: linux/amd64 | |
| - platform: arm64 | |
| runner: ubuntu-24.04-arm | |
| docker-platform: linux/arm64 | |
| runs-on: ${{ matrix.runner }} | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: false | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GHCR_PAT }} | |
| - name: Build and push base image by digest | |
| id: bake | |
| uses: docker/bake-action@v6 | |
| with: | |
| source: . | |
| files: | | |
| ./docker/docker-bake.hcl | |
| ./docker/docker-bake-${{matrix.platform}}.hcl | |
| ${{ github.event_name != 'pull_request' && './docker/docker-bake-ci.hcl' || '' }} | |
| targets: base | |
| set: | | |
| ${{ github.event_name != 'pull_request' && '*.output=type=image,push-by-digest=true,name-canonical=true,push=true' || '' }} | |
| ${{ github.event_name == 'pull_request' && '*.cache-to=' || '' }} | |
| env: | |
| TAG: ${{ needs.prepare.outputs.version }} | |
| CACHE_PLATFORM: ${{ matrix.platform }} | |
| - name: Export digest | |
| if: github.event_name != 'pull_request' | |
| run: | | |
| mkdir -p /tmp/digests/interactem | |
| digest="${{ fromJSON(steps.bake.outputs.metadata).base['containerimage.digest'] }}" | |
| touch "/tmp/digests/interactem/${digest#sha256:}" | |
| - name: Upload digest | |
| if: github.event_name != 'pull_request' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: base-digests-${{ matrix.platform }} | |
| path: /tmp/digests | |
| if-no-files-found: error | |
| retention-days: 1 | |
| build-services: | |
| needs: [prepare, build-base] | |
| strategy: | |
| matrix: | |
| include: | |
| - platform: amd64 | |
| runner: ubuntu-24.04 | |
| - platform: arm64 | |
| runner: ubuntu-24.04-arm | |
| runs-on: ${{ matrix.runner }} | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: false | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GHCR_PAT }} | |
| - name: Build and push all services by digest | |
| id: bake | |
| uses: docker/bake-action@v6 | |
| with: | |
| source: . | |
| files: | | |
| ./docker/docker-bake.hcl | |
| ./docker/docker-bake-${{ matrix.platform }}.hcl | |
| ${{ github.event_name != 'pull_request' && './docker/docker-bake-ci.hcl' || '' }} | |
| targets: prod | |
| set: | | |
| ${{ github.event_name != 'pull_request' && '*.output=type=image,push-by-digest=true,name-canonical=true,push=true' || '' }} | |
| ${{ github.event_name == 'pull_request' && '*.cache-to=' || '' }} | |
| env: | |
| TAG: ${{ needs.prepare.outputs.version }} | |
| CACHE_PLATFORM: ${{ matrix.platform }} | |
| - name: Export digests | |
| if: github.event_name != 'pull_request' | |
| run: | | |
| mkdir -p /tmp/digests | |
| METADATA='${{ steps.bake.outputs.metadata }}' | |
| SERVICES='${{ needs.prepare.outputs.services }}' | |
| # Iterate through each service and extract its digest | |
| for service in $(echo "$SERVICES" | jq -r '.[]'); do | |
| digest=$(echo "$METADATA" | jq -r ".\"${service}\"[\"containerimage.digest\"]") | |
| if [ -n "$digest" ] && [ "$digest" != "null" ]; then | |
| mkdir -p "/tmp/digests/${service}" | |
| touch "/tmp/digests/${service}/${digest#sha256:}" | |
| echo "Exported digest for ${service}: ${digest}" | |
| else | |
| echo "Warning: No digest found for ${service}" | |
| fi | |
| done | |
| - name: Upload digests | |
| if: github.event_name != 'pull_request' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: service-digests-${{ matrix.platform }} | |
| path: /tmp/digests | |
| if-no-files-found: error | |
| retention-days: 1 | |
| merge-manifests: | |
| if: github.event_name != 'pull_request' | |
| needs: [prepare, build-base, build-services] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| strategy: | |
| matrix: | |
| service: ${{ fromJson(needs.prepare.outputs.all-images) }} | |
| steps: | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GHCR_PAT }} | |
| - name: Download base digests | |
| if: matrix.service == 'interactem' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: /tmp/digests | |
| pattern: base-digests-* | |
| merge-multiple: true | |
| - name: Download service digests | |
| if: matrix.service != 'interactem' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: /tmp/digests | |
| pattern: service-digests-* | |
| merge-multiple: true | |
| - name: Create manifest list and push | |
| run: | | |
| SERVICE="${{ matrix.service }}" | |
| DIGEST_DIR="/tmp/digests/${SERVICE}" | |
| # 1. Gather all the digest arguments (platforms) | |
| DIGEST_ARGS="" | |
| for digest_file in "${DIGEST_DIR}"/*; do | |
| # Guard against empty directory | |
| [ -e "$digest_file" ] || continue | |
| sha=$(basename "${digest_file}") | |
| DIGEST_ARGS="${DIGEST_ARGS} ${{ env.REGISTRY }}/${SERVICE}@sha256:${sha}" | |
| done | |
| if [ -z "$DIGEST_ARGS" ]; then | |
| echo "::error::No digests found for service ${SERVICE}!" | |
| exit 1 | |
| fi | |
| # 2. Extract clean tag names from the JSON output using jq | |
| # Input: ["ghcr.io/owner/repo:latest", "ghcr.io/owner/repo:sha-123"] | |
| # Output: "latest sha-123" | |
| TAGS=$(echo '${{ needs.prepare.outputs.meta-json }}' | jq -r '.tags[] | split(":") | last') | |
| # 3. Build the tag arguments for the current service | |
| TAG_ARGS="" | |
| echo "Preparing tags for ${SERVICE}:" | |
| for tag in $TAGS; do | |
| TARGET_TAG="${{ env.REGISTRY }}/${SERVICE}:${tag}" | |
| echo " -> ${TARGET_TAG}" | |
| TAG_ARGS="${TAG_ARGS} -t ${TARGET_TAG}" | |
| done | |
| # 4. Create and Push | |
| # Print command for debugging transparency | |
| echo "Running: docker buildx imagetools create ${TAG_ARGS} ${DIGEST_ARGS}" | |
| docker buildx imagetools create ${TAG_ARGS} ${DIGEST_ARGS} | |
| - name: Inspect image | |
| # Inspect the SHA version specifically to verify the unique build exists | |
| run: | | |
| # Get the SHA tag specifically for verification | |
| SHA_TAG=$(echo '${{ needs.prepare.outputs.meta-json }}' | jq -r '.tags[] | select(contains("sha-")) | split(":") | last' | head -n 1) | |
| docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ matrix.service }}:${SHA_TAG} |