Build, Test, and Deploy #6671
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build, Test, and Deploy | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| types: | |
| - opened | |
| - synchronize | |
| workflow_run: | |
| workflows: ["Fork PR Gate"] | |
| types: [completed] | |
| merge_group: | |
| workflow_dispatch: | |
| inputs: | |
| skip-stage-deploys: | |
| description: 'Skip stage deployments' | |
| required: false | |
| default: 'false' | |
| type: choice | |
| options: | |
| - 'false' | |
| - 'true' | |
| trigger-sql-generation: | |
| description: 'Force SQL generation to run' | |
| required: false | |
| default: 'false' | |
| type: choice | |
| options: | |
| - 'false' | |
| - 'true' | |
| # Drop all default GITHUB_TOKEN permissions; each job declares its own least-privilege set. | |
| permissions: {} | |
| env: | |
| PYTHON_VERSION: '3.11' | |
| PRIVATE_REPO: mozilla/private-bigquery-etl | |
| jobs: | |
| decide-runs: | |
| name: Decide what checks to run based on changed files | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| pull-requests: read # for resolve-pr-base on workflow_run events | |
| # The workflow_run path is only for fork PRs (post gate-approval). For | |
| # same-repo PRs the native pull_request path already runs everything, so | |
| # firing build.yml again from workflow_run just duplicates work and | |
| # leaves an orphan "Build" rolled-up check sitting in_progress. | |
| # workflow_run.pull_requests[0] is populated for same-repo PRs and empty | |
| # for fork PRs — the same heuristic resolve-pr-base relies on below — | |
| # so the null check filters out the non-fork case. | |
| if: &is-allowed-event >- | |
| ${{ | |
| (github.event_name == 'workflow_run' | |
| && github.event.workflow_run.conclusion == 'success' | |
| && github.event.workflow_run.pull_requests[0] == null) | |
| || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork != true) | |
| || github.event_name == 'push' | |
| || github.event_name == 'merge_group' | |
| || github.event_name == 'workflow_dispatch' | |
| }} | |
| outputs: | |
| validate-routines: ${{ steps.set-flags.outputs.validate-routines }} | |
| deploy: ${{ steps.set-flags.outputs.deploy }} | |
| validate-sql: ${{ steps.set-flags.outputs.validate-sql }} | |
| validate-bqetl: ${{ steps.set-flags.outputs.validate-bqetl }} | |
| steps: | |
| - &checkout-with-history | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| filter: blob:none | |
| fetch-depth: 0 | |
| ref: ${{ github.event.workflow_run.head_sha || github.event.pull_request.head.sha || github.sha }} | |
| # Fork code only runs here via workflow_run, after the per-SHA | |
| # external-fork approval in fork-gate.yml — that gate is the control, | |
| # so opt past checkout v7's pull_request_target protection. | |
| allow-unsafe-pr-checkout: true | |
| # For workflow_run (fork PRs post-gate), the PR event payload isn't | |
| # available. Resolve the PR's base SHA via the API so changed-files can | |
| # still compute a real diff instead of conservatively running everything. | |
| # workflow_run.pull_requests[0] is reliable for same-repo PRs but often | |
| # empty for fork PRs, so we fall back to looking up by head SHA. | |
| - id: resolve-pr-base | |
| if: github.event_name == 'workflow_run' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| HEAD_SHA: ${{ github.event.workflow_run.head_sha }} | |
| PR_FROM_EVENT: ${{ github.event.workflow_run.pull_requests[0].number }} | |
| run: | | |
| set -euo pipefail | |
| pr_number="$PR_FROM_EVENT" | |
| if [ -z "$pr_number" ] || [ "$pr_number" = "null" ]; then | |
| pr_number=$(gh api \ | |
| "/repos/${{ github.repository }}/commits/$HEAD_SHA/pulls" \ | |
| --jq '.[0].number') | |
| fi | |
| if [ -z "$pr_number" ] || [ "$pr_number" = "null" ]; then | |
| echo "Could not resolve PR for head $HEAD_SHA; falling back to full validation." | |
| echo "base_sha=" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| base_sha=$(gh api "/repos/${{ github.repository }}/pulls/$pr_number" --jq '.base.sha') | |
| echo "Resolved PR #$pr_number, base $base_sha" | |
| echo "base_sha=$base_sha" >> "$GITHUB_OUTPUT" | |
| - id: changed-files-routines | |
| # Skip if we couldn't resolve a base SHA for workflow_run (run everything). | |
| if: github.event_name != 'workflow_run' || steps.resolve-pr-base.outputs.base_sha != '' | |
| uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 | |
| with: | |
| files: | | |
| requirements.txt | |
| bigquery_etl/routine/** | |
| .github/workflows/** | |
| sql/mozfun/** | |
| sql/moz-fx-data-shared-prod/udf/** | |
| sql/moz-fx-data-shared-prod/udf_js/** | |
| bqetl_project.yaml | |
| base_sha: ${{ steps.resolve-pr-base.outputs.base_sha || github.event.pull_request.base.sha || github.event.before }} | |
| - id: changed-files-sql | |
| if: github.event_name != 'workflow_run' || steps.resolve-pr-base.outputs.base_sha != '' | |
| uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 | |
| with: | |
| files: | | |
| requirements.txt | |
| sql/** | |
| sql_generators/** | |
| dags.yaml | |
| .github/workflows/** | |
| bigquery_etl/query_scheduling/** | |
| tests/sql/** | |
| bqetl_project.yaml | |
| base_sha: ${{ steps.resolve-pr-base.outputs.base_sha || github.event.pull_request.base.sha || github.event.before }} | |
| - id: changed-files-bqetl | |
| if: github.event_name != 'workflow_run' || steps.resolve-pr-base.outputs.base_sha != '' | |
| uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 | |
| with: | |
| files: | | |
| bqetl_project.yaml | |
| bigquery_etl/** | |
| tests/** | |
| script/bqetl | |
| script/entrypoint | |
| requirements.txt | |
| requirements.in | |
| .github/workflows/** | |
| base_sha: ${{ steps.resolve-pr-base.outputs.base_sha || github.event.pull_request.base.sha || github.event.before }} | |
| - name: Decide which jobs to run | |
| id: set-flags | |
| env: | |
| VALIDATE_ROUTINES: ${{ steps.changed-files-routines.outputs.any_changed }} | |
| VALIDATE_SQL: ${{ steps.changed-files-sql.outputs.any_changed }} | |
| VALIDATE_BQETL: ${{ steps.changed-files-bqetl.outputs.any_changed }} | |
| EVENT_NAME: ${{ github.event_name }} | |
| PR_BASE_RESOLVED: ${{ steps.resolve-pr-base.outputs.base_sha }} | |
| run: | | |
| # For workflow_run when PR base SHA couldn't be resolved, fall back to | |
| # running everything (conservative). Otherwise the changed-files | |
| # outputs are valid for both event paths. | |
| if [[ "$EVENT_NAME" == "workflow_run" && -z "$PR_BASE_RESOLVED" ]]; then | |
| echo "validate-routines=true" >> $GITHUB_OUTPUT | |
| echo "validate-sql=true" >> $GITHUB_OUTPUT | |
| echo "validate-bqetl=true" >> $GITHUB_OUTPUT | |
| echo "deploy=false" >> $GITHUB_OUTPUT | |
| exit 0 | |
| fi | |
| echo "validate-routines=$VALIDATE_ROUTINES" >> $GITHUB_OUTPUT | |
| echo "validate-sql=$VALIDATE_SQL" >> $GITHUB_OUTPUT | |
| echo "validate-bqetl=$VALIDATE_BQETL" >> $GITHUB_OUTPUT | |
| if [[ "$EVENT_NAME" == "workflow_run" ]]; then | |
| echo "deploy=false" >> $GITHUB_OUTPUT | |
| elif [[ "$GITHUB_REF" == "refs/heads/main" ]]; then | |
| echo "deploy=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "deploy=false" >> $GITHUB_OUTPUT | |
| fi | |
| build: | |
| name: Build Environment | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| environment: &build-env | |
| name: dev | |
| deployment: false | |
| if: *is-allowed-event | |
| steps: | |
| - &checkout | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| filter: blob:none | |
| ref: ${{ github.event.workflow_run.head_sha || github.event.pull_request.head.sha || github.sha }} | |
| # Fork ref gated by the external-fork approval (see checkout-with-history). | |
| allow-unsafe-pr-checkout: true | |
| - &setup-python | |
| name: Set up Python | |
| id: setup_python | |
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| cache-dependency-path: 'requirements.txt' | |
| - &restore-venv | |
| name: Restore cached virtualenv | |
| id: restore-venv | |
| uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| key: venv-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements.txt') }} | |
| path: .venv | |
| - name: Build | |
| run: | | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| pip install --no-deps -r requirements.txt | |
| pip-sync --pip-args=--no-deps | |
| - name: Save cached virtualenv | |
| # Don't let forked PRs poison the shared cache. For `pull_request` | |
| # events `head.repo.fork != true` filters forks directly; for | |
| # `workflow_run` (gate-approved fork PRs) the PR payload is null so | |
| # the clause evaluates true and fork runs DO write to cache. Safety | |
| # rests on two invariants: | |
| # 1. The cache key namespaces by hashFiles('requirements.txt'), so | |
| # a fork that changes deps lands in a separate key. | |
| # 2. The build step is hermetic w.r.t. files outside requirements.txt: | |
| # `pip install --no-deps -r requirements.txt` + `pip-sync | |
| # --pip-args=--no-deps` doesn't consult setup.py / pyproject.toml / | |
| # repo-local pip.conf (there is no setup.py; pyproject.toml is only | |
| # read when building the local project, which we don't do; pip | |
| # ignores pip.conf in the working tree). | |
| # If the build step ever changes to invoke the local project (e.g. | |
| # `pip install .` / `-e .`) or to read other config files, expand the | |
| # cache key to hash those files too — otherwise a fork could quietly | |
| # publish a poisoned venv under main's cache key. | |
| if: steps.restore-venv.outputs.cache-hit != 'true' && github.event.pull_request.head.repo.fork != true | |
| uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| key: venv-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements.txt') }} | |
| path: .venv | |
| verify-requirements: | |
| name: Verify Requirements | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| environment: *build-env | |
| needs: [build] | |
| steps: | |
| - *checkout | |
| - *setup-python | |
| - *restore-venv | |
| - name: Verify that requirements.txt contains the right dependencies for this python version | |
| run: | | |
| PATH=".venv/bin:$PATH" pip-compile --allow-unsafe --generate-hashes --quiet | |
| git diff --exit-code -G '^ *[^# ]' -- requirements.txt | |
| verify-format-yaml: | |
| name: Verify YAML Format | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| environment: *build-env | |
| needs: [build] | |
| steps: | |
| - *checkout | |
| - *setup-python | |
| - *restore-venv | |
| - name: Yamllint Test | |
| run: PATH=".venv/bin:$PATH" yamllint -c .yamllint.yaml . | |
| verify-format-sql: | |
| name: Verify SQL Format | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| environment: *build-env | |
| needs: [build, decide-runs] | |
| if: needs.decide-runs.outputs.validate-sql == 'true' || needs.decide-runs.outputs.validate-routines == 'true' || needs.decide-runs.outputs.deploy == 'true' | |
| steps: | |
| - *checkout | |
| - *setup-python | |
| - *restore-venv | |
| - name: Verify that SQL is correctly formatted | |
| run: | | |
| PATH=".venv/bin:$PATH" script/bqetl format --check \ | |
| $(git ls-tree -d HEAD --name-only) | |
| test-bqetl: | |
| name: Test bqetl | |
| runs-on: ubuntu-latest | |
| needs: [build, decide-runs] | |
| if: | | |
| (needs.decide-runs.outputs.validate-bqetl == 'true' || needs.decide-runs.outputs.deploy == 'true') | |
| permissions: | |
| id-token: write | |
| contents: read | |
| environment: &job-env | |
| name: dev | |
| deployment: false | |
| steps: | |
| - *checkout | |
| - *setup-python | |
| - *restore-venv | |
| - &auth-gcp-dryrun | |
| name: Authenticate to GCP (OIDC) | |
| id: auth-dryrun | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 | |
| with: | |
| workload_identity_provider: ${{ vars.GCPV2_GITHUB_WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ secrets.GCP_DRYRUN_SERVICE_ACCOUNT_EMAIL }} | |
| token_format: 'id_token' | |
| id_token_audience: 'https://us-central1-moz-fx-data-shared-prod.cloudfunctions.net/dryrun' | |
| id_token_include_email: true | |
| - name: PyTest with linters | |
| env: | |
| GOOGLE_GHA_ID_TOKEN: ${{ steps.auth-dryrun.outputs.id_token }} | |
| run: | | |
| PATH=".venv/bin:$PATH" pytest --black --flake8 \ | |
| --isort --mypy-ignore-missing-imports --pydocstyle \ | |
| -m "not (routine or sql or integration)" \ | |
| -p no:bigquery_etl.pytest_plugin.routine \ | |
| -p no:bigquery_etl.pytest_plugin.sql \ | |
| -n 8 | |
| integration: | |
| name: Integration Tests | |
| runs-on: ubuntu-latest | |
| needs: [build, decide-runs] | |
| if: | | |
| (needs.decide-runs.outputs.validate-bqetl == 'true' || needs.decide-runs.outputs.deploy == 'true') | |
| permissions: | |
| id-token: write | |
| contents: read | |
| environment: *job-env | |
| steps: | |
| - *checkout | |
| - *setup-python | |
| - *restore-venv | |
| - &auth-gcp-stage | |
| name: Authenticate to GCP and Generate ID Token | |
| id: auth-stage | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 | |
| with: | |
| workload_identity_provider: ${{ vars.GCPV2_GITHUB_WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ secrets.GCP_STAGE_SERVICE_ACCOUNT_EMAIL }} | |
| - name: PyTest Integration Test | |
| env: | |
| GOOGLE_PROJECT_ID: ${{ steps.auth-stage.outputs.project_id }} | |
| run: | | |
| PATH=".venv/bin:$PATH" script/entrypoint -m 'integration' -n 8 \ | |
| -p no:bigquery_etl.pytest_plugin.routine \ | |
| -p no:bigquery_etl.pytest_plugin.sql | |
| generate-sql: | |
| name: Generate SQL | |
| runs-on: ubuntu-latest | |
| needs: [build, decide-runs] | |
| if: | | |
| (needs.decide-runs.outputs.validate-sql == 'true' || needs.decide-runs.outputs.validate-routines == 'true' || needs.decide-runs.outputs.deploy == 'true') | |
| permissions: | |
| id-token: write | |
| contents: read | |
| environment: *job-env | |
| steps: | |
| - &checkout-with-creds | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: true | |
| fetch-depth: 0 | |
| ref: ${{ github.event.workflow_run.head_sha || github.event.pull_request.head.sha || github.sha }} | |
| # Fork ref gated by the external-fork approval (see checkout-with-history). | |
| allow-unsafe-pr-checkout: true | |
| - *setup-python | |
| - *restore-venv | |
| - *auth-gcp-dryrun | |
| - name: Generate SQL content | |
| env: | |
| TRIGGER_SQL_GENERATION: ${{ github.event.inputs.trigger-sql-generation }} | |
| GOOGLE_GHA_ID_TOKEN: ${{ steps.auth-dryrun.outputs.id_token }} | |
| GOOGLE_PROJECT_ID: ${{ steps.auth-dryrun.outputs.project_id }} | |
| run: | | |
| # Fetch main branch for merge-base | |
| git fetch origin main --no-tags | |
| MAIN_MERGE_BASE=$(git merge-base origin/main HEAD || true) | |
| echo "merge base with main: $MAIN_MERGE_BASE" | |
| git clone -b generated-sql https://github.com/mozilla/bigquery-etl ~/remote-generated-sql | |
| cd ~/remote-generated-sql | |
| if [[ -n $MAIN_MERGE_BASE ]] && git show-ref --tags c-${MAIN_MERGE_BASE:0:9} --quiet; then | |
| echo "Tag c-${MAIN_MERGE_BASE:0:9} exists in generated sql" | |
| git checkout tags/c-${MAIN_MERGE_BASE:0:9} | |
| else | |
| export LATEST_TAG_GENERATED_SQL=`git describe --tags --abbrev=0` | |
| LATEST_TAG_GENERATED_SQL=${LATEST_TAG_GENERATED_SQL/c-/""} | |
| echo "Latest tag on generated-sql branch: c-$LATEST_TAG_GENERATED_SQL" | |
| fi | |
| # Get the state in which main was in when the generated-sql branch was created and pushed | |
| cd ~ | |
| git clone -b main https://github.com/mozilla/bigquery-etl ~/remote-bigquery-etl-main | |
| cd ~/remote-bigquery-etl-main | |
| export LATEST_COMMIT_ON_MAIN=`git rev-parse --short HEAD` | |
| git checkout $LATEST_TAG_GENERATED_SQL | |
| echo "Latest commit on main: $LATEST_COMMIT_ON_MAIN" | |
| if [[ -n $MAIN_MERGE_BASE ]]; then | |
| echo "Checking out $MAIN_MERGE_BASE for diff" | |
| git checkout $MAIN_MERGE_BASE | |
| fi | |
| cd $GITHUB_WORKSPACE | |
| export GENERATED_SQL_CHANGED=false | |
| if [ "$TRIGGER_SQL_GENERATION" = "true" ]; then | |
| echo "trigger-sql-generation was set to true; run SQL generation" | |
| GENERATED_SQL_CHANGED=true | |
| fi | |
| if [ "$GITHUB_REF_NAME" = main ]; then | |
| echo "On main branch; generate SQL" | |
| GENERATED_SQL_CHANGED=true | |
| fi | |
| if [[ $(git diff --no-index --name-only --diff-filter=DR ~/remote-bigquery-etl-main/sql sql) ]]; then | |
| echo "SQL file was removed; re-generate SQL" | |
| GENERATED_SQL_CHANGED=true | |
| fi | |
| if [[ $(diff -qr --no-dereference sql_generators ~/remote-bigquery-etl-main/sql_generators) ]]; then | |
| echo "SQL generators changed; re-generate SQL" | |
| GENERATED_SQL_CHANGED=true | |
| fi | |
| mkdir -p /tmp/workspace/generated-sql | |
| cp bqetl_project.yaml /tmp/workspace/generated-sql/ | |
| if [ "$GENERATED_SQL_CHANGED" = "true" ]; then | |
| echo "Changes made will affect generated SQL. Run SQL generators." | |
| cp -r sql/ /tmp/workspace/generated-sql/sql | |
| PATH=".venv/bin:$PATH" script/bqetl generate all \ | |
| --ignore derived_view_schemas \ | |
| --output-dir /tmp/workspace/generated-sql/sql/ \ | |
| --target-project moz-fx-data-shared-prod | |
| PATH=".venv/bin:$PATH" script/bqetl format /tmp/workspace/generated-sql/sql/ | |
| else | |
| echo "Changes made don't affect generated SQL. Use content from generated-sql" | |
| cp -r ~/remote-generated-sql/sql/ /tmp/workspace/generated-sql/sql | |
| cp -a sql/. /tmp/workspace/generated-sql/sql | |
| fi | |
| PATH=".venv/bin:$PATH" script/bqetl query schema render \ | |
| /tmp/workspace/generated-sql/sql/ \ | |
| --sql-dir=/tmp/workspace/generated-sql/sql/ \ | |
| --output-dir=/tmp/workspace/generated-sql/sql/ | |
| PATH=".venv/bin:$PATH" script/bqetl dependency record \ | |
| --skip-existing \ | |
| "/tmp/workspace/generated-sql/sql/" | |
| PATH=".venv/bin:$PATH" script/bqetl metadata update \ | |
| --sql-dir /tmp/workspace/generated-sql/sql/ \ | |
| /tmp/workspace/generated-sql/sql/ | |
| PATH=".venv/bin:$PATH" script/bqetl monitoring update /tmp/workspace/generated-sql/sql/ | |
| - name: Copy schema cache for reuse in other jobs | |
| run: | | |
| mkdir -p /tmp/workspace/schema-cache | |
| if [ -d /tmp/bigquery_etl_schemas ]; then | |
| cp -r /tmp/bigquery_etl_schemas /tmp/workspace/schema-cache/ | |
| fi | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: generated-sql | |
| path: /tmp/workspace/generated-sql | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: schema-cache | |
| path: /tmp/workspace/schema-cache | |
| trigger-private-ci: | |
| name: Trigger Private CI | |
| runs-on: ubuntu-latest | |
| needs: [generate-sql, decide-runs] | |
| if: | | |
| (github.event_name == 'push' || github.event_name == 'pull_request' || github.event_name == 'merge_group' || | |
| (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success')) && | |
| (needs.decide-runs.outputs.validate-sql == 'true' || needs.decide-runs.outputs.deploy == 'true') | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| outputs: | |
| pr-number: ${{ steps.context.outputs.pr_number }} | |
| commit-sha: ${{ steps.context.outputs.commit_sha }} | |
| environment: *job-env | |
| steps: | |
| - name: Determine context | |
| id: context | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| HEAD_SHA: ${{ github.event.workflow_run.head_sha || github.event.pull_request.head.sha || github.sha }} | |
| PR_FROM_WR: ${{ github.event.workflow_run.pull_requests[0].number }} | |
| EVENT_PR_NUMBER: ${{ github.event.pull_request.number }} | |
| REPO: ${{ github.repository }} | |
| run: | | |
| echo "commit_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT" | |
| if [[ "$GITHUB_EVENT_NAME" == "push" && "$GITHUB_REF" == "refs/heads/main" ]]; then | |
| echo "is_main_push=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "is_main_push=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| # Resolve PR number across all event types | |
| if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then | |
| echo "pr_number=${EVENT_PR_NUMBER}" >> "$GITHUB_OUTPUT" | |
| elif [[ "$GITHUB_EVENT_NAME" == "workflow_run" ]]; then | |
| # Fork PRs via workflow_run: resolve PR number from head SHA | |
| pr_number="$PR_FROM_WR" | |
| if [[ -z "$pr_number" || "$pr_number" == "null" ]]; then | |
| pr_number=$(gh api \ | |
| "/repos/${REPO}/commits/$HEAD_SHA/pulls" \ | |
| --jq '.[0].number' 2>/dev/null || echo "") | |
| fi | |
| echo "pr_number=${pr_number}" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "pr_number=" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Trigger private-bigquery-etl CI | |
| id: dispatch | |
| env: | |
| TRIGGER_TOKEN: ${{ secrets.PRIVATE_BQETL_TRIGGER_TOKEN }} | |
| COMMIT_SHA: ${{ steps.context.outputs.commit_sha }} | |
| PR_NUMBER: ${{ steps.context.outputs.pr_number }} | |
| RUN_ID: ${{ github.run_id }} | |
| IS_MAIN: ${{ steps.context.outputs.is_main_push }} | |
| run: | | |
| if [[ -z "$TRIGGER_TOKEN" ]]; then | |
| echo "::warning::No private repo token available (fork PR?), skipping dispatch" | |
| exit 0 | |
| fi | |
| HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST \ | |
| -H "Authorization: Bearer ${TRIGGER_TOKEN}" \ | |
| -H "Accept: application/vnd.github+json" \ | |
| "https://api.github.com/repos/${PRIVATE_REPO}/dispatches" \ | |
| -d "$(jq -n \ | |
| --arg sha "$COMMIT_SHA" \ | |
| --arg pr "$PR_NUMBER" \ | |
| --arg run_id "$RUN_ID" \ | |
| --argjson is_main "${IS_MAIN:-false}" \ | |
| '{event_type: "bqetl-ci", client_payload: {bqetl_commit_sha: $sha, bqetl_pr_number: $pr, bqetl_run_id: $run_id, is_main_push: $is_main}}')") | |
| if [[ "$HTTP_STATUS" != "204" ]]; then | |
| echo "::error::Failed to dispatch private CI (HTTP $HTTP_STATUS)" | |
| exit 1 | |
| fi | |
| echo "Dispatched bqetl-ci event to private-bigquery-etl" | |
| docs: | |
| name: Build and Deploy Documentation | |
| runs-on: ubuntu-latest | |
| needs: [generate-sql] | |
| if: &is-main-deploy | |
| github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') | |
| permissions: | |
| contents: write | |
| environment: &deploy-env | |
| name: dev | |
| deployment: true | |
| steps: | |
| - *checkout | |
| - *setup-python | |
| - *restore-venv | |
| - &download-generated-sql | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: generated-sql | |
| path: /tmp/workspace/generated-sql | |
| - ©-generated-sql | |
| name: Move generated SQL into place | |
| run: | | |
| rm -rf sql/ | |
| cp -r /tmp/workspace/generated-sql/sql sql/ | |
| - name: Generate documentation | |
| run: | | |
| PATH=".venv/bin:$PATH" script/bqetl docs generate --output_dir=generated_docs/ | |
| - name: Deploy documentation to GitHub Pages | |
| run: | | |
| cd generated_docs/ | |
| git config --global user.name "GitHub Actions docs job" | |
| git config --global user.email "dataops@mozilla.com" | |
| PATH="../.venv/bin:$PATH" mkdocs gh-deploy \ | |
| --remote-name https://x-access-token:${{ github.token }}@github.com/mozilla/bigquery-etl \ | |
| --force \ | |
| -m "[ci skip] Deployed {sha} with MkDocs version: {version}" | |
| deploy-changes-to-stage: | |
| name: Deploy Changes to Stage | |
| runs-on: ubuntu-latest | |
| needs: [generate-sql, decide-runs] | |
| if: | | |
| (needs.decide-runs.outputs.validate-sql == 'true' || needs.decide-runs.outputs.validate-routines == 'true' || needs.decide-runs.outputs.deploy == 'true') | |
| permissions: | |
| id-token: write | |
| contents: read | |
| environment: *job-env | |
| steps: | |
| - *checkout | |
| - *setup-python | |
| - *restore-venv | |
| - *download-generated-sql | |
| - &download-schema-cache | |
| name: Download schema cache if available | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| continue-on-error: true | |
| with: | |
| name: schema-cache | |
| path: /tmp/workspace/schema-cache | |
| - name: Restore schema cache from generate-sql job | |
| run: | | |
| if [ -d /tmp/workspace/schema-cache/bigquery_etl_schemas ]; then | |
| cp -r /tmp/workspace/schema-cache/bigquery_etl_schemas /tmp/ | |
| echo "Restored schema cache from generate-sql job" | |
| fi | |
| - name: Move generated-sql into place | |
| run: | | |
| rm -rf sql/ | |
| cp -r /tmp/workspace/generated-sql/sql sql | |
| - name: Pull in generated-sql branch from remote | |
| run: | | |
| git clone --single-branch --branch generated-sql \ | |
| https://github.com/mozilla/bigquery-etl \ | |
| generated-sql | |
| # Normalize file permissions to avoid false positives in diffs | |
| find generated-sql/sql -type f -exec chmod 644 {} \; 2>/dev/null || true | |
| find sql -type f -exec chmod 644 {} \; 2>/dev/null || true | |
| - *auth-gcp-dryrun | |
| - *auth-gcp-stage | |
| - &write-bqetl-targets | |
| name: Write bqetl_targets.yaml | |
| run: | | |
| cat > bqetl_targets.yaml <<'EOF' | |
| stage: | |
| project_id: moz-fx-data-integration-tests | |
| dataset: "{{ git.branch }}_{{ git.commit }}{% if run_id %}_{{ run_id }}{% endif %}" | |
| artifact_prefix: "{{ artifact.project_id }}__{{ artifact.dataset_id }}__" | |
| grant_dryrun_access: true | |
| expire_after_hours: 12 | |
| rewrite_tests: true | |
| EOF | |
| - name: Deploy changes to stage | |
| timeout-minutes: 120 | |
| env: | |
| SKIP_STAGE_DEPLOYS: ${{ github.event.inputs.skip-stage-deploys }} | |
| GOOGLE_GHA_ID_TOKEN: ${{ steps.auth-dryrun.outputs.id_token }} | |
| run: | | |
| if [ "$SKIP_STAGE_DEPLOYS" = "true" ]; then | |
| echo "SKIP_STAGE_DEPLOYS=true, skipping stage deploy" | |
| exit 0 | |
| fi | |
| PATHS="$(git diff --no-index --name-only --diff-filter=d generated-sql/sql sql)" || true | |
| if [ -z "$PATHS" ]; then | |
| echo "No changed SQL paths, skipping stage deploy" | |
| exit 0 | |
| fi | |
| echo $PATHS | |
| git diff --no-index --diff-filter=d generated-sql/sql sql | head -1000 || true | |
| PATH=".venv/bin:$PATH" script/bqetl \ | |
| --target stage \ | |
| --run-id "${GITHUB_RUN_ID}" \ | |
| deploy --tables --views --isolated \ | |
| $PATHS | |
| - name: Copy generated SQL to temporary stage directory | |
| run: | | |
| mkdir -p /tmp/workspace/staged-generated-sql | |
| cp -r sql/ /tmp/workspace/staged-generated-sql/sql | |
| cp -r tests/ /tmp/workspace/staged-generated-sql/tests | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: staged-generated-sql | |
| path: /tmp/workspace/staged-generated-sql | |
| dry-run-sql: | |
| name: Dry Run SQL | |
| runs-on: ubuntu-latest | |
| needs: [deploy-changes-to-stage, decide-runs] | |
| if: | | |
| (needs.decide-runs.outputs.validate-sql == 'true' || needs.decide-runs.outputs.validate-routines == 'true' || needs.decide-runs.outputs.deploy == 'true') | |
| permissions: | |
| id-token: write | |
| contents: read | |
| environment: *job-env | |
| steps: | |
| - *checkout | |
| - *setup-python | |
| - *restore-venv | |
| - &download-staged-sql | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: staged-generated-sql | |
| path: /tmp/workspace/staged-generated-sql | |
| - ©-staged-sql | |
| name: Move sql deployed on stage into place | |
| run: | | |
| rm -rf sql/ | |
| cp -r /tmp/workspace/staged-generated-sql/sql sql | |
| rm -rf tests/ | |
| cp -r /tmp/workspace/staged-generated-sql/tests tests | |
| - *auth-gcp-dryrun | |
| - name: Dry run queries | |
| env: | |
| SKIP_STAGE_DEPLOYS: ${{ github.event.inputs.skip-stage-deploys }} | |
| GOOGLE_GHA_ID_TOKEN: ${{ steps.auth-dryrun.outputs.id_token }} | |
| run: | | |
| if [ "$GITHUB_REF_NAME" = main ]; then | |
| echo "Skip dryruns on main. Dryruns will run in Airflow" | |
| exit 0 | |
| elif git log --format=%B --no-merges -n 1 | | |
| grep -qF '[run-tests]'; then | |
| echo "Check dry run for all queries because [run-tests] in" \ | |
| "commit message" | |
| PATHS=sql | |
| elif [ "$SKIP_STAGE_DEPLOYS" = "true" ]; then | |
| PATHS=sql | |
| elif [ -d "sql/moz-fx-data-integration-tests" ]; then | |
| PATHS="sql/moz-fx-data-integration-tests" | |
| else | |
| # If integration tests dir doesn't exist, skip dryrun | |
| echo "Integration tests directory not found, skipping dryrun" | |
| exit 0 | |
| fi | |
| echo $PATHS | |
| PATH=".venv/bin:$PATH" script/bqetl dryrun --validate-schemas $PATHS | |
| test-sql: | |
| name: Test SQL | |
| runs-on: ubuntu-latest | |
| needs: [deploy-changes-to-stage, decide-runs] | |
| if: | | |
| (needs.decide-runs.outputs.validate-sql == 'true' || needs.decide-runs.outputs.validate-routines == 'true' || needs.decide-runs.outputs.deploy == 'true') | |
| permissions: | |
| id-token: write | |
| contents: read | |
| environment: *job-env | |
| steps: | |
| - *checkout | |
| - *setup-python | |
| - *restore-venv | |
| - *download-staged-sql | |
| - *copy-staged-sql | |
| - *auth-gcp-dryrun | |
| - *auth-gcp-stage | |
| - name: Run SQL tests | |
| env: | |
| # unique id to use as dataset suffix | |
| CI_RUN_ID: ${{ github.run_id }}_${{ github.run_number }} | |
| GOOGLE_PROJECT_ID: ${{ steps.auth-stage.outputs.project_id }} | |
| GOOGLE_GHA_ID_TOKEN: ${{ steps.auth-dryrun.outputs.id_token }} | |
| run: | | |
| PATH=".venv/bin:$PATH" script/entrypoint -m sql -n 8 -p no:bigquery_etl.pytest_plugin.routine | |
| test-routines: | |
| name: Test Routines | |
| runs-on: ubuntu-latest | |
| needs: [deploy-changes-to-stage, decide-runs] | |
| if: | | |
| (needs.decide-runs.outputs.validate-routines == 'true' || needs.decide-runs.outputs.deploy == 'true') | |
| permissions: | |
| id-token: write | |
| contents: read | |
| environment: *job-env | |
| steps: | |
| - *checkout | |
| - *setup-python | |
| - *restore-venv | |
| - *download-staged-sql | |
| - *copy-staged-sql | |
| - *auth-gcp-stage | |
| - name: Run routine tests | |
| env: | |
| # unique id to use as dataset suffix | |
| CI_RUN_ID: ${{ github.run_id }}_${{ github.run_number }} | |
| GOOGLE_PROJECT_ID: ${{ steps.auth-stage.outputs.project_id }} | |
| run: | | |
| PATH=".venv/bin:$PATH" script/entrypoint -m routine -n 8 | |
| - name: Validate doc examples | |
| env: | |
| GOOGLE_PROJECT_ID: ${{ steps.auth-stage.outputs.project_id }} | |
| run: | | |
| PATH=".venv/bin:$PATH" script/bqetl routine validate --docs-only | |
| validate-views: | |
| name: Validate Views | |
| runs-on: ubuntu-latest | |
| needs: [deploy-changes-to-stage, decide-runs] | |
| if: | | |
| (needs.decide-runs.outputs.validate-sql == 'true' || needs.decide-runs.outputs.validate-routines == 'true' || needs.decide-runs.outputs.deploy == 'true') | |
| permissions: | |
| id-token: write | |
| contents: read | |
| environment: *job-env | |
| steps: | |
| - *checkout | |
| - *setup-python | |
| - *restore-venv | |
| - *download-staged-sql | |
| - *copy-staged-sql | |
| - *auth-gcp-dryrun | |
| - name: Validate views | |
| env: | |
| GOOGLE_GHA_ID_TOKEN: ${{ steps.auth-dryrun.outputs.id_token }} | |
| run: | | |
| PATH=".venv/bin:$PATH" script/bqetl view validate | |
| validate-metadata: | |
| name: Validate Metadata | |
| needs: [build, decide-runs] | |
| runs-on: ubuntu-latest | |
| if: | | |
| (needs.decide-runs.outputs.validate-sql == 'true' || needs.decide-runs.outputs.deploy == 'true') | |
| permissions: | |
| id-token: write | |
| contents: read | |
| environment: *job-env | |
| steps: | |
| - *checkout | |
| - *setup-python | |
| - *restore-venv | |
| - name: Validate workgroup access configuration on main | |
| run: | | |
| PATH=".venv/bin:$PATH" script/bqetl metadata validate-workgroups sql/ | |
| - *auth-gcp-dryrun | |
| - name: Verify that metadata files are valid | |
| env: | |
| GOOGLE_GHA_ID_TOKEN: ${{ steps.auth-dryrun.outputs.id_token }} | |
| run: | | |
| PATH=".venv/bin:$PATH" script/bqetl query validate \ | |
| --no-dryrun --skip-format-sql | |
| - name: Copy generated SQL for debugging | |
| if: failure() | |
| run: | | |
| mkdir -p /tmp/debug_artifacts | |
| cp -r sql /tmp/debug_artifacts/sql | |
| - name: Upload debug artifacts | |
| if: failure() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: validate-metadata-debug-artifacts | |
| path: /tmp/debug_artifacts | |
| validate-backfills: | |
| name: Validate Backfills | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| environment: *build-env | |
| needs: [generate-sql, decide-runs] | |
| if: needs.decide-runs.outputs.validate-sql == 'true' || needs.decide-runs.outputs.deploy == 'true' | |
| steps: | |
| - *checkout | |
| - *setup-python | |
| - *restore-venv | |
| - *download-generated-sql | |
| - *copy-generated-sql | |
| - *auth-gcp-dryrun | |
| - name: Verify that backfill.yaml files are valid | |
| env: | |
| GOOGLE_GHA_ID_TOKEN: ${{ steps.auth-dryrun.outputs.id_token }} | |
| run: | | |
| PATH=".venv/bin:$PATH" script/bqetl backfill validate --dry-run-custom-query | |
| push-generated-sql: | |
| name: Push Generated SQL | |
| runs-on: ubuntu-latest | |
| needs: [generate-sql] | |
| if: *is-main-deploy | |
| environment: *deploy-env | |
| steps: | |
| - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: generated-sql | |
| path: /tmp/workspace/generated-sql | |
| - name: Setup SSH key for generated-sql | |
| run: | | |
| mkdir -p ~/.ssh | |
| echo "${{ secrets.DEPLOY_KEY_GENERATED_SQL }}" > ~/.ssh/id_rsa | |
| chmod 600 ~/.ssh/id_rsa | |
| ssh-keyscan github.com >> ~/.ssh/known_hosts | |
| - name: Push to generated-sql branch | |
| run: | | |
| git config --global user.name "GitHub Actions generate-sql job" | |
| git config --global user.email "dataops+generated-sql@mozilla.com" | |
| git clone --single-branch --branch generated-sql \ | |
| git@github.com:mozilla/bigquery-etl.git \ | |
| generated-sql | |
| cd generated-sql/ | |
| rm -rf sql/ | |
| cp -r /tmp/workspace/generated-sql/sql sql | |
| git add . | |
| if git diff --cached --quiet; then | |
| echo "No changes to push" | |
| else | |
| git commit -m "Auto-push due to change on main branch [ci skip]" \ | |
| -m "Created from $GITHUB_SHA" | |
| if git rev-parse "c-$GITHUB_SHA" >/dev/null 2>&1; then | |
| echo "Tag c-$GITHUB_SHA already exists, skipping" | |
| else | |
| git tag "c-$GITHUB_SHA" | |
| git push origin "c-$GITHUB_SHA" | |
| fi | |
| git push | |
| fi | |
| reset-stage-env: | |
| name: Reset Stage Environment | |
| runs-on: ubuntu-latest | |
| needs: [test-sql, validate-views, validate-metadata, dry-run-sql, test-routines, decide-runs] | |
| if: | | |
| (needs.decide-runs.outputs.validate-sql == 'true' || needs.decide-runs.outputs.validate-routines == 'true' || needs.decide-runs.outputs.deploy == 'true') | |
| permissions: | |
| id-token: write | |
| contents: read | |
| environment: *job-env | |
| steps: | |
| - *checkout | |
| - *setup-python | |
| - *restore-venv | |
| - *auth-gcp-stage | |
| - *write-bqetl-targets | |
| - name: Delete stage datasets | |
| env: | |
| GOOGLE_PROJECT_ID: ${{ steps.auth-stage.outputs.project_id }} | |
| run: | | |
| PATH=".venv/bin:$PATH" script/bqetl \ | |
| --target stage \ | |
| --run-id "${GITHUB_RUN_ID}" \ | |
| target clean --yes | |
| deploy-to-pypi: | |
| name: Deploy to PyPI | |
| runs-on: ubuntu-latest | |
| if: startsWith(github.ref, 'refs/tags/') | |
| permissions: | |
| contents: read | |
| id-token: write | |
| environment: | |
| name: pypi | |
| url: https://pypi.org/p/mozilla-bigquery-etl | |
| deployment: true | |
| steps: | |
| - *checkout-with-history | |
| - name: Verify tag is on main branch | |
| run: | | |
| git fetch origin main | |
| if ! git merge-base --is-ancestor $GITHUB_SHA origin/main; then | |
| echo "Error: Tag is not on main branch. Only tags on main can be deployed to PyPI." | |
| exit 1 | |
| fi | |
| echo "Verified: Tag is on main branch" | |
| - name: Set up Python | |
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install deployment tools | |
| run: pip install --upgrade build setuptools wheel twine | |
| - name: Build distribution files | |
| run: python -m build --sdist --wheel | |
| - name: Publish distribution 📦 to PyPI | |
| uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 | |
| # Post a single "Build" rolled-up check in `in_progress` state at the very | |
| # start of a workflow_run, so the PR Checks UI shows live progress instead | |
| # of "skipped" until the per-job checks are reported at the end. The id is | |
| # passed via job output to `report-checks` which PATCHes it to completed. | |
| start-build-check: | |
| name: Start build check | |
| # Same fork-only filter as is-allowed-event — see comment there. | |
| if: >- | |
| github.event_name == 'workflow_run' | |
| && github.event.workflow_run.conclusion == 'success' | |
| && github.event.workflow_run.pull_requests[0] == null | |
| runs-on: ubuntu-latest | |
| permissions: | |
| checks: write | |
| outputs: | |
| check_id: ${{ steps.create.outputs.check_id }} | |
| steps: | |
| - id: create | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| HEAD_SHA: ${{ github.event.workflow_run.head_sha }} | |
| DETAILS_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| set -euo pipefail | |
| check_id=$(gh api -X POST "/repos/${{ github.repository }}/check-runs" \ | |
| -f name='Build' \ | |
| -f head_sha="$HEAD_SHA" \ | |
| -f status=in_progress \ | |
| -f details_url="$DETAILS_URL" \ | |
| --jq .id) | |
| # Fail loudly if the API call returned nothing; otherwise downstream | |
| # report-checks would PATCH /check-runs/<empty> and 404, leaving an | |
| # orphan in_progress check on the PR forever. | |
| if [[ -z "$check_id" || "$check_id" == "null" ]]; then | |
| echo "::error::Failed to create rolled-up Build check (empty id)" | |
| exit 1 | |
| fi | |
| echo "check_id=$check_id" >> "$GITHUB_OUTPUT" | |
| # Surface upstream job results as Checks-API check runs on the PR's head SHA. | |
| # Only runs when build.yml is triggered via `workflow_run` (i.e. after the | |
| # fork-gate approval). For same-repo PRs the native pull_request checks | |
| # already appear in the PR UI, so this job is unnecessary there. | |
| # | |
| # `needs:` enforces ordering (run after all upstream jobs complete). The | |
| # actual list of jobs to report is derived dynamically from the Actions Jobs | |
| # API at runtime, so new jobs added to this workflow automatically appear in | |
| # the fork-PR Checks UI without needing edits here. | |
| report-checks: | |
| name: Report PR checks | |
| # !cancelled() so we still report when some upstream job failed. | |
| # Same fork-only filter as is-allowed-event — see comment there. | |
| if: | | |
| !cancelled() && | |
| github.event_name == 'workflow_run' && | |
| github.event.workflow_run.conclusion == 'success' && | |
| github.event.workflow_run.pull_requests[0] == null | |
| needs: | |
| - start-build-check | |
| - decide-runs | |
| - build | |
| - verify-requirements | |
| - verify-format-yaml | |
| - verify-format-sql | |
| - test-bqetl | |
| - integration | |
| - generate-sql | |
| - trigger-private-ci | |
| - docs | |
| - deploy-changes-to-stage | |
| - dry-run-sql | |
| - test-sql | |
| - test-routines | |
| - validate-views | |
| - validate-metadata | |
| - validate-backfills | |
| runs-on: ubuntu-latest | |
| permissions: | |
| checks: write | |
| actions: read # for listing jobs in this workflow run | |
| steps: | |
| - name: Post check runs for each upstream job | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| HEAD_SHA: ${{ github.event.workflow_run.head_sha }} | |
| RUN_ID: ${{ github.run_id }} | |
| DETAILS_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| BUILD_CHECK_ID: ${{ needs.start-build-check.outputs.check_id }} | |
| run: | | |
| set -euo pipefail | |
| # Iterate the Actions Jobs API for this run; each check-run's | |
| # details_url points at the specific job's log so failing checks | |
| # deep-link directly to the failure (rather than the run overview). | |
| # Skip the reporter jobs themselves to avoid self-reporting. | |
| overall=success | |
| gh api --paginate \ | |
| "/repos/${{ github.repository }}/actions/runs/$RUN_ID/jobs" \ | |
| --jq '.jobs[] | |
| | select(.name != "Report PR checks" | |
| and .name != "Report fork-gate result" | |
| and .name != "Start build check") | |
| | "\(.name)\t\(.conclusion // "neutral")\t\(.html_url)"' \ | |
| | while IFS=$'\t' read -r name conclusion html_url; do | |
| case "$conclusion" in | |
| success) c=success ;; | |
| failure) c=failure ;; | |
| cancelled) c=cancelled ;; | |
| skipped) c=skipped ;; | |
| *) c=neutral ;; | |
| esac | |
| echo "Reporting: $name -> $c ($html_url)" | |
| gh api -X POST "/repos/${{ github.repository }}/check-runs" \ | |
| -f name="$name" \ | |
| -f head_sha="$HEAD_SHA" \ | |
| -f status=completed \ | |
| -f conclusion="$c" \ | |
| -f details_url="$html_url" >/dev/null | |
| done | |
| # PATCH the rolled-up "Build" check (created by start-build-check) to | |
| # completed. Conclusion fails if any upstream `needs` failed/cancelled. | |
| # Skip the PATCH if start-build-check failed (empty id) — otherwise | |
| # we'd 404 against /check-runs/<empty>. The orphan in_progress check | |
| # is the lesser evil; start-build-check's failure is itself visible. | |
| if [[ -z "$BUILD_CHECK_ID" ]]; then | |
| echo "::warning::No BUILD_CHECK_ID — start-build-check must have failed; skipping rollup PATCH." | |
| else | |
| for r in $(echo '${{ toJSON(needs) }}' | jq -r 'to_entries[] | .value.result'); do | |
| case "$r" in | |
| failure|cancelled) overall=failure ;; | |
| esac | |
| done | |
| gh api -X PATCH "/repos/${{ github.repository }}/check-runs/$BUILD_CHECK_ID" \ | |
| -f status=completed \ | |
| -f conclusion="$overall" \ | |
| -f details_url="$DETAILS_URL" >/dev/null | |
| fi | |
| # Surface gate rejection / failure as a single check on the PR. Without this, | |
| # a rejected approval (or errored gate) shows nothing in the PR UI. | |
| report-gate-failure: | |
| name: Report fork-gate result | |
| if: github.event_name == 'workflow_run' && github.event.workflow_run.conclusion != 'success' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| checks: write | |
| steps: | |
| - name: Post failure check | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| HEAD_SHA: ${{ github.event.workflow_run.head_sha }} | |
| GATE_URL: ${{ github.event.workflow_run.html_url }} | |
| GATE_CONCLUSION: ${{ github.event.workflow_run.conclusion }} | |
| run: | | |
| set -euo pipefail | |
| summary="The fork-PR approval gate concluded with \`$GATE_CONCLUSION\`. " | |
| summary+="CI did not run. A maintainer must approve the gate workflow " | |
| summary+="for this push, or re-run if it errored." | |
| gh api -X POST "/repos/${{ github.repository }}/check-runs" \ | |
| -f name='Fork PR Gate' \ | |
| -f head_sha="$HEAD_SHA" \ | |
| -f status=completed \ | |
| -f conclusion=failure \ | |
| -f details_url="$GATE_URL" \ | |
| -f "output[title]=Gate $GATE_CONCLUSION" \ | |
| -f "output[summary]=$summary" >/dev/null |