docs(holdout): document the shipped system honestly; remove vestigial split (#259) #238
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy CF Pages | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: [main] | |
| paths: | |
| - 'src/synthbench/publish.py' | |
| - 'leaderboard-results/**' | |
| - 'site/**' | |
| - '.github/workflows/cf-pages.yml' | |
| pull_request: | |
| paths: | |
| - 'src/synthbench/publish.py' | |
| - 'leaderboard-results/**' | |
| - 'site/**' | |
| - '.github/workflows/cf-pages.yml' | |
| permissions: | |
| contents: read | |
| deployments: write | |
| pull-requests: write | |
| concurrency: | |
| group: cf-pages-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| deploy: | |
| runs-on: ubuntu-latest | |
| # Fork PRs have no access to secrets; skip to avoid false-negative deploys. | |
| if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.14' | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version: '24' | |
| - run: pip install -e .[r2] | |
| - name: Install Doppler CLI | |
| uses: dopplerhq/cli-action@4819d808ab99e5cde19a0637a16536a4038fad73 # v4 | |
| # sb-vkz doppler-reorg: pull shared Cloudflare/R2 creds from the | |
| # dataviking-platform/prd project (read-only DOPPLER_PLATFORM_TOKEN) | |
| # instead of synthbench/prd duplicates. R2_BUCKET stays on synthbench | |
| # because the bucket name is product-specific. | |
| - name: Load shared infra secrets from dataviking-platform | |
| env: | |
| DOPPLER_TOKEN: ${{ secrets.DOPPLER_PLATFORM_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| for name in CLOUDFLARE_API_TOKEN CLOUDFLARE_ACCOUNT_ID R2_ACCOUNT_ID R2_ACCESS_KEY_ID R2_SECRET_ACCESS_KEY; do | |
| value=$(doppler secrets get "$name" --project dataviking-platform --config prd --plain) | |
| echo "::add-mask::$value" | |
| printf '%s<<__DVP_EOF__\n%s\n__DVP_EOF__\n' "$name" "$value" >> "$GITHUB_ENV" | |
| done | |
| - name: Generate leaderboard data | |
| run: synthbench publish-data --results-dir leaderboard-results --output site/src/data/leaderboard.json | |
| - name: Generate run explorer artifacts | |
| # SYNTHBENCH_PUBLISH_STRICT=1 makes gating fail closed: if the R2 | |
| # uploader is not configured, the build FAILS here instead of | |
| # deploying without gated data. Gated artifacts are never written to | |
| # site/public/data/ — the old local-write fallback shipped | |
| # license-restricted human_distribution data to the public origin. | |
| run: synthbench publish-runs --results-dir leaderboard-results --output-dir site/public/data | |
| env: | |
| R2_BUCKET: ${{ secrets.R2_BUCKET }} | |
| SYNTHBENCH_PUBLISH_STRICT: "1" | |
| # Also re-checks that no gated-dataset artifact is present under the | |
| # local publish output (license-gating backstop). | |
| - name: Verify publish integrity (sb-4zy) | |
| run: python3 scripts/verify-publish-integrity.py | |
| - name: Load Supabase public keys for Astro build | |
| env: | |
| DOPPLER_TOKEN: ${{ secrets.DOPPLER_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| # Astro client needs PUBLIC_-prefixed env vars baked at build time. | |
| # Source from synthbench/prd Doppler (where Supabase product secrets live). | |
| for src in SUPABASE_URL:PUBLIC_SUPABASE_URL SUPABASE_ANON_KEY:PUBLIC_SUPABASE_ANON_KEY; do | |
| from="${src%%:*}"; to="${src##*:}" | |
| value=$(doppler secrets get "$from" --project synthbench --config prd --plain) | |
| echo "::add-mask::$value" | |
| printf '%s<<__SB_EOF__\n%s\n__SB_EOF__\n' "$to" "$value" >> "$GITHUB_ENV" | |
| done | |
| - name: Build Astro site | |
| working-directory: site | |
| env: | |
| SITE_URL: https://synthbench.org | |
| BASE_PATH: / | |
| PUBLIC_GATED_API_BASE: https://api.synthbench.org | |
| run: | | |
| npm ci | |
| npx biome format --write src/data/leaderboard.json | |
| npx tsc --noEmit | |
| npm run build | |
| # Make-or-break guard: the gated-data edge guard is an advanced-mode | |
| # Pages Function (site/public/_worker.js → site/dist/_worker.js). Wrangler | |
| # only bundles it when it sits at the root of the deployed directory. If | |
| # Astro ever stops copying it, gated /data artifacts would silently 404 | |
| # (and Always-Online would keep serving the stale 200) instead of | |
| # returning a fresh no-store 403. Fail the deploy rather than ship without | |
| # the guard. | |
| - name: Verify gated-data edge guard is bundled | |
| working-directory: site | |
| run: | | |
| test -f dist/_worker.js || { echo "FAIL: dist/_worker.js missing — refusing to deploy without the gated-data edge guard"; exit 1; } | |
| test -f dist/_routes.json || { echo "FAIL: dist/_routes.json missing — edge guard routing would be unscoped"; exit 1; } | |
| echo "Edge guard present: dist/_worker.js + dist/_routes.json" | |
| - name: Publish to Cloudflare Pages | |
| # apiToken/accountId come from $GITHUB_ENV (sourced from Doppler | |
| # dataviking-platform/prd in the earlier step); wrangler reads the | |
| # CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID env vars directly. | |
| uses: cloudflare/wrangler-action@ebbaa1584979971c8614a24965b4405ff95890e0 # v4 | |
| with: | |
| apiToken: ${{ env.CLOUDFLARE_API_TOKEN }} | |
| accountId: ${{ env.CLOUDFLARE_ACCOUNT_ID }} | |
| command: pages deploy site/dist --project-name=synthbench --branch=${{ github.head_ref || github.ref_name }} | |
| gitHubToken: ${{ secrets.GITHUB_TOKEN }} |