docs(holdout): document the shipped system honestly; remove vestigial… #804
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| # `labeled` is included so applying the `vrt-baseline-update` label starts | |
| # a run immediately — no extra push needed (#311). When `types` is present | |
| # the defaults must be listed explicitly. | |
| types: [opened, synchronize, reopened, labeled] | |
| # Adding `labeled` means any label application starts a full CI run on the | |
| # same SHA as a possibly in-flight run. Dedupe per-PR so label adds (and the | |
| # baseline commit's synchronize event) supersede rather than stack. Pushes to | |
| # main never cancel each other — the group is keyed on the unique SHA there. | |
| concurrency: | |
| group: ci-${{ github.event.pull_request.number || github.sha }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.14" | |
| - run: pip install ruff | |
| - run: ruff check . | |
| - run: ruff format --check . | |
| - name: Guard against hyphenated sibling-project regression (sb-1el9) | |
| # The sibling project's canonical names are `synthpanel` (PyPI package) | |
| # and `SynthPanel` (GitHub repo). The stale pre-rename hyphenated form | |
| # should never reappear. Historical audit notes (post-flip-audit-*.md) | |
| # are allowed to reference the old name while documenting the rename. | |
| # The pattern is assembled at runtime so this step doesn't match itself. | |
| run: | | |
| needle="synth""-""panel" | |
| matches=$(grep -rnI "$needle" \ | |
| --exclude-dir=node_modules \ | |
| --exclude-dir=.git \ | |
| --exclude-dir=leaderboard-results \ | |
| --exclude-dir=dist \ | |
| --exclude-dir=.astro \ | |
| --exclude="post-flip-audit-*.md" \ | |
| . || true) | |
| if [ -n "$matches" ]; then | |
| echo "::error::Hyphenated sibling-project name found — use 'synthpanel' (PyPI) or 'SynthPanel' (repo)." | |
| echo "$matches" | |
| exit 1 | |
| fi | |
| test: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12", "3.13"] | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - run: pip install -e '.[dev]' | |
| - run: pytest tests/ | |
| - name: Null-agent baseline SPS floor (sb-lhoh) | |
| # Explicit gate surfacing baseline drift in the CI log even when | |
| # the pytest line above is buried in other output. Mirrors the | |
| # assertion in tests/test_baseline_floors.py; kept at the end so | |
| # the pytest run still executes on matrix failures. | |
| run: python3 scripts/baseline-floors-report.py | |
| validate-submissions: | |
| runs-on: ubuntu-latest | |
| # Required gate. The historical data-quality follow-up (sb-7gn) backfilled | |
| # stale jsd/tau/parity, normalized partial-mass distributions, and added | |
| # an all-zero sentinel exception in _validate_distributions, so PRs that | |
| # touch leaderboard-results/ are now safe to block on validation errors. | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.14" | |
| - run: pip install -e . | |
| - name: Find added / modified submissions | |
| id: changed | |
| run: | | |
| set -euo pipefail | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| base_sha="${{ github.event.pull_request.base.sha }}" | |
| else | |
| base_sha="$(git rev-parse HEAD^ 2>/dev/null || echo '')" | |
| fi | |
| # NUL-delimited so filenames with spaces (e.g. "... t=0.3_...json") | |
| # survive argv splitting. Written to a file so xargs -0 can fan out. | |
| list=/tmp/synthbench-changed-files.txt | |
| : > "$list" | |
| if [[ -n "$base_sha" ]]; then | |
| git diff -z --name-only --diff-filter=AM "$base_sha" HEAD -- \ | |
| 'leaderboard-results/*.json' > "$list" | |
| fi | |
| count=$(tr -cd '\0' < "$list" | wc -c | tr -d ' ') | |
| echo "count=$count" >> "$GITHUB_OUTPUT" | |
| echo "Submissions to validate: $count" | |
| tr '\0' '\n' < "$list" | |
| - name: Validate new submissions | |
| if: steps.changed.outputs.count != '0' | |
| # Tier 1+2 errors (including the recomputed-SPS check) block the | |
| # merge; --tier3 additionally runs the statistical anomaly detectors | |
| # with the existing leaderboard as the peer pool for the cross-run | |
| # outlier check, so reviewers see anomaly warnings in the CI log | |
| # before approving a submission PR. | |
| run: | | |
| xargs -0 -a /tmp/synthbench-changed-files.txt \ | |
| synthbench validate --tier3 --peers leaderboard-results | |
| worker-data-proxy: | |
| # sb-io1: Cloudflare Worker that gates R2 reads on a valid Supabase JWT. | |
| # Runs in isolation (no dependency on site/ or Python) — just the Worker | |
| # package's own lint/typecheck/test gates. | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: workers/data-proxy | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version: "24" | |
| - name: Install | |
| run: npm install --no-audit --no-fund | |
| - name: Biome lint | |
| run: npm run lint | |
| - name: Typecheck | |
| run: npm run typecheck | |
| - name: Test | |
| run: npm test | |
| site: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.14" | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version: "24" | |
| - run: pip install -e . | |
| - name: Generate leaderboard data | |
| run: synthbench publish-data --results-dir leaderboard-results --output site/src/data/leaderboard.json | |
| - name: Generate run explorer artifacts | |
| run: synthbench publish-runs --results-dir leaderboard-results --output-dir site/public/data | |
| - name: Verify publish integrity (sb-4zy) | |
| run: python3 scripts/verify-publish-integrity.py | |
| - name: Install site dependencies | |
| working-directory: site | |
| run: npm ci | |
| - name: Format generated data | |
| working-directory: site | |
| run: npx biome format --write src/data/leaderboard.json | |
| - name: Typecheck | |
| working-directory: site | |
| run: npx tsc --noEmit | |
| - name: Biome lint | |
| working-directory: site | |
| run: npx biome check . | |
| - name: Test Pages Functions (gated-data edge guard) | |
| working-directory: site | |
| run: npm run test:functions | |
| - name: Build site | |
| working-directory: site | |
| run: npm run build | |
| - name: Verify deployment artifact | |
| working-directory: site | |
| run: | | |
| echo "Checking site/dist/ matches expected deployment structure..." | |
| test -d dist || { echo "FAIL: site/dist/ directory missing"; exit 1; } | |
| test -f dist/index.html || { echo "FAIL: dist/index.html missing"; exit 1; } | |
| test -d dist/_astro || { echo "FAIL: dist/_astro/ directory missing"; exit 1; } | |
| test -f dist/favicon.svg || { echo "FAIL: dist/favicon.svg missing"; exit 1; } | |
| # Gated-data edge guard (advanced-mode Pages Function) MUST ship in the | |
| # deploy root, or gated /data artifacts would 404-then-Always-Online | |
| # instead of returning a fresh no-store 403. Fail the build if Astro | |
| # ever stops copying these from site/public/. | |
| test -f dist/_worker.js || { echo "FAIL: dist/_worker.js missing — gated-data edge guard would NOT deploy"; exit 1; } | |
| test -f dist/_routes.json || { echo "FAIL: dist/_routes.json missing — edge guard routing would be unscoped"; exit 1; } | |
| echo "Deployment artifact verified:" | |
| ls -la dist/ | |
| echo "---" | |
| ls dist/_astro/ | head -5 | |
| visual: | |
| # Smoke VRT (sb-o3b): runs in Ubuntu. Baselines live under | |
| # __screenshots__/desktop-chromium/ and are generated in CI, not on | |
| # dev machines. Apply the `vrt-baseline-update` PR label to regenerate | |
| # baselines and commit them back to the PR branch automatically (#311): | |
| # 1. The `labeled` event starts a run immediately. | |
| # 2. The run regenerates baselines, removes the label, and pushes the | |
| # commit AS the org `DV cross-repo` GitHub App (token minted from | |
| # Doppler dataviking-platform/prd creds). Unlike GITHUB_TOKEN | |
| # pushes, an App-token push fires `pull_request: synchronize`, so | |
| # CI re-runs on the new HEAD and the required checks land there — | |
| # no manual label-remove + empty-commit dance. | |
| # 3. The follow-up run is a plain compare-mode run (label already | |
| # removed). If label removal ever fails, the follow-up run | |
| # regenerates to a no-op diff, commits nothing, and proceeds to the | |
| # compare-mode test — a commit-message guard below is the final | |
| # backstop against a regen/push loop. | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| # Needed to remove the vrt-baseline-update label after committing. | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| # Branch checkout (not the merge ref) so the label-triggered update | |
| # step can commit new baselines back to the PR branch. Falls back | |
| # to push refs for non-PR events. The baseline push itself uses the | |
| # cross-repo App token (see the commit step) — GITHUB_TOKEN pushes | |
| # don't re-trigger CI. Fork PRs skip the regen flow entirely | |
| # (same-repo guard in the vrt_gate step). | |
| ref: ${{ github.event.pull_request.head.ref || github.ref }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.14" | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version: "24" | |
| - run: pip install -e . | |
| - name: Generate leaderboard data | |
| run: synthbench publish-data --results-dir leaderboard-results --output site/src/data/leaderboard.json | |
| - name: Generate run explorer artifacts | |
| run: synthbench publish-runs --results-dir leaderboard-results --output-dir site/public/data | |
| - name: Verify publish integrity (sb-4zy) | |
| run: python3 scripts/verify-publish-integrity.py | |
| - name: Install site dependencies | |
| working-directory: site | |
| run: npm ci | |
| - name: Install Playwright browsers | |
| working-directory: site | |
| run: npx playwright install --with-deps chromium | |
| - name: Format generated data | |
| working-directory: site | |
| run: npx biome format --write src/data/leaderboard.json | |
| - name: Build site | |
| working-directory: site | |
| run: npm run build | |
| - name: Decide whether to regenerate baselines | |
| id: vrt_gate | |
| # Same-repo guard: fork PRs have no secrets and can't be pushed to, | |
| # so the regen flow is skipped entirely (see site/e2e/README.md). | |
| # Loop backstop: if HEAD is already a baseline-update bot commit, | |
| # never regenerate again — even if the label somehow survived | |
| # removal AND regen were non-deterministic, the chain stops here. | |
| if: >- | |
| github.event_name == 'pull_request' && | |
| contains(github.event.pull_request.labels.*.name, 'vrt-baseline-update') && | |
| github.event.pull_request.head.repo.full_name == github.repository | |
| run: | | |
| set -euo pipefail | |
| last_subject=$(git log -1 --pretty=%s) | |
| if [[ "$last_subject" == "chore(vrt): update baselines via vrt-baseline-update label" ]]; then | |
| echo "HEAD is already a baseline-update commit — skipping regen, running compare-mode only." | |
| echo "run=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "run=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Install Doppler CLI | |
| if: steps.vrt_gate.outputs.run == 'true' | |
| uses: dopplerhq/cli-action@4819d808ab99e5cde19a0637a16536a4038fad73 # v4 | |
| - name: Load TF App credentials from dataviking-platform | |
| if: steps.vrt_gate.outputs.run == 'true' | |
| # Same Doppler sourcing pattern as pages.yml / cf-pages.yml (sb-vkz): | |
| # shared org creds live in dataviking-platform/prd, read via the | |
| # read-only DOPPLER_PLATFORM_TOKEN. The PEM is multiline, so mask | |
| # line-by-line (::add-mask:: only matches whole single-line values). | |
| # The TF GitHub App is used (NOT DV_CROSS_REPO_APP): verified | |
| # 2026-07-14 via the App API that the TF App is installed org-wide | |
| # (repository_selection=all, includes synthbench) with | |
| # contents: write, while the DV cross-repo App has contents: read | |
| # and is only installed on the mtg-* repos. | |
| env: | |
| DOPPLER_TOKEN: ${{ secrets.DOPPLER_PLATFORM_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| for name in TF_GITHUB_APP_ID TF_GITHUB_APP_PEM_FILE; do | |
| value=$(doppler secrets get "$name" --project dataviking-platform --config prd --plain) | |
| while IFS= read -r line; do | |
| [[ -n "$line" ]] && echo "::add-mask::$line" | |
| done <<< "$value" | |
| printf '%s<<__DVP_EOF__\n%s\n__DVP_EOF__\n' "$name" "$value" >> "$GITHUB_ENV" | |
| done | |
| - name: Mint App token | |
| # Pushing with this token (instead of GITHUB_TOKEN) makes the | |
| # baseline commit fire `pull_request: synchronize`, so CI re-runs on | |
| # the new HEAD automatically. Same pattern as SynthPanel's | |
| # auto-tag.yml release-bot push. | |
| if: steps.vrt_gate.outputs.run == 'true' | |
| id: app_token | |
| uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 | |
| with: | |
| app-id: ${{ env.TF_GITHUB_APP_ID }} | |
| private-key: ${{ env.TF_GITHUB_APP_PEM_FILE }} | |
| - name: Update baselines (label-triggered) | |
| if: steps.vrt_gate.outputs.run == 'true' | |
| working-directory: site | |
| env: | |
| CI: "true" | |
| run: npx playwright test --config=e2e/playwright.config.ts --update-snapshots=all smoke.visual | |
| - name: Commit updated baselines | |
| if: steps.vrt_gate.outputs.run == 'true' | |
| env: | |
| # Via env (not inline ${{ }}) so the token never lands in command | |
| # echo; create-github-app-token also registers it as masked. | |
| APP_TOKEN: ${{ steps.app_token.outputs.token }} | |
| GH_TOKEN: ${{ github.token }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| HEAD_REF: ${{ github.event.pull_request.head.ref }} | |
| run: | | |
| set -euo pipefail | |
| git config user.name "dataviking-infra-terraform[bot]" | |
| git config user.email "274205567+dataviking-infra-terraform[bot]@users.noreply.github.com" | |
| git add site/e2e/visual/__screenshots__/ | |
| # Remove the label BEFORE pushing so the synchronize event payload | |
| # of the follow-up run no longer carries it — that run is then a | |
| # plain compare-mode run. Removal is best-effort: if it fails, the | |
| # follow-up run regenerates to a no-op diff and commits nothing | |
| # (and the HEAD-commit guard in vrt_gate is the hard backstop). | |
| gh pr edit "$PR_NUMBER" --remove-label vrt-baseline-update \ | |
| || echo "::warning::Could not remove vrt-baseline-update label; follow-up run will no-op the regen." | |
| if git diff --cached --quiet; then | |
| echo "No baseline changes to commit." | |
| else | |
| git commit -m "chore(vrt): update baselines via vrt-baseline-update label" | |
| # Push AS the cross-repo App so the new HEAD re-triggers CI | |
| # (GITHUB_TOKEN pushes do not start workflow runs). | |
| git push "https://x-access-token:${APP_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "HEAD:${HEAD_REF}" | |
| fi | |
| - name: Run visual regression tests | |
| working-directory: site | |
| env: | |
| CI: "true" | |
| run: npx playwright test --config=e2e/playwright.config.ts smoke.visual | |
| - name: Upload VRT report on failure | |
| if: failure() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: vrt-report | |
| path: | | |
| site/playwright-report/ | |
| site/e2e/test-results/ | |
| retention-days: 14 | |
| merge-ready: | |
| if: always() | |
| # `visual` is a required gate as of sb-04v: smoke VRT (sb-o3b) proved stable | |
| # across 10+ consecutive PRs with Ubuntu-generated baselines. If this job | |
| # legitimately fails on intentional visual changes, apply the | |
| # `vrt-baseline-update` label to the PR to regenerate baselines. | |
| needs: [lint, test, site, validate-submissions, visual, worker-data-proxy] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check required jobs passed | |
| run: | | |
| if [[ "${{ needs.lint.result }}" != "success" || "${{ needs.test.result }}" != "success" || "${{ needs.site.result }}" != "success" || "${{ needs.validate-submissions.result }}" != "success" || "${{ needs.visual.result }}" != "success" || "${{ needs.worker-data-proxy.result }}" != "success" ]]; then | |
| echo "Required jobs failed" | |
| exit 1 | |
| fi |