data(subpop): clean Haiku constituent re-run + ensemble re-blend #805

Workflow file for this run

.github/workflows/ci.yml at 193500e

	name: CI

	on:
	push:
	branches: [main]
	pull_request:
	# `labeled` is included so applying the `vrt-baseline-update` label starts
	# a run immediately — no extra push needed (#311). When `types` is present
	# the defaults must be listed explicitly.
	types: [opened, synchronize, reopened, labeled]

	# Adding `labeled` means any label application starts a full CI run on the
	# same SHA as a possibly in-flight run. Dedupe per-PR so label adds (and the
	# baseline commit's synchronize event) supersede rather than stack. Pushes to
	# main never cancel each other — the group is keyed on the unique SHA there.
	concurrency:
	group: ci-${{ github.event.pull_request.number \|\| github.sha }}
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}

	jobs:
	lint:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v6
	- uses: actions/setup-python@v6
	with:
	python-version: "3.14"
	- run: pip install ruff
	- run: ruff check .
	- run: ruff format --check .
	- name: Guard against hyphenated sibling-project regression (sb-1el9)
	# The sibling project's canonical names are `synthpanel` (PyPI package)
	# and `SynthPanel` (GitHub repo). The stale pre-rename hyphenated form
	# should never reappear. Historical audit notes (post-flip-audit-*.md)
	# are allowed to reference the old name while documenting the rename.
	# The pattern is assembled at runtime so this step doesn't match itself.
	run: \|
	needle="synth""-""panel"
	matches=$(grep -rnI "$needle" \
	--exclude-dir=node_modules \
	--exclude-dir=.git \
	--exclude-dir=leaderboard-results \
	--exclude-dir=dist \
	--exclude-dir=.astro \
	--exclude="post-flip-audit-*.md" \
	. \|\| true)
	if [ -n "$matches" ]; then
	echo "::error::Hyphenated sibling-project name found — use 'synthpanel' (PyPI) or 'SynthPanel' (repo)."
	echo "$matches"
	exit 1
	fi

	test:
	runs-on: ubuntu-latest
	strategy:
	matrix:
	python-version: ["3.10", "3.11", "3.12", "3.13"]
	steps:
	- uses: actions/checkout@v6
	- uses: actions/setup-python@v6
	with:
	python-version: ${{ matrix.python-version }}
	- run: pip install -e '.[dev]'
	- run: pytest tests/
	- name: Null-agent baseline SPS floor (sb-lhoh)
	# Explicit gate surfacing baseline drift in the CI log even when
	# the pytest line above is buried in other output. Mirrors the
	# assertion in tests/test_baseline_floors.py; kept at the end so
	# the pytest run still executes on matrix failures.
	run: python3 scripts/baseline-floors-report.py

	validate-submissions:
	runs-on: ubuntu-latest
	# Required gate. The historical data-quality follow-up (sb-7gn) backfilled
	# stale jsd/tau/parity, normalized partial-mass distributions, and added
	# an all-zero sentinel exception in _validate_distributions, so PRs that
	# touch leaderboard-results/ are now safe to block on validation errors.
	steps:
	- uses: actions/checkout@v6
	with:
	fetch-depth: 0
	- uses: actions/setup-python@v6
	with:
	python-version: "3.14"
	- run: pip install -e .
	- name: Find added / modified submissions
	id: changed
	run: \|
	set -euo pipefail
	if [[ "${{ github.event_name }}" == "pull_request" ]]; then
	base_sha="${{ github.event.pull_request.base.sha }}"
	else
	base_sha="$(git rev-parse HEAD^ 2>/dev/null \|\| echo '')"
	fi
	# NUL-delimited so filenames with spaces (e.g. "... t=0.3_...json")
	# survive argv splitting. Written to a file so xargs -0 can fan out.
	list=/tmp/synthbench-changed-files.txt
	: > "$list"
	if [[ -n "$base_sha" ]]; then
	git diff -z --name-only --diff-filter=AM "$base_sha" HEAD -- \
	'leaderboard-results/*.json' > "$list"
	fi
	count=$(tr -cd '\0' < "$list" \| wc -c \| tr -d ' ')
	echo "count=$count" >> "$GITHUB_OUTPUT"
	echo "Submissions to validate: $count"
	tr '\0' '\n' < "$list"
	- name: Validate new submissions
	if: steps.changed.outputs.count != '0'
	# Tier 1+2 errors (including the recomputed-SPS check) block the
	# merge; --tier3 additionally runs the statistical anomaly detectors
	# with the existing leaderboard as the peer pool for the cross-run
	# outlier check, so reviewers see anomaly warnings in the CI log
	# before approving a submission PR.
	run: \|
	xargs -0 -a /tmp/synthbench-changed-files.txt \
	synthbench validate --tier3 --peers leaderboard-results

	worker-data-proxy:
	# sb-io1: Cloudflare Worker that gates R2 reads on a valid Supabase JWT.
	# Runs in isolation (no dependency on site/ or Python) — just the Worker
	# package's own lint/typecheck/test gates.
	runs-on: ubuntu-latest
	defaults:
	run:
	working-directory: workers/data-proxy
	steps:
	- uses: actions/checkout@v6
	- uses: actions/setup-node@v6
	with:
	node-version: "24"
	- name: Install
	run: npm install --no-audit --no-fund
	- name: Biome lint
	run: npm run lint
	- name: Typecheck
	run: npm run typecheck
	- name: Test
	run: npm test

	site:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v6
	- uses: actions/setup-python@v6
	with:
	python-version: "3.14"
	- uses: actions/setup-node@v6
	with:
	node-version: "24"
	- run: pip install -e .
	- name: Generate leaderboard data
	run: synthbench publish-data --results-dir leaderboard-results --output site/src/data/leaderboard.json
	- name: Generate run explorer artifacts
	run: synthbench publish-runs --results-dir leaderboard-results --output-dir site/public/data
	- name: Verify publish integrity (sb-4zy)
	run: python3 scripts/verify-publish-integrity.py
	- name: Install site dependencies
	working-directory: site
	run: npm ci
	- name: Format generated data
	working-directory: site
	run: npx biome format --write src/data/leaderboard.json
	- name: Typecheck
	working-directory: site
	run: npx tsc --noEmit
	- name: Biome lint
	working-directory: site
	run: npx biome check .
	- name: Test Pages Functions (gated-data edge guard)
	working-directory: site
	run: npm run test:functions
	- name: Build site
	working-directory: site
	run: npm run build
	- name: Verify deployment artifact
	working-directory: site
	run: \|
	echo "Checking site/dist/ matches expected deployment structure..."
	test -d dist \|\| { echo "FAIL: site/dist/ directory missing"; exit 1; }
	test -f dist/index.html \|\| { echo "FAIL: dist/index.html missing"; exit 1; }
	test -d dist/_astro \|\| { echo "FAIL: dist/_astro/ directory missing"; exit 1; }
	test -f dist/favicon.svg \|\| { echo "FAIL: dist/favicon.svg missing"; exit 1; }
	# Gated-data edge guard (advanced-mode Pages Function) MUST ship in the
	# deploy root, or gated /data artifacts would 404-then-Always-Online
	# instead of returning a fresh no-store 403. Fail the build if Astro
	# ever stops copying these from site/public/.
	test -f dist/_worker.js \|\| { echo "FAIL: dist/_worker.js missing — gated-data edge guard would NOT deploy"; exit 1; }
	test -f dist/_routes.json \|\| { echo "FAIL: dist/_routes.json missing — edge guard routing would be unscoped"; exit 1; }
	echo "Deployment artifact verified:"
	ls -la dist/
	echo "---"
	ls dist/_astro/ \| head -5

	visual:
	# Smoke VRT (sb-o3b): runs in Ubuntu. Baselines live under
	# __screenshots__/desktop-chromium/ and are generated in CI, not on
	# dev machines. Apply the `vrt-baseline-update` PR label to regenerate
	# baselines and commit them back to the PR branch automatically (#311):
	# 1. The `labeled` event starts a run immediately.
	# 2. The run regenerates baselines, removes the label, and pushes the
	# commit AS the org `DV cross-repo` GitHub App (token minted from
	# Doppler dataviking-platform/prd creds). Unlike GITHUB_TOKEN
	# pushes, an App-token push fires `pull_request: synchronize`, so
	# CI re-runs on the new HEAD and the required checks land there —
	# no manual label-remove + empty-commit dance.
	# 3. The follow-up run is a plain compare-mode run (label already
	# removed). If label removal ever fails, the follow-up run
	# regenerates to a no-op diff, commits nothing, and proceeds to the
	# compare-mode test — a commit-message guard below is the final
	# backstop against a regen/push loop.
	runs-on: ubuntu-latest
	permissions:
	contents: write
	# Needed to remove the vrt-baseline-update label after committing.
	pull-requests: write
	steps:
	- uses: actions/checkout@v6
	with:
	# Branch checkout (not the merge ref) so the label-triggered update
	# step can commit new baselines back to the PR branch. Falls back
	# to push refs for non-PR events. The baseline push itself uses the
	# cross-repo App token (see the commit step) — GITHUB_TOKEN pushes
	# don't re-trigger CI. Fork PRs skip the regen flow entirely
	# (same-repo guard in the vrt_gate step).
	ref: ${{ github.event.pull_request.head.ref \|\| github.ref }}
	repository: ${{ github.event.pull_request.head.repo.full_name \|\| github.repository }}
	token: ${{ secrets.GITHUB_TOKEN }}
	fetch-depth: 0
	- uses: actions/setup-python@v6
	with:
	python-version: "3.14"
	- uses: actions/setup-node@v6
	with:
	node-version: "24"
	- run: pip install -e .
	- name: Generate leaderboard data
	run: synthbench publish-data --results-dir leaderboard-results --output site/src/data/leaderboard.json
	- name: Generate run explorer artifacts
	run: synthbench publish-runs --results-dir leaderboard-results --output-dir site/public/data
	- name: Verify publish integrity (sb-4zy)
	run: python3 scripts/verify-publish-integrity.py
	- name: Install site dependencies
	working-directory: site
	run: npm ci
	- name: Install Playwright browsers
	working-directory: site
	run: npx playwright install --with-deps chromium
	- name: Format generated data
	working-directory: site
	run: npx biome format --write src/data/leaderboard.json
	- name: Build site
	working-directory: site
	run: npm run build
	- name: Decide whether to regenerate baselines
	id: vrt_gate
	# Same-repo guard: fork PRs have no secrets and can't be pushed to,
	# so the regen flow is skipped entirely (see site/e2e/README.md).
	# Loop backstop: if HEAD is already a baseline-update bot commit,
	# never regenerate again — even if the label somehow survived
	# removal AND regen were non-deterministic, the chain stops here.
	if: >-
	github.event_name == 'pull_request' &&
	contains(github.event.pull_request.labels.*.name, 'vrt-baseline-update') &&
	github.event.pull_request.head.repo.full_name == github.repository
	run: \|
	set -euo pipefail
	last_subject=$(git log -1 --pretty=%s)
	if [[ "$last_subject" == "chore(vrt): update baselines via vrt-baseline-update label" ]]; then
	echo "HEAD is already a baseline-update commit — skipping regen, running compare-mode only."
	echo "run=false" >> "$GITHUB_OUTPUT"
	else
	echo "run=true" >> "$GITHUB_OUTPUT"
	fi
	- name: Install Doppler CLI
	if: steps.vrt_gate.outputs.run == 'true'
	uses: dopplerhq/cli-action@4819d808ab99e5cde19a0637a16536a4038fad73 # v4
	- name: Load TF App credentials from dataviking-platform
	if: steps.vrt_gate.outputs.run == 'true'
	# Same Doppler sourcing pattern as pages.yml / cf-pages.yml (sb-vkz):
	# shared org creds live in dataviking-platform/prd, read via the
	# read-only DOPPLER_PLATFORM_TOKEN. The PEM is multiline, so mask
	# line-by-line (::add-mask:: only matches whole single-line values).
	# The TF GitHub App is used (NOT DV_CROSS_REPO_APP): verified
	# 2026-07-14 via the App API that the TF App is installed org-wide
	# (repository_selection=all, includes synthbench) with
	# contents: write, while the DV cross-repo App has contents: read
	# and is only installed on the mtg-* repos.
	env:
	DOPPLER_TOKEN: ${{ secrets.DOPPLER_PLATFORM_TOKEN }}
	run: \|
	set -euo pipefail
	for name in TF_GITHUB_APP_ID TF_GITHUB_APP_PEM_FILE; do
	value=$(doppler secrets get "$name" --project dataviking-platform --config prd --plain)
	while IFS= read -r line; do
	[[ -n "$line" ]] && echo "::add-mask::$line"
	done <<< "$value"
	printf '%s<<__DVP_EOF__\n%s\n__DVP_EOF__\n' "$name" "$value" >> "$GITHUB_ENV"
	done
	- name: Mint App token
	# Pushing with this token (instead of GITHUB_TOKEN) makes the
	# baseline commit fire `pull_request: synchronize`, so CI re-runs on
	# the new HEAD automatically. Same pattern as SynthPanel's
	# auto-tag.yml release-bot push.
	if: steps.vrt_gate.outputs.run == 'true'
	id: app_token
	uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
	with:
	app-id: ${{ env.TF_GITHUB_APP_ID }}
	private-key: ${{ env.TF_GITHUB_APP_PEM_FILE }}
	- name: Update baselines (label-triggered)
	if: steps.vrt_gate.outputs.run == 'true'
	working-directory: site
	env:
	CI: "true"
	run: npx playwright test --config=e2e/playwright.config.ts --update-snapshots=all smoke.visual
	- name: Commit updated baselines
	if: steps.vrt_gate.outputs.run == 'true'
	env:
	# Via env (not inline ${{ }}) so the token never lands in command
	# echo; create-github-app-token also registers it as masked.
	APP_TOKEN: ${{ steps.app_token.outputs.token }}
	GH_TOKEN: ${{ github.token }}
	PR_NUMBER: ${{ github.event.pull_request.number }}
	HEAD_REF: ${{ github.event.pull_request.head.ref }}
	run: \|
	set -euo pipefail
	git config user.name "dataviking-infra-terraform[bot]"
	git config user.email "274205567+dataviking-infra-terraform[bot]@users.noreply.github.com"
	git add site/e2e/visual/__screenshots__/
	# Remove the label BEFORE pushing so the synchronize event payload
	# of the follow-up run no longer carries it — that run is then a
	# plain compare-mode run. Removal is best-effort: if it fails, the
	# follow-up run regenerates to a no-op diff and commits nothing
	# (and the HEAD-commit guard in vrt_gate is the hard backstop).
	gh pr edit "$PR_NUMBER" --remove-label vrt-baseline-update \
	\|\| echo "::warning::Could not remove vrt-baseline-update label; follow-up run will no-op the regen."
	if git diff --cached --quiet; then
	echo "No baseline changes to commit."
	else
	git commit -m "chore(vrt): update baselines via vrt-baseline-update label"
	# Push AS the cross-repo App so the new HEAD re-triggers CI
	# (GITHUB_TOKEN pushes do not start workflow runs).
	git push "https://x-access-token:${APP_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "HEAD:${HEAD_REF}"
	fi
	- name: Run visual regression tests
	working-directory: site
	env:
	CI: "true"
	run: npx playwright test --config=e2e/playwright.config.ts smoke.visual
	- name: Upload VRT report on failure
	if: failure()
	uses: actions/upload-artifact@v7
	with:
	name: vrt-report
	path: \|
	site/playwright-report/
	site/e2e/test-results/
	retention-days: 14

	merge-ready:
	if: always()
	# `visual` is a required gate as of sb-04v: smoke VRT (sb-o3b) proved stable
	# across 10+ consecutive PRs with Ubuntu-generated baselines. If this job
	# legitimately fails on intentional visual changes, apply the
	# `vrt-baseline-update` label to the PR to regenerate baselines.
	needs: [lint, test, site, validate-submissions, visual, worker-data-proxy]
	runs-on: ubuntu-latest
	steps:
	- name: Check required jobs passed
	run: \|
	if [[ "${{ needs.lint.result }}" != "success" \|\| "${{ needs.test.result }}" != "success" \|\| "${{ needs.site.result }}" != "success" \|\| "${{ needs.validate-submissions.result }}" != "success" \|\| "${{ needs.visual.result }}" != "success" \|\| "${{ needs.worker-data-proxy.result }}" != "success" ]]; then
	echo "Required jobs failed"
	exit 1
	fi

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

data(subpop): clean Haiku constituent re-run + ensemble re-blend #805

Workflow file

data(subpop): clean Haiku constituent re-run + ensemble re-blend #805

Uh oh!

Workflow file for this run