docs(correctness): pin joinorder_synthetic cross-surface dtype root cause + fix layer #932

Workflow file for this run

	name: Develop PR

	on:
	pull_request:
	branches:
	- develop

	concurrency:
	group: develop-pr-${{ github.ref }}
	cancel-in-progress: true

	permissions:
	contents: read

	jobs:
	ci-paths:
	name: ci-paths
	runs-on: ubuntu-latest
	outputs:
	safe-content-only: ${{ steps.classify.outputs.safe-content-only }}
	needs-code-ci: ${{ steps.classify.outputs.needs-code-ci }}
	content-guard-needed: ${{ steps.classify.outputs.content-guard-needed }}
	changed-paths: ${{ steps.classify.outputs.changed-paths }}
	code-paths: ${{ steps.classify.outputs.code-paths }}
	unknown-paths: ${{ steps.classify.outputs.unknown-paths }}
	estimated-runner-minutes-saved: ${{ steps.classify.outputs.estimated-runner-minutes-saved }}
	steps:
	- name: Checkout code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Fetch base branch
	run: git fetch --no-tags origin "${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}"

	- name: Classify changed paths
	id: classify
	# Bare python is intentional: the classifier is stdlib-only so
	# content-only PRs do not pay a dependency-sync cost before deciding.
	run: \|
	python scripts/path_filter_decision.py \
	--base-ref "origin/${{ github.base_ref }}" \
	--json-out ci-path-decision.json \
	--lists-dir .ci-paths \
	--github-output "$GITHUB_OUTPUT" \
	--summary "$GITHUB_STEP_SUMMARY"

	- name: Upload path decision
	uses: actions/upload-artifact@v4
	with:
	name: ci-path-decision
	path: ci-path-decision.json
	retention-days: 30

	content-guard:
	name: content-guard
	needs: ci-paths
	if: ${{ needs.ci-paths.outputs.content-guard-needed == 'true' }}
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Fetch base branch
	run: git fetch --no-tags origin "${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}"

	- name: Install uv
	uses: astral-sh/setup-uv@v4

	- name: Install dependencies
	run: \|
	uv sync --group dev

	- name: Recreate path lists
	# Bare python is intentional: the classifier is stdlib-only.
	run: \|
	python scripts/path_filter_decision.py \
	--base-ref "origin/${{ github.base_ref }}" \
	--json-out ci-path-decision.json \
	--lists-dir .ci-paths

	- name: Validate YAML hygiene
	shell: bash
	run: \|
	if [ ! -s .ci-paths/yaml.txt ]; then
	echo "No YAML content paths changed."
	exit 0
	fi
	mapfile -t yaml_paths < .ci-paths/yaml.txt
	uv run -- pre-commit run check-yaml --files "${yaml_paths[@]}"

	- name: Validate artifact hygiene
	run: uv run -- python _project/scripts/artifact_hygiene_check.py --all-tracked

	- name: Validate markdown hygiene
	shell: bash
	run: \|
	if [ ! -s .ci-paths/markdown.txt ]; then
	echo "No markdown content paths changed."
	exit 0
	fi
	mapfile -t markdown_paths < .ci-paths/markdown.txt
	uv run -- pre-commit run markdownlint --files "${markdown_paths[@]}"

	- name: Validate TODO graph
	shell: bash
	run: \|
	if [ ! -s .ci-paths/todo.txt ]; then
	echo "No TODO/DONE YAML paths changed."
	exit 0
	fi
	while IFS= read -r todo_path; do
	if [ ! -e "$todo_path" ]; then
	echo "Skipping deleted TODO/DONE path: $todo_path"
	continue
	fi
	uv run --project _project/scripts -- python _project/scripts/validate_todo.py "$todo_path"
	done < .ci-paths/todo.txt
	uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph

	- name: Validate executable TODO docs
	shell: bash
	run: \|
	if [ ! -s .ci-paths/todo.txt ]; then
	echo "No TODO/DONE YAML paths changed."
	exit 0
	fi
	uv run -- python -m pytest tests/unit/test_todo_executable_docs.py -q

	- name: Validate docs references
	shell: bash
	run: \|
	if [ ! -s .ci-paths/docs.txt ]; then
	echo "No docs paths changed."
	exit 0
	fi
	make docs-validate

	- name: Validate public contract drift guard
	shell: bash
	run: \|
	if [ ! -s .ci-paths/docs.txt ] && [ ! -s .ci-paths/markdown.txt ]; then
	echo "No docs or markdown paths changed."
	exit 0
	fi
	uv run -- python - <<'PY'
	from pathlib import Path

	contract = Path("docs/reference/public-contracts.md").read_text(encoding="utf-8")
	readme = Path("README.md").read_text(encoding="utf-8")

	forbidden_readme_count_claims = [
	"Twenty-Two Benchmarks",
	"SQL Platforms (",
	"DataFrame Platforms (",
	]
	offenders = [phrase for phrase in forbidden_readme_count_claims if phrase in readme]
	if offenders:
	raise SystemExit(
	"README.md has exact hand-maintained count claims; use registry-derived checks "
	f"or mark them non-authoritative: {offenders}"
	)

	required_surfaces = [
	"CLI commands and documented options",
	"Top-level Python wrapper facades",
	"`benchbox.base.BaseBenchmark`",
	"`BaseBenchmark.run_with_platform`",
	"Adapter subclassing hooks and base mixins",
	"Platform registry metadata",
	"MCP tools",
	"Result JSON bundles",
	"Explorer read model and generated browser inputs",
	"Public submission validator behavior",
	"SQL compatibility rule catalog",
	"Generated compatibility docs",
	"`benchbox.experimental` namespace",
	"`_project` scripts, audits, and analysis artifacts",
	"TODO, DONE, and ADR/future-state docs",
	]
	missing_surfaces = [surface for surface in required_surfaces if surface not in contract]
	if missing_surfaces:
	raise SystemExit(f"public-contracts.md missing required surfaces: {missing_surfaces}")

	required_statuses = [
	"`stable`",
	"`beta`",
	"`experimental`",
	"`repo_only`",
	"`deprecated`",
	"`document_only`",
	]
	missing_statuses = [status for status in required_statuses if status not in contract]
	if missing_statuses:
	raise SystemExit(f"public-contracts.md missing support_status values: {missing_statuses}")

	from benchbox.core.benchmark_registry import list_benchmark_ids, list_loader_benchmark_ids
	from benchbox.core.platform_registry import PlatformRegistry
	from benchbox.core.results.schema import SCHEMA_VERSION

	platforms = PlatformRegistry.get_all_platform_metadata()
	sql_capable = sum(1 for item in platforms.values() if item.get("capabilities", {}).get("supports_sql"))
	dataframe_capable = sum(
	1 for item in platforms.values() if item.get("capabilities", {}).get("supports_dataframe")
	)
	dual_mode = sum(
	1
	for item in platforms.values()
	if item.get("capabilities", {}).get("supports_sql")
	and item.get("capabilities", {}).get("supports_dataframe")
	)

	source_fragments = [
	f"{len(list_benchmark_ids())} benchmark metadata entries",
	f"{len(list_loader_benchmark_ids())} loader-resolved IDs",
	f"{len(platforms)} platform metadata entries",
	f"{sql_capable} SQL-capable",
	f"{dataframe_capable} DataFrame-capable",
	f"{dual_mode} dual-mode",
	f"Current result schema version: `{SCHEMA_VERSION}`",
	]
	missing_fragments = [fragment for fragment in source_fragments if fragment not in contract]
	if missing_fragments:
	raise SystemExit(f"public-contracts.md source-derived claims are stale: {missing_fragments}")
	PY

	code-lint:
	name: lint
	needs: ci-paths
	if: ${{ needs.ci-paths.outputs.needs-code-ci == 'true' }}
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.11'

	- name: Install uv
	uses: astral-sh/setup-uv@v4

	- name: Install dependencies
	run: \|
	uv sync --group dev

	- name: Lint and format check with ruff
	run: \|
	uv run ruff check .
	uv run ruff format --check .

	- name: Type check with ty
	run: uv run ty check

	- name: Validate test marker annotations
	run: make lint-markers

	- name: Validate artifact hygiene
	run: make artifact-hygiene

	- name: Timing policy (wall-clock allowlist)
	run: uv run -- python _project/scripts/timing_policy_check.py --strict

	- name: Compatibility governance drift check
	run: make compat-docs-check

	- name: Oracle coverage map drift check
	run: make oracle-coverage-map-check

	- name: Public contract drift check
	run: \|
	uv run -- python - <<'PY'
	from pathlib import Path
	from benchbox.core.benchmark_registry import list_benchmark_ids, list_loader_benchmark_ids
	from benchbox.core.platform_registry import PlatformRegistry
	from benchbox.core.results.schema import SCHEMA_VERSION

	contract = Path("docs/reference/public-contracts.md").read_text(encoding="utf-8")
	readme = Path("README.md").read_text(encoding="utf-8")

	forbidden_readme_count_claims = [
	"Twenty-Two Benchmarks",
	"SQL Platforms (",
	"DataFrame Platforms (",
	]
	offenders = [phrase for phrase in forbidden_readme_count_claims if phrase in readme]
	if offenders:
	raise SystemExit(
	"README.md has exact hand-maintained count claims; use registry-derived checks "
	f"or mark them non-authoritative: {offenders}"
	)

	platforms = PlatformRegistry.get_all_platform_metadata()
	sql_capable = sum(1 for item in platforms.values() if item.get("capabilities", {}).get("supports_sql"))
	dataframe_capable = sum(
	1 for item in platforms.values() if item.get("capabilities", {}).get("supports_dataframe")
	)
	dual_mode = sum(
	1
	for item in platforms.values()
	if item.get("capabilities", {}).get("supports_sql")
	and item.get("capabilities", {}).get("supports_dataframe")
	)

	source_fragments = [
	f"{len(list_benchmark_ids())} benchmark metadata entries",
	f"{len(list_loader_benchmark_ids())} loader-resolved IDs",
	f"{len(platforms)} platform metadata entries",
	f"{sql_capable} SQL-capable",
	f"{dataframe_capable} DataFrame-capable",
	f"{dual_mode} dual-mode",
	f"Current result schema version: `{SCHEMA_VERSION}`",
	]
	missing_fragments = [fragment for fragment in source_fragments if fragment not in contract]
	if missing_fragments:
	raise SystemExit(f"public-contracts.md source-derived claims are stale: {missing_fragments}")
	PY

	- name: Dependency audit (no unused declarations)
	run: make audit-deps

	- name: Release curation list drift check
	run: uv run -- python scripts/check_release_curation.py

	code-test:
	name: test (ubuntu-latest, 3.12)
	needs: ci-paths
	if: ${{ needs.ci-paths.outputs.needs-code-ci == 'true' }}
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Set up Python 3.12
	uses: actions/setup-python@v5
	with:
	python-version: '3.12'

	- name: Install uv
	uses: astral-sh/setup-uv@v4

	- name: Create virtual environment
	run: uv venv

	- name: Install dependencies
	run: \|
	uv sync --group dev

	- name: Run fast tests
	run: \|
	# -p pytest_cov re-enables coverage plugin (disabled by default in pytest.ini)
	uv run -- python -m pytest tests -m "fast and not (slow or stress or resource_heavy or live_integration)" --tb=short -p pytest_cov --cov=benchbox --cov-report=xml --cov-report=term-missing --cov-fail-under=70

	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v4
	with:
	file: ./coverage.xml
	fail_ci_if_error: false # Don't fail CI if Codecov upload fails (token may not be configured)
	token: ${{ secrets.CODECOV_TOKEN }}

	correctness-gate:
	name: correctness-gate
	needs: ci-paths
	if: ${{ needs.ci-paths.outputs.needs-code-ci == 'true' }}
	runs-on: ubuntu-latest
	timeout-minutes: 20
	steps:
	- name: Checkout code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Set up Python 3.12
	uses: actions/setup-python@v5
	with:
	python-version: '3.12'

	- name: Install uv
	uses: astral-sh/setup-uv@v4

	- name: Create virtual environment
	run: uv venv

	- name: Install dependencies
	run: uv sync --group dev

	- name: Run bounded real-result correctness gate
	run: make test-correctness-gate

	- name: Run TPC-Havoc variant-equivalence gate
	run: make tpchavoc-equivalence-report

	- name: Run TPC-Havoc DataFrame variant-equivalence gate
	run: make tpchavoc-dataframe-equivalence-report

	- name: Run SSB cross-surface SQL<->DataFrame equivalence gate
	run: make ssb-cross-surface-equivalence-report

	- name: Run CoffeeShop cross-surface SQL<->DataFrame equivalence gate
	run: make coffeeshop-cross-surface-equivalence-report

	postgres-integration:
	name: postgres-integration
	needs: ci-paths
	if: ${{ needs.ci-paths.outputs.needs-code-ci == 'true' }}
	runs-on: ubuntu-latest
	timeout-minutes: 15
	# Non-blocking: service container startup can be flaky; failures surfaced as
	# warnings rather than a gate block to avoid spurious PR failures.
	continue-on-error: true
	services:
	postgres:
	image: postgres:17
	env:
	POSTGRES_USER: benchbox
	POSTGRES_PASSWORD: benchbox
	POSTGRES_DB: benchbox_test
	ports:
	- 5432:5432
	options: >-
	--health-cmd pg_isready
	--health-interval 5s
	--health-timeout 5s
	--health-retries 10
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Set up Python 3.12
	uses: actions/setup-python@v5
	with:
	python-version: '3.12'

	- name: Install uv
	uses: astral-sh/setup-uv@v4

	- name: Install dependencies
	run: uv sync --group dev

	- name: Run PostgreSQL live integration tests
	run: \|
	uv run -- python -m pytest tests/integration/platforms/test_postgresql_live.py \
	-m "live_postgresql" --tb=short -v

	# Second-engine equivalence SAMPLE (non-blocking, see continue-on-error
	# above). The DuckDB SQL gate (correctness-gate job) stays the hard
	# blocker; this samples TPC-Havoc variant equivalence one engine over to
	# catch translation-induced divergence. Both canonical and variants are
	# translated to the postgres dialect and compared on this same instance.
	- name: Run TPC-Havoc PostgreSQL variant-equivalence sample
	env:
	PGHOST: localhost
	PGPORT: '5432'
	PGUSER: benchbox
	PGPASSWORD: benchbox
	PGDATABASE: benchbox_test
	run: \|
	uv run -- python -m pytest \
	tests/integration/platforms/test_tpchavoc_postgres_equivalence.py \
	-m "live_postgresql" -n 0 --tb=short -v

	datafusion-integration:
	name: datafusion-integration
	needs: ci-paths
	if: ${{ needs.ci-paths.outputs.needs-code-ci == 'true' }}
	runs-on: ubuntu-latest
	timeout-minutes: 15
	# Non-blocking: this is the THIRD-engine equivalence SAMPLE, not a gate.
	# The DuckDB SQL gate (correctness-gate job) stays the hard blocker.
	# DataFusion is in-process (no service container), and its execution gaps
	# DIFFER from Postgres's, so this samples translation-induced divergence a
	# third engine over. continue-on-error keeps it a sample, not a blocker.
	continue-on-error: true
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Set up Python 3.12
	uses: actions/setup-python@v5
	with:
	python-version: '3.12'

	- name: Install uv
	uses: astral-sh/setup-uv@v4

	- name: Install dependencies
	run: uv sync --group dev

	# Third-engine equivalence SAMPLE plus the DataFusion .tbl trailing-delimiter
	# load regression. Both canonical and variants are translated to the
	# datafusion dialect and compared on the same in-process session (shared
	# translation cancels out); DATAFUSION_TPCHAVOC_SKIPS variants are excluded,
	# never marked equivalent. Both files are slow-marked, so -m "slow" selects
	# them (overriding the default fast-lane deselect).
	- name: Run TPC-Havoc DataFusion variant-equivalence sample and .tbl load regression
	run: \|
	uv run -- python -m pytest \
	tests/integration/platforms/test_tpchavoc_datafusion_equivalence.py \
	tests/integration/platforms/test_datafusion_tbl_load.py \
	-m "slow" -n 0 --tb=short -v

	clickhouse-integration:
	name: clickhouse-integration
	needs: ci-paths
	if: ${{ needs.ci-paths.outputs.needs-code-ci == 'true' }}
	runs-on: ubuntu-latest
	timeout-minutes: 15
	# Non-blocking: this is the FOURTH-engine equivalence SAMPLE, not a gate.
	# The DuckDB SQL gate (correctness-gate job) stays the hard blocker.
	# ClickHouse runs in-process via chDB (clickhouse-local), no service container,
	# and is the first sampled engine on a NATIVE non-Postgres SQLGlot dialect, so
	# this samples translation-induced divergence through a different seam path.
	# Unlike the other three engines it is NOT systematic-zero - it surfaces real
	# engine-semantic result differences classified in CLICKHOUSE_KNOWN_DIVERGENCES.
	# continue-on-error keeps it a sample, not a blocker.
	continue-on-error: true
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Set up Python 3.12
	uses: actions/setup-python@v5
	with:
	python-version: '3.12'

	- name: Install uv
	uses: astral-sh/setup-uv@v4

	- name: Install dependencies
	run: uv sync --group dev

	# Fourth-engine equivalence SAMPLE. Both canonical and variants are
	# translated to the NATIVE clickhouse dialect and compared on the same
	# in-process chDB instance (shared translation cancels out);
	# CLICKHOUSE_TPCHAVOC_SKIPS variants are excluded, never marked equivalent,
	# and the residue is classified against CLICKHOUSE_KNOWN_DIVERGENCES. The
	# live sweep is tpchavoc + slow-marked; `-m "tpchavoc"` with a pinned path
	# selects it (overriding the default fast-lane deselect).
	- name: Run TPC-Havoc ClickHouse variant-equivalence sample
	run: \|
	uv run -- python -m pytest \
	tests/integration/platforms/test_tpchavoc_clickhouse_equivalence.py \
	-m "tpchavoc" -n 0 --tb=short -v

	explorer-tokens:
	name: explorer-tokens
	needs: ci-paths
	if: ${{ needs.ci-paths.outputs.needs-code-ci == 'true' }}
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Fetch base branch
	run: git fetch --no-tags origin "${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}"

	- name: Detect results-explorer/src changes
	id: scope
	shell: bash
	run: \|
	if git diff --name-only "origin/${{ github.base_ref }}...HEAD" \
	\| grep -qE '^results-explorer/src/'; then
	echo "run=true" >> "$GITHUB_OUTPUT"
	else
	echo "run=false" >> "$GITHUB_OUTPUT"
	echo "No results-explorer/src paths changed; skipping token scan."
	fi

	- name: Run explorer token scan
	if: ${{ steps.scope.outputs.run == 'true' }}
	# Bare python is intentional: the scan is stdlib-only so this gate
	# does not pay a dependency-sync cost before deciding.
	run: make lint-explorer-tokens

	- name: Run public site theme token scan
	# Bare python is intentional: the scan is stdlib-only so this gate
	# does not pay a dependency-sync cost before deciding.
	run: make lint-site-theme-tokens

	audit-sha:
	name: audit-sha
	needs: ci-paths
	if: ${{ contains(needs.ci-paths.outputs.changed-paths, '_project/audits/') \|\| contains(needs.ci-paths.outputs.changed-paths, 'results-explorer/e2e/captures/') }}
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Fetch base branch
	env:
	BASE_REF: ${{ github.base_ref }}
	run: git fetch --no-tags origin "$BASE_REF:refs/remotes/origin/$BASE_REF"

	- name: Install uv
	uses: astral-sh/setup-uv@v4

	- name: Validate changed audit develop SHA stamps
	shell: bash
	run: \|
	set -euo pipefail
	git diff --name-only --diff-filter=ACMRT "origin/${{ github.base_ref }}...HEAD" \
	\| grep -E '^_project/audits/.*\.md$' > /tmp/changed-audits.txt \|\| true
	if [ ! -s /tmp/changed-audits.txt ]; then
	echo "No changed audit Markdown files to validate."
	exit 0
	fi
	while IFS= read -r audit_path; do
	make audit-sha-check FILE="$audit_path" AUDIT_SHA_TARGET_REF="origin/${{ github.base_ref }}"
	done < /tmp/changed-audits.txt

	ci-required-result:
	name: ci-required-result
	needs: [ci-paths, content-guard, code-lint, code-test, correctness-gate, explorer-tokens, audit-sha]
	if: always()
	runs-on: ubuntu-latest
	steps:
	- name: Aggregate required result
	env:
	CI_PATHS_RESULT: ${{ needs.ci-paths.result }}
	CONTENT_RESULT: ${{ needs.content-guard.result }}
	LINT_RESULT: ${{ needs.code-lint.result }}
	TEST_RESULT: ${{ needs.code-test.result }}
	CORRECTNESS_RESULT: ${{ needs.correctness-gate.result }}
	EXPLORER_TOKENS_RESULT: ${{ needs.explorer-tokens.result }}
	AUDIT_SHA_RESULT: ${{ needs.audit-sha.result }}
	CONTENT_GUARD_NEEDED: ${{ needs.ci-paths.outputs.content-guard-needed }}
	NEEDS_CODE_CI: ${{ needs.ci-paths.outputs.needs-code-ci }}
	SAFE_CONTENT_ONLY: ${{ needs.ci-paths.outputs.safe-content-only }}
	run: \|
	set -euo pipefail

	if [ "$CI_PATHS_RESULT" != "success" ]; then
	echo "ci-paths=$CI_PATHS_RESULT"
	exit 1
	fi

	if [ "$CONTENT_GUARD_NEEDED" = "true" ] && [ "$CONTENT_RESULT" != "success" ]; then
	echo "content-guard=$CONTENT_RESULT"
	exit 1
	fi

	if [ "$AUDIT_SHA_RESULT" != "success" ] \
	&& [ "$AUDIT_SHA_RESULT" != "skipped" ]; then
	echo "audit-sha=$AUDIT_SHA_RESULT"
	exit 1
	fi

	if [ "$NEEDS_CODE_CI" = "false" ]; then
	echo "Content-only PR; skipped Python code CI."
	echo "safe-content-only=$SAFE_CONTENT_ONLY"
	exit 0
	fi

	# explorer-tokens is gated on needs-code-ci; when it runs it must
	# pass. A "skipped" result (no results-explorer/src changes within
	# the code-CI tier) counts as success here.
	if [ "$EXPLORER_TOKENS_RESULT" != "success" ] \
	&& [ "$EXPLORER_TOKENS_RESULT" != "skipped" ]; then
	echo "explorer-tokens=$EXPLORER_TOKENS_RESULT"
	exit 1
	fi

	if [ "$LINT_RESULT" = "success" ] \
	&& [ "$TEST_RESULT" = "success" ] \
	&& [ "$CORRECTNESS_RESULT" = "success" ]; then
	echo "Code/infra PR; lint, fast tests, and correctness gate passed."
	exit 0
	fi

	echo "code-lint=$LINT_RESULT code-test=$TEST_RESULT correctness-gate=$CORRECTNESS_RESULT"
	exit 1

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

docs(correctness): pin joinorder_synthetic cross-surface dtype root cause + fix layer #932

Workflow file

docs(correctness): pin joinorder_synthetic cross-surface dtype root cause + fix layer #932

Uh oh!

Workflow file for this run