diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 26f4dc44..9fd3311f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,3 +2,7 @@ /evals/docs/ @supabase/docs /apps/web/src/data/docs-eval-results.json @supabase/docs + +/evals/cli/ @supabase/cli +/experiments/cli/ @supabase/cli +/apps/web/src/data/cli-eval-results.json @supabase/cli diff --git a/.github/workflows/append-gh-pages-history.yml b/.github/workflows/append-gh-pages-history.yml index 8061dab3..04e96e15 100644 --- a/.github/workflows/append-gh-pages-history.yml +++ b/.github/workflows/append-gh-pages-history.yml @@ -7,6 +7,7 @@ on: - apps/web/src/data/eval-results.json - apps/web/src/data/regression-eval-results.json - apps/web/src/data/docs-eval-results.json + - apps/web/src/data/cli-eval-results.json workflow_dispatch: permissions: @@ -58,8 +59,9 @@ jobs: append_if_changed apps/web/src/data/eval-results.json results.jsonl append_if_changed apps/web/src/data/regression-eval-results.json regression-results.jsonl append_if_changed apps/web/src/data/docs-eval-results.json docs-results.jsonl + append_if_changed apps/web/src/data/cli-eval-results.json cli-results.jsonl - for history in results.jsonl regression-results.jsonl docs-results.jsonl; do + for history in results.jsonl regression-results.jsonl docs-results.jsonl cli-results.jsonl; do [ -f "$history" ] && git add "$history" done if git diff --cached --quiet; then diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 0d072e91..e7520f4c 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -41,6 +41,14 @@ on: type: boolean required: false default: false + cli_stable_version: + description: "Pin the cli suite's stable CLI version instead of resolving npm's latest dist-tag (blank to resolve)" + required: false + default: "" + cli_beta_version: + description: "Pin the cli suite's beta CLI version instead of resolving npm's beta dist-tag (blank to resolve)" + required: false + default: "" schedule: - cron: '15 6 * * *' pull_request: @@ -80,6 +88,8 @@ jobs: filter_changed: ${{ steps.inputs.outputs.filter_changed }} do_merge: ${{ steps.inputs.outputs.do_merge }} suite: ${{ steps.inputs.outputs.suite }} + cli_stable_version: ${{ steps.inputs.outputs.cli_stable_version }} + cli_beta_version: ${{ steps.inputs.outputs.cli_beta_version }} steps: - name: Prepare inputs id: inputs @@ -95,22 +105,28 @@ jobs: runs="${{ inputs.runs }}" timeout_sec="${{ inputs.timeout_sec }}" sandbox_concurrency="${{ inputs.sandbox_concurrency }}" + cli_stable_version="${{ inputs.cli_stable_version }}" + cli_beta_version="${{ inputs.cli_beta_version }}" elif [ "${{ github.event_name }}" = "schedule" ]; then experiments_override="" eval_id="" - suite="regression" - experiment_suite="regression" + suite="regression,cli" + experiment_suite="regression,cli" runs="3" timeout_sec="720" sandbox_concurrency="250" + cli_stable_version="" + cli_beta_version="" else experiments_override="" eval_id="" - suite="benchmark,regression,docs" - experiment_suite="benchmark,no-skills,regression,docs" + suite="benchmark,regression,docs,cli" + experiment_suite="benchmark,no-skills,regression,docs,cli" runs="3" timeout_sec="720" sandbox_concurrency="250" + cli_stable_version="" + cli_beta_version="" fi suite_json="$(jq -Rc 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))' <<< "$suite")" @@ -142,6 +158,8 @@ jobs: echo "sandbox_concurrency=$sandbox_concurrency" echo "filter_changed=$filter_changed" echo "do_merge=$do_merge" + echo "cli_stable_version=$cli_stable_version" + echo "cli_beta_version=$cli_beta_version" } >> "$GITHUB_OUTPUT" - name: Checkout @@ -224,6 +242,7 @@ jobs: benchmark) experiment_suites=(benchmark no-skills) ;; regression) experiment_suites=(regression) ;; docs) experiment_suites=(docs) ;; + cli) experiment_suites=(cli) ;; *) continue ;; esac @@ -273,6 +292,8 @@ jobs: VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} + SUPABASE_CLI_STABLE_VERSION: ${{ needs.prepare.outputs.cli_stable_version }} + SUPABASE_CLI_BETA_VERSION: ${{ needs.prepare.outputs.cli_beta_version }} steps: - name: Checkout uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 @@ -301,6 +322,8 @@ jobs: echo "OPENAI_API_KEY=${OPENAI_API_KEY}" echo "AI_GATEWAY_API_KEY=${AI_GATEWAY_API_KEY}" echo "XAI_API_KEY=${XAI_API_KEY}" + echo "SUPABASE_CLI_STABLE_VERSION=${SUPABASE_CLI_STABLE_VERSION}" + echo "SUPABASE_CLI_BETA_VERSION=${SUPABASE_CLI_BETA_VERSION}" } > .env - name: Run evals @@ -466,7 +489,7 @@ jobs: pnpm --filter @supabase-evals/framework export-results -- "${export_args[@]}" fi - for eval_suite in regression docs; do + for eval_suite in regression docs cli; do if jq -e --arg s "$eval_suite" 'any(.[]; .eval_suite == $s)' <<< "$pairs" > /dev/null; then export_args=(--suite "$eval_suite" --runs "${{ needs.prepare.outputs.runs }}" --output "apps/web/src/data/${eval_suite}-eval-results.json") if [ "${{ needs.prepare.outputs.do_merge }}" = "true" ]; then diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2964ab1d..3d8739fe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,6 +9,7 @@ First, determine the eval suite for your scenario: - **Regression** evals are suitable for most scenarios. If we notice agents make a narrow mistake, we track it here to reproduce the issue, verify a fix, and monitor for regression. These scenarios are not included in the benchmark so they don't inflate scores. - **Benchmark** evals are scenarios we've intentionally selected for the published benchmark report. These should be representative of the user journey on Supabase to cover a breadth of dimensions. - **Docs** evals are owned by docs team for their own analysis of how agents interpret docs pages, refreshed as-needed. +- **CLI** evals are owned by the CLI team: one scenario run unchanged across forced CLI environments (Docker available / daemon unreachable / absent; pinned, stable and beta CLI) via the `cli` experiment suite. Then add a folder under `evals//` containing: @@ -68,3 +69,19 @@ Common workflows: - **Add or change a docs eval.** Add the scenario under `evals/docs//` (see [Adding an eval](#adding-an-eval)), open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview. - **Refresh every docs eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: docs` and `experiment_suite: docs`. It opens a draft PR with the updated `docs-eval-results.json` for you to review and merge. - **Analyze results over time.** Every merge that changes `docs-eval-results.json` appends a snapshot to [`docs-results.jsonl`](https://supabase.github.io/evals/docs-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl) and [regression](https://supabase.github.io/evals/regression-results.jsonl) histories. + +## CLI evals + +The CLI team owns `evals/cli/` and its results. CLI evals run on `codex-gpt-6-luna-cli-{pinned,stable,beta,nodaemon,absent}` under `experiments/cli/`: `pinned` runs the repo's pinned CLI version, `stable`/`beta` install the latest stable or beta CLI, and `nodaemon`/`absent` additionally force Docker-less sandboxes — comparing the same scenario across CLI environments. + +Which evals each arm picks up: + +- **pinned, stable, beta** run every `interface: cli` eval that isn't `hostedProject: true`. +- **nodaemon, absent** additionally only run evals that also set `needsDocker: false` and `projectRunning: false` — the harness cannot pre-start a stack, or link a hosted project, without Docker. + +Common workflows: + +- **Add or change a CLI eval.** Add the scenario under `evals/cli//` (see [Adding an eval](#adding-an-eval)); set `needsDocker: false` in its `PROMPT.md` frontmatter if it can run without Docker, open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview. +- **Refresh every CLI eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: cli` and `experiment_suite: cli`. It opens a draft PR with the updated `cli-eval-results.json` for you to review and merge. Leave `cli_stable_version`/`cli_beta_version` blank to resolve npm's latest dist-tags, or pin them to reproduce a specific run. +- **Analyze results over time.** Every merge that changes `cli-eval-results.json` appends a snapshot to [`cli-results.jsonl`](https://supabase.github.io/evals/cli-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl), [regression](https://supabase.github.io/evals/regression-results.jsonl), and [docs](https://supabase.github.io/evals/docs-results.jsonl) histories. +- **Run the unit tests.** `pnpm --filter @supabase-evals/framework test:cli-lib` (the CLI skip predicates in `experiments/cli/lib/` plus every CLI eval's scorer tests) — also part of `pnpm check`. diff --git a/README.md b/README.md index b1ee8e53..1bc7f0ed 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Agent-backed runs require the relevant provider key in `.env` (e.g. `OPENAI_API_ - An **eval** is one scenario under `evals///`. It contains the prompt, scorer, and optional starting state for the two environments: `remote/` (the hosted project) and `local/` (the agent's working files). - An **experiment** is one agent/runtime/model setup under `experiments//.experiment.ts`. Its ID remains ``. -- An **eval suite** is a named set of evals to run together. An eval's suite is its parent folder under `evals/` (`benchmark`, `regression`, `docs`, or `other`). +- An **eval suite** is a named set of evals to run together. An eval's suite is its parent folder under `evals/` (`benchmark`, `regression`, `docs`, `cli`, or `other`). - An **experiment suite** is a named set of experiments with related configurations, for head to head comparisons. - An **agent** is the model driver that receives the eval prompt and calls the configured tools. - A **runtime** is the local Supabase-like environment and tool surface an experiment gives to the agent. diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index 14cb6b9e..43e74318 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -369,6 +369,7 @@ async function runOne( stepCount?: number; toolCallCount: number; agentRunDurationMs: number; + cliVersion?: string; } > { const prompt = parseEvalMarkdown( @@ -479,6 +480,10 @@ async function runOne( // Runs after scoring so the scorer sees what the agent actually saw, not rehydrated content. await rehydrateTruncatedDocsResults(session.sandbox, run.toolCalls); + // The marker names the CLI binary the sandbox actually staged; falls back to + // the frontmatter pin only when there's no marker to read (e.g. skipCliInstall). + const marker = await session.scoringContext.environmentMarker(); + return { ...last, run: runIndex, @@ -493,6 +498,7 @@ async function runOne( stepCount: run.stepCount, toolCallCount: run.toolCalls.length, agentRunDurationMs: run.durationMs, + cliVersion: marker?.cliVersion ?? ev.metadata.cliVersion, }; } @@ -556,6 +562,8 @@ async function runOne( stepCount: run.stepCount, toolCallCount: run.toolCalls.length, agentRunDurationMs: run.durationMs, + // No sandbox in tools mode, so no marker to read; only the frontmatter pin applies. + cliVersion: ev.metadata.cliVersion, }; } diff --git a/apps/framework/package.json b/apps/framework/package.json index 770ad8ac..57307260 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -4,7 +4,7 @@ "version": "0.0.1", "type": "module", "scripts": { - "check": "pnpm typecheck && pnpm test && pnpm test:framework && pnpm test:vercel-runner", + "check": "pnpm typecheck && pnpm test && pnpm test:framework && pnpm test:vercel-runner && pnpm test:cli-lib", "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts", "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry", "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke", @@ -12,7 +12,8 @@ "typecheck": "tsc --noEmit", "test": "vitest run harness", "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", - "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts lib/cli-args.test.ts lib/experiment-files.test.ts lib/sample-sets.test.ts", + "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts scripts/export-results.test.ts lib/cli-args.test.ts lib/experiment-files.test.ts lib/sample-sets.test.ts", + "test:cli-lib": "vitest run --root ../.. --passWithNoTests experiments/cli evals/cli", "export-results": "node --import tsx/esm scripts/export-results.ts", "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts", "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts" diff --git a/apps/framework/scripts/export-results.test.ts b/apps/framework/scripts/export-results.test.ts new file mode 100644 index 00000000..5b285f6f --- /dev/null +++ b/apps/framework/scripts/export-results.test.ts @@ -0,0 +1,64 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import type { ExperimentExportMetadata } from './export-results.js'; +import { readResultFile } from './export-results.js'; + +// build-docs-010-edge-function-auth is the one eval whose PROMPT.md pins +// `cliVersion: 2.109.1`, needed here to exercise the frontmatter side of the +// precedence. +const PINNED_EVAL_ID = 'build-docs-010-edge-function-auth'; + +describe('readResultFile cliVersion precedence', () => { + const experimentMetadata = new Map(); + + const writeRawResult = (dir: string, raw: Record) => { + const filePath = join(dir, 'result.json'); + writeFileSync(filePath, JSON.stringify(raw)); + return filePath; + }; + + it('carries the run value that actually ran over the frontmatter pin', async () => { + const temporary = mkdtempSync(join(tmpdir(), 'export-results-test-')); + try { + const filePath = writeRawResult(temporary, { + experiment: 'test-experiment', + eval: PINNED_EVAL_ID, + interface: 'cli', + cliVersion: '2.118.0-beta.60', + }); + + const result = await readResultFile( + filePath, + 'test-experiment/run-1/result.json', + experimentMetadata + ); + + expect(result?.cliVersion).toBe('2.118.0-beta.60'); + } finally { + rmSync(temporary, { recursive: true, force: true }); + } + }); + + it('falls back to the frontmatter pin when the run recorded no version', async () => { + const temporary = mkdtempSync(join(tmpdir(), 'export-results-test-')); + try { + const filePath = writeRawResult(temporary, { + experiment: 'test-experiment', + eval: PINNED_EVAL_ID, + interface: 'cli', + }); + + const result = await readResultFile( + filePath, + 'test-experiment/run-1/result.json', + experimentMetadata + ); + + expect(result?.cliVersion).toBe('2.109.1'); + } finally { + rmSync(temporary, { recursive: true, force: true }); + } + }); +}); diff --git a/apps/framework/scripts/export-results.ts b/apps/framework/scripts/export-results.ts index 50e8912f..3a69e4c1 100644 --- a/apps/framework/scripts/export-results.ts +++ b/apps/framework/scripts/export-results.ts @@ -47,7 +47,7 @@ const OUTPUT_PATH = join( 'eval-results.json' ); -type ExperimentExportMetadata = { +export type ExperimentExportMetadata = { display: ExperimentDisplayMetadata; experimentSuite?: ExperimentSuite; }; @@ -122,7 +122,7 @@ async function readPrompt(evalId: string) { }; } -async function readResultFile( +export async function readResultFile( filePath: string, sourcePath: string, experimentMetadata: Map @@ -152,7 +152,9 @@ async function readResultFile( topic: promptData?.topic ?? parsedResult.topic, suite: promptData?.suite ?? parsedResult.suite, interface: promptData?.interface ?? parsedResult.interface, - cliVersion: promptData?.cliVersion ?? parsedResult.cliVersion, + // The run's recorded version (the binary that actually ran) wins over the + // frontmatter pin, which only names what the eval requested. + cliVersion: parsedResult.cliVersion ?? promptData?.cliVersion, passed: parsedResult.passed === true, checks: parsedResult.checks, skills: parsedResult.skills, diff --git a/apps/framework/scripts/run-vercel-evals.test.ts b/apps/framework/scripts/run-vercel-evals.test.ts index ab4cec9a..d4c9f41d 100644 --- a/apps/framework/scripts/run-vercel-evals.test.ts +++ b/apps/framework/scripts/run-vercel-evals.test.ts @@ -178,6 +178,16 @@ describe('resolveChannelPins', () => { resolveChannelPins(new Set(['stable'])) ).rejects.toThrow('npm unreachable'); }); + + it('treats a blank or whitespace-only env var as unset, resolving it instead', async () => { + process.env[STABLE_ENV] = ' '; + vi.mocked(resolveCliVersion).mockImplementation(async () => '2.117.0'); + + const pins = await resolveChannelPins(new Set(['stable'])); + + expect(pins[STABLE_ENV]).toBe('2.117.0'); + expect(resolveCliVersion).toHaveBeenCalledWith('stable'); + }); }); describe('requiredCliChannels', () => { diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index 8de4ba29..3b0f2936 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -99,8 +99,10 @@ export async function resolveChannelPins( const resolved = await Promise.all( [...channels].map(async (channel) => { const envVar = CLI_CHANNEL_ENV[channel]; - const override = process.env[envVar]; - return [envVar, override ?? (await resolveCliVersion(channel))] as const; + // A workflow that exports a blank input still sets the env var, so + // blank/whitespace must be treated as unset rather than as a pin of ''. + const override = process.env[envVar]?.trim(); + return [envVar, override || (await resolveCliVersion(channel))] as const; }) ); return Object.fromEntries(resolved); diff --git a/apps/framework/tsconfig.json b/apps/framework/tsconfig.json index 46ea7676..6c6f48e7 100644 --- a/apps/framework/tsconfig.json +++ b/apps/framework/tsconfig.json @@ -6,5 +6,6 @@ "shims", "scripts", "../../evals/**/EVAL.ts" - ] + ], + "exclude": ["../../experiments/**/*.test.ts"] } diff --git a/experiments/cli/codex-gpt-6-luna-cli-absent.experiment.ts b/experiments/cli/codex-gpt-6-luna-cli-absent.experiment.ts new file mode 100644 index 00000000..fc3a01d3 --- /dev/null +++ b/experiments/cli/codex-gpt-6-luna-cli-absent.experiment.ts @@ -0,0 +1,12 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; +import { codexGpt6Luna } from '../presets.js'; +import { skipUnlessDockerless } from './lib/skip.js'; + +export default defineExperiment({ + ...codexGpt6Luna, + suite: ['cli'], + // beta: the Docker-less path only exists in the managed stack, which ships in beta. + localStack: localStackRuntime({ cliVersion: 'beta', docker: 'absent' }), + skipEval: skipUnlessDockerless, +}); diff --git a/experiments/cli/codex-gpt-6-luna-cli-beta.experiment.ts b/experiments/cli/codex-gpt-6-luna-cli-beta.experiment.ts new file mode 100644 index 00000000..86338628 --- /dev/null +++ b/experiments/cli/codex-gpt-6-luna-cli-beta.experiment.ts @@ -0,0 +1,11 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; +import { codexGpt6Luna } from '../presets.js'; +import { skipUnlessCli } from './lib/skip.js'; + +export default defineExperiment({ + ...codexGpt6Luna, + suite: ['cli'], + localStack: localStackRuntime({ cliVersion: 'beta' }), + skipEval: skipUnlessCli, +}); diff --git a/experiments/cli/codex-gpt-6-luna-cli-nodaemon.experiment.ts b/experiments/cli/codex-gpt-6-luna-cli-nodaemon.experiment.ts new file mode 100644 index 00000000..c5d8166e --- /dev/null +++ b/experiments/cli/codex-gpt-6-luna-cli-nodaemon.experiment.ts @@ -0,0 +1,12 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; +import { codexGpt6Luna } from '../presets.js'; +import { skipUnlessDockerless } from './lib/skip.js'; + +export default defineExperiment({ + ...codexGpt6Luna, + suite: ['cli'], + // beta: the Docker-less path only exists in the managed stack, which ships in beta. + localStack: localStackRuntime({ cliVersion: 'beta', docker: 'no-daemon' }), + skipEval: skipUnlessDockerless, +}); diff --git a/experiments/cli/codex-gpt-6-luna-cli-pinned.experiment.ts b/experiments/cli/codex-gpt-6-luna-cli-pinned.experiment.ts new file mode 100644 index 00000000..5de01107 --- /dev/null +++ b/experiments/cli/codex-gpt-6-luna-cli-pinned.experiment.ts @@ -0,0 +1,9 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { codexGpt6Luna } from '../presets.js'; +import { skipUnlessCli } from './lib/skip.js'; + +export default defineExperiment({ + ...codexGpt6Luna, + suite: ['cli'], + skipEval: skipUnlessCli, +}); diff --git a/experiments/cli/codex-gpt-6-luna-cli-stable.experiment.ts b/experiments/cli/codex-gpt-6-luna-cli-stable.experiment.ts new file mode 100644 index 00000000..fe68bc7c --- /dev/null +++ b/experiments/cli/codex-gpt-6-luna-cli-stable.experiment.ts @@ -0,0 +1,13 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; +import { codexGpt6Luna } from '../presets.js'; +import { skipUnlessCli } from './lib/skip.js'; + +export default defineExperiment({ + ...codexGpt6Luna, + suite: ['cli'], + // Currently equal to the pin (npm `latest` == SUPABASE_CLI_VERSION); kept + // as drift insurance between pin bumps. + localStack: localStackRuntime({ cliVersion: 'stable' }), + skipEval: skipUnlessCli, +}); diff --git a/experiments/cli/lib/skip.test.ts b/experiments/cli/lib/skip.test.ts new file mode 100644 index 00000000..b9e1dc13 --- /dev/null +++ b/experiments/cli/lib/skip.test.ts @@ -0,0 +1,111 @@ +import type { EvalMetadata } from '@supabase-evals/core'; +import { describe, expect, it } from 'vitest'; +import { skipUnlessCli, skipUnlessDockerless } from './skip.js'; + +const baseMetadata: EvalMetadata = { + stage: 'build', + product: ['database'], + topic: ['sdk'], + interface: 'cli', +}; + +describe('skipUnlessCli', () => { + it('runs a cli eval that is not hosted-linked', () => { + expect(skipUnlessCli({ id: 'e', metadata: baseMetadata })).toBe(false); + }); + + it('skips a non-cli eval', () => { + expect( + skipUnlessCli({ + id: 'e', + metadata: { ...baseMetadata, interface: 'mcp' }, + }) + ).toBe(true); + }); + + it('skips a hosted-linked eval', () => { + expect( + skipUnlessCli({ + id: 'e', + metadata: { ...baseMetadata, hostedProject: true }, + }) + ).toBe(true); + }); + + it('runs a cli eval with hostedProject explicitly false', () => { + expect( + skipUnlessCli({ + id: 'e', + metadata: { ...baseMetadata, hostedProject: false }, + }) + ).toBe(false); + }); +}); + +describe('skipUnlessDockerless', () => { + const dockerlessMetadata: EvalMetadata = { + ...baseMetadata, + needsDocker: false, + projectRunning: false, + }; + + it('runs a cli eval that needs no Docker and has no pre-started stack', () => { + expect( + skipUnlessDockerless({ id: 'e', metadata: dockerlessMetadata }) + ).toBe(false); + }); + + it('skips a non-cli eval', () => { + expect( + skipUnlessDockerless({ + id: 'e', + metadata: { ...dockerlessMetadata, interface: 'mcp' }, + }) + ).toBe(true); + }); + + it('skips a hosted-linked eval', () => { + expect( + skipUnlessDockerless({ + id: 'e', + metadata: { ...dockerlessMetadata, hostedProject: true }, + }) + ).toBe(true); + }); + + it('skips an eval that needs Docker', () => { + expect( + skipUnlessDockerless({ + id: 'e', + metadata: { ...dockerlessMetadata, needsDocker: true }, + }) + ).toBe(true); + }); + + it('skips an eval whose needsDocker is unset (defaults to true)', () => { + expect( + skipUnlessDockerless({ + id: 'e', + metadata: { ...baseMetadata, projectRunning: false }, + }) + ).toBe(true); + }); + + it('skips an eval whose stack is already running', () => { + expect( + skipUnlessDockerless({ + id: 'e', + metadata: { ...dockerlessMetadata, projectRunning: true }, + }) + ).toBe(true); + }); + + it('skips an eval whose projectRunning is unset (defaults to true)', () => { + expect( + skipUnlessDockerless({ + id: 'e', + metadata: { ...baseMetadata, needsDocker: false }, + }) + ).toBe(true); + }); +}); diff --git a/experiments/cli/lib/skip.ts b/experiments/cli/lib/skip.ts new file mode 100644 index 00000000..704f1f6a --- /dev/null +++ b/experiments/cli/lib/skip.ts @@ -0,0 +1,21 @@ +import type { EvalMetadata } from '@supabase-evals/core'; + +/** Runs only evals that exercise the real CLI (never hosted-linked ones, which seed .temp pins instead). */ +export function skipUnlessCli(ev: { + id: string; + metadata: EvalMetadata; +}): boolean { + return ev.metadata.interface !== 'cli' || ev.metadata.hostedProject === true; +} + +/** A Docker-less sandbox can only run evals that declare they need no Docker and don't expect a pre-started stack. */ +export function skipUnlessDockerless(ev: { + id: string; + metadata: EvalMetadata; +}): boolean { + return ( + skipUnlessCli(ev) || + ev.metadata.needsDocker !== false || + ev.metadata.projectRunning !== false + ); +} diff --git a/packages/core/src/eval-metadata.test.ts b/packages/core/src/eval-metadata.test.ts new file mode 100644 index 00000000..f88163ba --- /dev/null +++ b/packages/core/src/eval-metadata.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from 'vitest'; +import { evalMetadataSchema, rawEvalResultSchema } from './eval-metadata.js'; + +describe('cliVersion schema', () => { + const BETA_VERSION = '2.118.0-beta.60'; + + it('round-trips a resolved beta version through the frontmatter metadata schema', () => { + const parsed = evalMetadataSchema.parse({ + stage: 'build', + product: ['database'], + topic: ['sql'], + interface: 'cli', + cliVersion: BETA_VERSION, + }); + + expect(parsed.cliVersion).toBe(BETA_VERSION); + }); + + it('round-trips a resolved beta version through the result schema', () => { + const parsed = rawEvalResultSchema.parse({ + experiment: 'test-experiment', + eval: 'test-eval', + interface: 'cli', + cliVersion: BETA_VERSION, + }); + + expect(parsed.cliVersion).toBe(BETA_VERSION); + }); + + it('still rejects a non-version string', () => { + expect(() => + evalMetadataSchema.parse({ + stage: 'build', + product: ['database'], + topic: ['sql'], + interface: 'cli', + cliVersion: 'stable', + }) + ).toThrow(); + }); +}); diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index fa699c88..d3078052 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -44,6 +44,7 @@ export const evalSuiteSchema = z.enum([ 'benchmark', 'regression', 'docs', + 'cli', 'other', ]); export const EVAL_SUITES = evalSuiteSchema.options; @@ -54,6 +55,7 @@ export const experimentSuiteSchema = z.enum([ 'no-skills', 'regression', 'docs', + 'cli', ]); export const EXPERIMENT_SUITES = experimentSuiteSchema.options; export type ExperimentSuite = z.infer; @@ -109,7 +111,9 @@ export type ExperimentDisplayMetadata = z.infer< export const evalInterfaceSchema = z.enum(['mcp', 'cli']); export const EVAL_INTERFACES = evalInterfaceSchema.options; export type EvalInterface = z.infer; -const cliVersionSchema = z.string().regex(/^\d+\.\d+\.\d+$/); +// Matches VERSION_RE in packages/sandbox/src/cli-channel.ts, so a resolved +// beta version (e.g. "2.118.0-beta.60") round-trips through this schema too. +const cliVersionSchema = z.string().regex(/^\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?$/); export type EvalMetadata = { stage: EvalStage;