feat(cli): add the cli eval suite and experiment suite plumbing

.github/CODEOWNERS

@@ -2,3 +2,7 @@
 
 /evals/docs/ @supabase/docs
 /apps/web/src/data/docs-eval-results.json @supabase/docs
+
+/evals/cli/ @supabase/cli
+/experiments/cli/ @supabase/cli
+/apps/web/src/data/cli-eval-results.json @supabase/cli

.github/workflows/append-gh-pages-history.yml

@@ -7,6 +7,7 @@ on:
       - apps/web/src/data/eval-results.json
       - apps/web/src/data/regression-eval-results.json
       - apps/web/src/data/docs-eval-results.json
+      - apps/web/src/data/cli-eval-results.json
   workflow_dispatch:
 
 permissions:
@@ -58,8 +59,9 @@ jobs:
           append_if_changed apps/web/src/data/eval-results.json results.jsonl
           append_if_changed apps/web/src/data/regression-eval-results.json regression-results.jsonl
           append_if_changed apps/web/src/data/docs-eval-results.json docs-results.jsonl
+          append_if_changed apps/web/src/data/cli-eval-results.json cli-results.jsonl
 
-          for history in results.jsonl regression-results.jsonl docs-results.jsonl; do
+          for history in results.jsonl regression-results.jsonl docs-results.jsonl cli-results.jsonl; do
             [ -f "$history" ] && git add "$history"
           done
           if git diff --cached --quiet; then

.github/workflows/eval-refresh.yml

@@ -41,6 +41,14 @@ on:
         type: boolean
         required: false
         default: false
+      cli_stable_version:
+        description: "Pin the cli suite's stable CLI version instead of resolving npm's latest dist-tag (blank to resolve)"
+        required: false
+        default: ""
+      cli_beta_version:
+        description: "Pin the cli suite's beta CLI version instead of resolving npm's beta dist-tag (blank to resolve)"
+        required: false
+        default: ""
   schedule:
     - cron: '15 6 * * *'
   pull_request:
@@ -80,6 +88,8 @@ jobs:
       filter_changed: ${{ steps.inputs.outputs.filter_changed }}
       do_merge: ${{ steps.inputs.outputs.do_merge }}
       suite: ${{ steps.inputs.outputs.suite }}
+      cli_stable_version: ${{ steps.inputs.outputs.cli_stable_version }}
+      cli_beta_version: ${{ steps.inputs.outputs.cli_beta_version }}
     steps:
       - name: Prepare inputs
         id: inputs
@@ -95,22 +105,28 @@ jobs:
             runs="${{ inputs.runs }}"
             timeout_sec="${{ inputs.timeout_sec }}"
             sandbox_concurrency="${{ inputs.sandbox_concurrency }}"
+            cli_stable_version="${{ inputs.cli_stable_version }}"
+            cli_beta_version="${{ inputs.cli_beta_version }}"
           elif [ "${{ github.event_name }}" = "schedule" ]; then
             experiments_override=""
             eval_id=""
-            suite="regression"
-            experiment_suite="regression"
+            suite="regression,cli"
+            experiment_suite="regression,cli"
             runs="3"
             timeout_sec="720"
             sandbox_concurrency="250"
+            cli_stable_version=""
+            cli_beta_version=""
           else
             experiments_override=""
             eval_id=""
-            suite="benchmark,regression,docs"
-            experiment_suite="benchmark,no-skills,regression,docs"
+            suite="benchmark,regression,docs,cli"
+            experiment_suite="benchmark,no-skills,regression,docs,cli"
             runs="3"
             timeout_sec="720"
             sandbox_concurrency="250"
+            cli_stable_version=""
+            cli_beta_version=""
           fi
 
           suite_json="$(jq -Rc 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))' <<< "$suite")"
@@ -142,6 +158,8 @@ jobs:
             echo "sandbox_concurrency=$sandbox_concurrency"
             echo "filter_changed=$filter_changed"
             echo "do_merge=$do_merge"
+            echo "cli_stable_version=$cli_stable_version"
+            echo "cli_beta_version=$cli_beta_version"
           } >> "$GITHUB_OUTPUT"
 
       - name: Checkout
@@ -224,6 +242,7 @@ jobs:
               benchmark) experiment_suites=(benchmark no-skills) ;;
               regression) experiment_suites=(regression) ;;
               docs) experiment_suites=(docs) ;;
+              cli) experiment_suites=(cli) ;;
               *) continue ;;
             esac
 
@@ -273,6 +292,8 @@ jobs:
       VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
       VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }}
       VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}
+      SUPABASE_CLI_STABLE_VERSION: ${{ needs.prepare.outputs.cli_stable_version }}
+      SUPABASE_CLI_BETA_VERSION: ${{ needs.prepare.outputs.cli_beta_version }}
     steps:
       - name: Checkout
         uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3
@@ -301,6 +322,8 @@ jobs:
             echo "OPENAI_API_KEY=${OPENAI_API_KEY}"
             echo "AI_GATEWAY_API_KEY=${AI_GATEWAY_API_KEY}"
             echo "XAI_API_KEY=${XAI_API_KEY}"
+            echo "SUPABASE_CLI_STABLE_VERSION=${SUPABASE_CLI_STABLE_VERSION}"
+            echo "SUPABASE_CLI_BETA_VERSION=${SUPABASE_CLI_BETA_VERSION}"
           } > .env
 
       - name: Run evals
@@ -466,7 +489,7 @@ jobs:
             pnpm --filter @supabase-evals/framework export-results -- "${export_args[@]}"
           fi
 
-          for eval_suite in regression docs; do
+          for eval_suite in regression docs cli; do
             if jq -e --arg s "$eval_suite" 'any(.[]; .eval_suite == $s)' <<< "$pairs" > /dev/null; then
               export_args=(--suite "$eval_suite" --runs "${{ needs.prepare.outputs.runs }}" --output "apps/web/src/data/${eval_suite}-eval-results.json")
               if [ "${{ needs.prepare.outputs.do_merge }}" = "true" ]; then

CONTRIBUTING.md

@@ -9,6 +9,7 @@ First, determine the eval suite for your scenario:
 - **Regression** evals are suitable for most scenarios. If we notice agents make a narrow mistake, we track it here to reproduce the issue, verify a fix, and monitor for regression. These scenarios are not included in the benchmark so they don't inflate scores.
 - **Benchmark** evals are scenarios we've intentionally selected for the published benchmark report. These should be representative of the user journey on Supabase to cover a breadth of dimensions.
 - **Docs** evals are owned by docs team for their own analysis of how agents interpret docs pages, refreshed as-needed.
+- **CLI** evals are owned by the CLI team: one scenario run unchanged across forced CLI environments (Docker available / daemon unreachable / absent; pinned, stable and beta CLI) via the `cli` experiment suite.
 
 Then add a folder under `evals/<suite>/` containing:
 
@@ -68,3 +69,19 @@ Common workflows:
 - **Add or change a docs eval.** Add the scenario under `evals/docs/<id>/` (see [Adding an eval](#adding-an-eval)), open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview.
 - **Refresh every docs eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: docs` and `experiment_suite: docs`. It opens a draft PR with the updated `docs-eval-results.json` for you to review and merge.
 - **Analyze results over time.** Every merge that changes `docs-eval-results.json` appends a snapshot to [`docs-results.jsonl`](https://supabase.github.io/evals/docs-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl) and [regression](https://supabase.github.io/evals/regression-results.jsonl) histories.
+
+## CLI evals
+
+The CLI team owns `evals/cli/` and its results. CLI evals run on `codex-gpt-6-luna-cli-{pinned,stable,beta,nodaemon,absent}` under `experiments/cli/`: `pinned` runs the repo's pinned CLI version, `stable`/`beta` install the latest stable or beta CLI, and `nodaemon`/`absent` additionally force Docker-less sandboxes — comparing the same scenario across CLI environments.
+
+Which evals each arm picks up:
+
+- **pinned, stable, beta** run every `interface: cli` eval that isn't `hostedProject: true`.
+- **nodaemon, absent** additionally only run evals that also set `needsDocker: false` and `projectRunning: false` — the harness cannot pre-start a stack, or link a hosted project, without Docker.
+
+Common workflows:
+
+- **Add or change a CLI eval.** Add the scenario under `evals/cli/<id>/` (see [Adding an eval](#adding-an-eval)); set `needsDocker: false` in its `PROMPT.md` frontmatter if it can run without Docker, open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview.
+- **Refresh every CLI eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: cli` and `experiment_suite: cli`. It opens a draft PR with the updated `cli-eval-results.json` for you to review and merge. Leave `cli_stable_version`/`cli_beta_version` blank to resolve npm's latest dist-tags, or pin them to reproduce a specific run.
+- **Analyze results over time.** Every merge that changes `cli-eval-results.json` appends a snapshot to [`cli-results.jsonl`](https://supabase.github.io/evals/cli-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl), [regression](https://supabase.github.io/evals/regression-results.jsonl), and [docs](https://supabase.github.io/evals/docs-results.jsonl) histories.
+- **Run the unit tests.** `pnpm --filter @supabase-evals/framework test:cli-lib` (the CLI skip predicates in `experiments/cli/lib/` plus every CLI eval's scorer tests) — also part of `pnpm check`.

README.md

@@ -29,7 +29,7 @@ Agent-backed runs require the relevant provider key in `.env` (e.g. `OPENAI_API_
 
 - An **eval** is one scenario under `evals/<suite>/<id>/`. It contains the prompt, scorer, and optional starting state for the two environments: `remote/` (the hosted project) and `local/` (the agent's working files).
 - An **experiment** is one agent/runtime/model setup under `experiments/<owner>/<name>.experiment.ts`. Its ID remains `<name>`.
-- An **eval suite** is a named set of evals to run together. An eval's suite is its parent folder under `evals/` (`benchmark`, `regression`, `docs`, or `other`).
+- An **eval suite** is a named set of evals to run together. An eval's suite is its parent folder under `evals/` (`benchmark`, `regression`, `docs`, `cli`, or `other`).
 - An **experiment suite** is a named set of experiments with related configurations, for head to head comparisons.
 - An **agent** is the model driver that receives the eval prompt and calls the configured tools.
 - A **runtime** is the local Supabase-like environment and tool surface an experiment gives to the agent.

apps/framework/harness/run-eval.ts

@@ -369,6 +369,7 @@ async function runOne(
     stepCount?: number;
     toolCallCount: number;
     agentRunDurationMs: number;
+    cliVersion?: string;
   }
 > {
   const prompt = parseEvalMarkdown(
@@ -479,6 +480,10 @@ async function runOne(
     // Runs after scoring so the scorer sees what the agent actually saw, not rehydrated content.
     await rehydrateTruncatedDocsResults(session.sandbox, run.toolCalls);
 
+    // The marker names the CLI binary the sandbox actually staged; falls back to
+    // the frontmatter pin only when there's no marker to read (e.g. skipCliInstall).
+    const marker = await session.scoringContext.environmentMarker();
+
     return {
       ...last,
       run: runIndex,
@@ -493,6 +498,7 @@ async function runOne(
       stepCount: run.stepCount,
       toolCallCount: run.toolCalls.length,
       agentRunDurationMs: run.durationMs,
+      cliVersion: marker?.cliVersion ?? ev.metadata.cliVersion,
     };
   }
 
@@ -556,6 +562,8 @@ async function runOne(
     stepCount: run.stepCount,
     toolCallCount: run.toolCalls.length,
     agentRunDurationMs: run.durationMs,
+    // No sandbox in tools mode, so no marker to read; only the frontmatter pin applies.
+    cliVersion: ev.metadata.cliVersion,
   };
 }
 

apps/framework/package.json

@@ -4,15 +4,16 @@
   "version": "0.0.1",
   "type": "module",
   "scripts": {
-    "check": "pnpm typecheck && pnpm test && pnpm test:framework && pnpm test:vercel-runner",
+    "check": "pnpm typecheck && pnpm test && pnpm test:framework && pnpm test:vercel-runner && pnpm test:cli-lib",
     "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts",
     "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry",
     "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke",
     "eval:vercel": "node --env-file=../../.env --import tsx/esm scripts/run-vercel-evals.ts",
     "typecheck": "tsc --noEmit",
     "test": "vitest run harness",
     "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts",
-    "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts lib/cli-args.test.ts lib/experiment-files.test.ts lib/sample-sets.test.ts",
+    "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts scripts/export-results.test.ts lib/cli-args.test.ts lib/experiment-files.test.ts lib/sample-sets.test.ts",
+    "test:cli-lib": "vitest run --root ../.. --passWithNoTests experiments/cli evals/cli",
     "export-results": "node --import tsx/esm scripts/export-results.ts",
     "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts",
     "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts"

apps/framework/scripts/export-results.test.ts

@@ -0,0 +1,64 @@
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { describe, expect, it } from 'vitest';
+import type { ExperimentExportMetadata } from './export-results.js';
+import { readResultFile } from './export-results.js';
+
+// build-docs-010-edge-function-auth is the one eval whose PROMPT.md pins
+// `cliVersion: 2.109.1`, needed here to exercise the frontmatter side of the
+// precedence.
+const PINNED_EVAL_ID = 'build-docs-010-edge-function-auth';
+
+describe('readResultFile cliVersion precedence', () => {
+  const experimentMetadata = new Map<string, ExperimentExportMetadata>();
+
+  const writeRawResult = (dir: string, raw: Record<string, unknown>) => {
+    const filePath = join(dir, 'result.json');
+    writeFileSync(filePath, JSON.stringify(raw));
+    return filePath;
+  };
+
+  it('carries the run value that actually ran over the frontmatter pin', async () => {
+    const temporary = mkdtempSync(join(tmpdir(), 'export-results-test-'));
+    try {
+      const filePath = writeRawResult(temporary, {
+        experiment: 'test-experiment',
+        eval: PINNED_EVAL_ID,
+        interface: 'cli',
+        cliVersion: '2.118.0-beta.60',
+      });
+
+      const result = await readResultFile(
+        filePath,
+        'test-experiment/run-1/result.json',
+        experimentMetadata
+      );
+
+      expect(result?.cliVersion).toBe('2.118.0-beta.60');
+    } finally {
+      rmSync(temporary, { recursive: true, force: true });
+    }
+  });
+
+  it('falls back to the frontmatter pin when the run recorded no version', async () => {
+    const temporary = mkdtempSync(join(tmpdir(), 'export-results-test-'));
+    try {
+      const filePath = writeRawResult(temporary, {
+        experiment: 'test-experiment',
+        eval: PINNED_EVAL_ID,
+        interface: 'cli',
+      });
+
+      const result = await readResultFile(
+        filePath,
+        'test-experiment/run-1/result.json',
+        experimentMetadata
+      );
+
+      expect(result?.cliVersion).toBe('2.109.1');
+    } finally {
+      rmSync(temporary, { recursive: true, force: true });
+    }
+  });
+});

apps/framework/scripts/export-results.ts

@@ -47,7 +47,7 @@ const OUTPUT_PATH = join(
   'eval-results.json'
 );
 
-type ExperimentExportMetadata = {
+export type ExperimentExportMetadata = {
   display: ExperimentDisplayMetadata;
   experimentSuite?: ExperimentSuite;
 };
@@ -122,7 +122,7 @@ async function readPrompt(evalId: string) {
   };
 }
 
-async function readResultFile(
+export async function readResultFile(
   filePath: string,
   sourcePath: string,
   experimentMetadata: Map<string, ExperimentExportMetadata>
@@ -152,7 +152,9 @@ async function readResultFile(
     topic: promptData?.topic ?? parsedResult.topic,
     suite: promptData?.suite ?? parsedResult.suite,
     interface: promptData?.interface ?? parsedResult.interface,
-    cliVersion: promptData?.cliVersion ?? parsedResult.cliVersion,
+    // The run's recorded version (the binary that actually ran) wins over the
+    // frontmatter pin, which only names what the eval requested.
+    cliVersion: parsedResult.cliVersion ?? promptData?.cliVersion,
     passed: parsedResult.passed === true,
     checks: parsedResult.checks,
     skills: parsedResult.skills,

apps/framework/scripts/run-vercel-evals.test.ts

@@ -178,6 +178,16 @@ describe('resolveChannelPins', () => {
       resolveChannelPins(new Set<CliChannel>(['stable']))
     ).rejects.toThrow('npm unreachable');
   });
+
+  it('treats a blank or whitespace-only env var as unset, resolving it instead', async () => {
+    process.env[STABLE_ENV] = '   ';
+    vi.mocked(resolveCliVersion).mockImplementation(async () => '2.117.0');
+
+    const pins = await resolveChannelPins(new Set<CliChannel>(['stable']));
+
+    expect(pins[STABLE_ENV]).toBe('2.117.0');
+    expect(resolveCliVersion).toHaveBeenCalledWith('stable');
+  });
 });
 
 describe('requiredCliChannels', () => {

apps/framework/scripts/run-vercel-evals.ts

@@ -99,8 +99,10 @@ export async function resolveChannelPins(
   const resolved = await Promise.all(
     [...channels].map(async (channel) => {
       const envVar = CLI_CHANNEL_ENV[channel];
-      const override = process.env[envVar];
-      return [envVar, override ?? (await resolveCliVersion(channel))] as const;
+      // A workflow that exports a blank input still sets the env var, so
+      // blank/whitespace must be treated as unset rather than as a pin of ''.
+      const override = process.env[envVar]?.trim();
+      return [envVar, override || (await resolveCliVersion(channel))] as const;
     })
   );
   return Object.fromEntries(resolved);

apps/framework/tsconfig.json

@@ -6,5 +6,6 @@
     "shims",
     "scripts",
     "../../evals/**/EVAL.ts"
-  ]
+  ],
+  "exclude": ["../../experiments/**/*.test.ts"]
 }

experiments/cli/codex-gpt-6-luna-cli-absent.experiment.ts

@@ -0,0 +1,12 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { codexGpt6Luna } from '../presets.js';
+import { skipUnlessDockerless } from './lib/skip.js';
+
+export default defineExperiment({
+  ...codexGpt6Luna,
+  suite: ['cli'],
+  // beta: the Docker-less path only exists in the managed stack, which ships in beta.
+  localStack: localStackRuntime({ cliVersion: 'beta', docker: 'absent' }),
+  skipEval: skipUnlessDockerless,
+});

experiments/cli/codex-gpt-6-luna-cli-beta.experiment.ts

@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { codexGpt6Luna } from '../presets.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...codexGpt6Luna,
+  suite: ['cli'],
+  localStack: localStackRuntime({ cliVersion: 'beta' }),
+  skipEval: skipUnlessCli,
+});

experiments/cli/codex-gpt-6-luna-cli-nodaemon.experiment.ts

@@ -0,0 +1,12 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { codexGpt6Luna } from '../presets.js';
+import { skipUnlessDockerless } from './lib/skip.js';
+
+export default defineExperiment({
+  ...codexGpt6Luna,
+  suite: ['cli'],
+  // beta: the Docker-less path only exists in the managed stack, which ships in beta.
+  localStack: localStackRuntime({ cliVersion: 'beta', docker: 'no-daemon' }),
+  skipEval: skipUnlessDockerless,
+});

experiments/cli/codex-gpt-6-luna-cli-pinned.experiment.ts

@@ -0,0 +1,9 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { codexGpt6Luna } from '../presets.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...codexGpt6Luna,
+  suite: ['cli'],
+  skipEval: skipUnlessCli,
+});