feat(framework): add cli suites, needsDocker, and CLI environment arms

.github/CODEOWNERS

@@ -2,3 +2,8 @@
 
 /evals/docs/ @supabase/docs
 /apps/web/src/data/docs-eval-results.json @supabase/docs
+
+/evals/cli/ @supabase/cli
+/apps/web/src/data/cli-eval-results.json @supabase/cli
+/experiments/_lib/ @supabase/cli
+/experiments/codex-gpt-5.6-luna-cli-*.ts @supabase/cli

.github/workflows/append-gh-pages-history.yml

@@ -7,6 +7,7 @@ on:
       - apps/web/src/data/eval-results.json
       - apps/web/src/data/regression-eval-results.json
       - apps/web/src/data/docs-eval-results.json
+      - apps/web/src/data/cli-eval-results.json
   workflow_dispatch:
 
 permissions:
@@ -58,8 +59,9 @@ jobs:
           append_if_changed apps/web/src/data/eval-results.json results.jsonl
           append_if_changed apps/web/src/data/regression-eval-results.json regression-results.jsonl
           append_if_changed apps/web/src/data/docs-eval-results.json docs-results.jsonl
+          append_if_changed apps/web/src/data/cli-eval-results.json cli-results.jsonl
 
-          for history in results.jsonl regression-results.jsonl docs-results.jsonl; do
+          for history in results.jsonl regression-results.jsonl docs-results.jsonl cli-results.jsonl; do
             [ -f "$history" ] && git add "$history"
           done
           if git diff --cached --quiet; then

.github/workflows/eval-refresh.yml

@@ -41,6 +41,14 @@ on:
         type: boolean
         required: false
         default: false
+      cli_stable_version:
+        description: "Pin the cli suite's stable CLI version instead of resolving npm's latest dist-tag (blank to resolve)"
+        required: false
+        default: ""
+      cli_beta_version:
+        description: "Pin the cli suite's beta CLI version instead of resolving npm's beta dist-tag (blank to resolve)"
+        required: false
+        default: ""
   schedule:
     - cron: '15 6 * * *'
   pull_request:
@@ -80,6 +88,8 @@ jobs:
       filter_changed: ${{ steps.inputs.outputs.filter_changed }}
       do_merge: ${{ steps.inputs.outputs.do_merge }}
       suite: ${{ steps.inputs.outputs.suite }}
+      cli_stable_version: ${{ steps.cli_versions.outputs.cli_stable_version }}
+      cli_beta_version: ${{ steps.cli_versions.outputs.cli_beta_version }}
     steps:
       - name: Prepare inputs
         id: inputs
@@ -98,16 +108,16 @@ jobs:
           elif [ "${{ github.event_name }}" = "schedule" ]; then
             experiments_override=""
             eval_id=""
-            suite="regression"
-            experiment_suite="regression"
+            suite="regression,cli"
+            experiment_suite="regression,cli"
             runs="3"
             timeout_sec="720"
             sandbox_concurrency="250"
           else
             experiments_override=""
             eval_id=""
-            suite="benchmark,regression,docs"
-            experiment_suite="benchmark,no-skills,regression,docs"
+            suite="benchmark,regression,docs,cli"
+            experiment_suite="benchmark,no-skills,regression,docs,cli"
             runs="3"
             timeout_sec="720"
             sandbox_concurrency="250"
@@ -224,6 +234,7 @@ jobs:
               benchmark) experiment_suites=(benchmark no-skills) ;;
               regression) experiment_suites=(regression) ;;
               docs) experiment_suites=(docs) ;;
+              cli) experiment_suites=(cli) ;;
               *) continue ;;
             esac
 
@@ -258,6 +269,33 @@ jobs:
 
           echo "pairs=$pairs" >> "$GITHUB_OUTPUT"
 
+      - name: Resolve CLI channel versions
+        id: cli_versions
+        env:
+          # `inputs.*` is only populated for workflow_dispatch; other event
+          # types see it as null, which the `||` below turns into "".
+          CLI_STABLE_OVERRIDE: ${{ github.event_name == 'workflow_dispatch' && inputs.cli_stable_version || '' }}
+          CLI_BETA_OVERRIDE: ${{ github.event_name == 'workflow_dispatch' && inputs.cli_beta_version || '' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          if ! jq -e 'index("cli") != null' <<< '${{ steps.inputs.outputs.suite }}' > /dev/null; then
+            {
+              echo "cli_stable_version="
+              echo "cli_beta_version="
+            } >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          pnpm --filter @supabase-evals/framework exec node --import tsx/esm -e "
+            import { resolveCliVersion } from '../../experiments/_lib/cli-channel.ts';
+            const stable = process.env.CLI_STABLE_OVERRIDE || (await resolveCliVersion('stable'));
+            const beta = process.env.CLI_BETA_OVERRIDE || (await resolveCliVersion('beta'));
+            console.log('cli_stable_version=' + stable);
+            console.log('cli_beta_version=' + beta);
+          " >> "$GITHUB_OUTPUT"
+
   run-evals:
     needs: prepare
     if: needs.prepare.outputs.pairs != '[]'
@@ -273,6 +311,8 @@ jobs:
       VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
       VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }}
       VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}
+      SUPABASE_CLI_STABLE_VERSION: ${{ needs.prepare.outputs.cli_stable_version }}
+      SUPABASE_CLI_BETA_VERSION: ${{ needs.prepare.outputs.cli_beta_version }}
     steps:
       - name: Checkout
         uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3
@@ -301,6 +341,8 @@ jobs:
             echo "OPENAI_API_KEY=${OPENAI_API_KEY}"
             echo "AI_GATEWAY_API_KEY=${AI_GATEWAY_API_KEY}"
             echo "XAI_API_KEY=${XAI_API_KEY}"
+            echo "SUPABASE_CLI_STABLE_VERSION=${SUPABASE_CLI_STABLE_VERSION}"
+            echo "SUPABASE_CLI_BETA_VERSION=${SUPABASE_CLI_BETA_VERSION}"
           } > .env
 
       - name: Run evals
@@ -464,7 +506,7 @@ jobs:
             pnpm --filter @supabase-evals/framework export-results -- "${export_args[@]}"
           fi
 
-          for eval_suite in regression docs; do
+          for eval_suite in regression docs cli; do
             if jq -e --arg s "$eval_suite" 'any(.[]; .eval_suite == $s)' <<< "$pairs" > /dev/null; then
               export_args=(--suite "$eval_suite" --runs "${{ needs.prepare.outputs.runs }}" --output "apps/web/src/data/${eval_suite}-eval-results.json")
               if [ "${{ needs.prepare.outputs.do_merge }}" = "true" ]; then

CONTRIBUTING.md

@@ -9,6 +9,7 @@ First, determine the eval suite for your scenario:
 - **Regression** evals are suitable for most scenarios. If we notice agents make a narrow mistake, we track it here to reproduce the issue, verify a fix, and monitor for regression. These scenarios are not included in the benchmark so they don't inflate scores.
 - **Benchmark** evals are scenarios we've intentionally selected for the published benchmark report. These should be representative of the user journey on Supabase to cover a breadth of dimensions.
 - **Docs** evals are owned by docs team for their own analysis of how agents interpret docs pages, refreshed as-needed.
+- **CLI** evals are owned by the CLI team: one scenario run unchanged across forced CLI environments (Docker available / daemon unreachable / absent; pinned, stable and beta CLI) via the `cli` experiment suite.
 
 Then add a folder under `evals/<suite>/` containing:
 
@@ -68,3 +69,19 @@ Common workflows:
 - **Add or change a docs eval.** Add the scenario under `evals/docs/<id>/` (see [Adding an eval](#adding-an-eval)), open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview.
 - **Refresh every docs eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: docs` and `experiment_suite: docs`. It opens a draft PR with the updated `docs-eval-results.json` for you to review and merge.
 - **Analyze results over time.** Every merge that changes `docs-eval-results.json` appends a snapshot to [`docs-results.jsonl`](https://supabase.github.io/evals/docs-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl) and [regression](https://supabase.github.io/evals/regression-results.jsonl) histories.
+
+## CLI evals
+
+The CLI team owns `evals/cli/` and its results. CLI evals run on the pinned-CLI baseline (`codex-gpt-5.6-luna`) plus `codex-gpt-5.6-luna-cli-{stable,beta,nodaemon,absent}`, which install the latest stable or beta CLI and force Docker-less sandboxes to compare the same scenario across CLI environments.
+
+Which evals each arm picks up:
+
+- **pinned, stable, beta** run every `interface: cli` eval that isn't `hostedProject: true`.
+- **nodaemon, absent** additionally only run evals that also set `needsDocker: false` and `projectRunning: false` — the harness cannot pre-start a stack, or link a hosted project, without Docker.
+
+Common workflows:
+
+- **Add or change a CLI eval.** Add the scenario under `evals/cli/<id>/` (see [Adding an eval](#adding-an-eval)); set `needsDocker: false` in its `PROMPT.md` frontmatter if it can run without Docker, open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview.
+- **Refresh every CLI eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: cli` and `experiment_suite: cli`. It opens a draft PR with the updated `cli-eval-results.json` for you to review and merge. Leave `cli_stable_version`/`cli_beta_version` blank to resolve npm's latest dist-tags, or pin them to reproduce a specific run.
+- **Analyze results over time.** Every merge that changes `cli-eval-results.json` appends a snapshot to [`cli-results.jsonl`](https://supabase.github.io/evals/cli-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl), [regression](https://supabase.github.io/evals/regression-results.jsonl), and [docs](https://supabase.github.io/evals/docs-results.jsonl) histories.
+- **Run the unit tests.** `pnpm --filter @supabase-evals/framework test:cli-lib` (the CLI runtime helpers plus every CLI eval's scorer tests) — also part of `pnpm check`.

README.md

@@ -29,7 +29,7 @@ Agent-backed runs require the relevant provider key in `.env` (e.g. `OPENAI_API_
 
 - An **eval** is one scenario under `evals/<suite>/<id>/`. It contains the prompt, scorer, and optional starting state for the two environments: `remote/` (the hosted project) and `local/` (the agent's working files).
 - An **experiment** is one agent/runtime/model setup under `experiments/<name>.ts`.
-- An **eval suite** is a named set of evals to run together. An eval's suite is its parent folder under `evals/` (`benchmark`, `regression`, `docs`, or `other`).
+- An **eval suite** is a named set of evals to run together. An eval's suite is its parent folder under `evals/` (`benchmark`, `regression`, `docs`, `cli`, or `other`).
 - An **experiment suite** is a named set of experiments with related configurations, for head to head comparisons.
 - An **agent** is the model driver that receives the eval prompt and calls the configured tools.
 - A **runtime** is the local Supabase-like environment and tool surface an experiment gives to the agent.
@@ -147,6 +147,8 @@ Set `cliVersion: 2.109.1` in an eval's frontmatter when it requires a specific S
 
 Scorers check what the agent produced, never what the harness provisioned: with `projectRunning: true` (the default) the running stack and the seeded `local/` workspace are setup, so score only the deltas the agent made on top; with `projectRunning: false` the agent creates that state itself, so depending on it is fair game.
 
+`needsDocker` defaults to `true`; set it `false` when the scenario can run, and is meaningful, without a Docker daemon (e.g. starting the stack is the agent's own job), so Docker-less experiments pick it up.
+
 Test the sandbox plumbing without an agent run (Docker required, not part of `pnpm check`):
 
 ```bash

apps/framework/package.json

@@ -4,7 +4,7 @@
   "version": "0.0.1",
   "type": "module",
   "scripts": {
-    "check": "pnpm typecheck && pnpm test && pnpm test:framework && pnpm test:vercel-runner",
+    "check": "pnpm typecheck && pnpm test && pnpm test:framework && pnpm test:vercel-runner && pnpm test:cli-lib",
     "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts",
     "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry",
     "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke",
@@ -13,6 +13,7 @@
     "test": "vitest run harness",
     "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts",
     "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts lib/cli-args.test.ts lib/sample-sets.test.ts",
+    "test:cli-lib": "vitest run --root ../.. experiments/_lib evals/cli",
     "export-results": "node --import tsx/esm scripts/export-results.ts",
     "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts",
     "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts"

apps/framework/scripts/run-vercel-evals.test.ts

@@ -1,6 +1,7 @@
 import { APIError } from '@vercel/sandbox';
-import { describe, expect, it } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
 import {
+  agentEnvironment,
   isRetryableSandboxCreateError,
   isTerminalSandboxCreateError,
   parsePairs,
@@ -9,6 +10,52 @@ import {
   expandJobs,
 } from './run-vercel-evals.js';
 
+const FORWARDED_ENV_NAMES = [
+  'ANTHROPIC_API_KEY',
+  'OPENAI_API_KEY',
+  'AI_GATEWAY_API_KEY',
+  'SUPABASE_CLI_STABLE_VERSION',
+  'SUPABASE_CLI_BETA_VERSION',
+];
+
+describe('agentEnvironment', () => {
+  const originalValues = new Map<string, string | undefined>();
+
+  beforeEach(() => {
+    for (const name of FORWARDED_ENV_NAMES) {
+      originalValues.set(name, process.env[name]);
+      delete process.env[name];
+    }
+  });
+
+  afterEach(() => {
+    for (const [name, value] of originalValues) {
+      if (value === undefined) delete process.env[name];
+      else process.env[name] = value;
+    }
+  });
+
+  it('forwards every configured name when set', () => {
+    for (const name of FORWARDED_ENV_NAMES) {
+      process.env[name] = `${name}-value`;
+    }
+
+    const lines = agentEnvironment().split('\n');
+    for (const name of FORWARDED_ENV_NAMES) {
+      expect(lines).toContain(`${name}=${name}-value`);
+    }
+  });
+
+  it('omits the CLI channel pins when unset', () => {
+    process.env.ANTHROPIC_API_KEY = 'anthropic-value';
+
+    const env = agentEnvironment();
+    expect(env).toBe('ANTHROPIC_API_KEY=anthropic-value');
+    expect(env).not.toContain('SUPABASE_CLI_STABLE_VERSION');
+    expect(env).not.toContain('SUPABASE_CLI_BETA_VERSION');
+  });
+});
+
 describe('Vercel eval controller', () => {
   it('bounds concurrent work and lets independent failures settle', async () => {
     let active = 0;

apps/framework/scripts/run-vercel-evals.ts

@@ -14,11 +14,16 @@ const ROOT = fileURLToPath(new URL('../../../', import.meta.url));
 /** Base for sandbox URLs printed during runs */
 const SANDBOX_DASHBOARD_URL =
   'https://vercel.com/supabase/evals-runner/sandboxes';
-const AGENT_ENV_NAMES = [
+const FORWARDED_ENV_NAMES = [
   'ANTHROPIC_API_KEY',
   'OPENAI_API_KEY',
   'AI_GATEWAY_API_KEY',
   'XAI_API_KEY',
+  // Pins the CLI channel versions the prepare job resolved for this run, so
+  // every sandbox job in the run scores against the same version instead of
+  // each independently re-resolving "latest" and drifting mid-run.
+  'SUPABASE_CLI_STABLE_VERSION',
+  'SUPABASE_CLI_BETA_VERSION',
 ];
 /**
  * Slack for the non-agent work inside `pnpm eval` (supabase start, resets,
@@ -544,10 +549,10 @@ function vercelCredentialsFromEnv(): {
   };
 }
 
-/** Serializes configured provider keys into the repo-root `.env` file. */
-function agentEnvironment(): string {
+/** Serializes configured provider keys and CLI channel pins into the sandbox's `.env` file. */
+export function agentEnvironment(): string {
   const lines: string[] = [];
-  for (const name of AGENT_ENV_NAMES) {
+  for (const name of FORWARDED_ENV_NAMES) {
     const value = process.env[name];
     if (value) lines.push(`${name}=${value}`);
   }

apps/framework/tsconfig.json

@@ -1,10 +1,11 @@
 {
   "extends": "../../tsconfig.base.json",
   "include": [
-    "experiments",
+    "../../experiments",
     "harness",
     "shims",
     "scripts",
     "../../evals/**/EVAL.ts"
-  ]
+  ],
+  "exclude": ["../../experiments/**/*.test.ts"]
 }