From 849dabdb32b8f47a2a940a844571ce5308208b66 Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Fri, 18 Sep 2026 09:22:48 +0100 Subject: [PATCH 1/2] feat(framework): add cli suites, needsDocker, and CLI environment arms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Give the CLI team the same home the docs team got in #282: a `cli` eval suite (`evals/cli/`, owned by @supabase/cli, exported to cli-eval-results.json with a cli-results.jsonl history) and a `cli` experiment suite scheduled nightly next to regression. Pairing stays folder-driven, so CLI experiment variants that force unusual environments never pair with other teams' evals, and vice versa. Add a `needsDocker` frontmatter field (default true). An eval that can run, and is meaningful, without a Docker daemon sets it false; experiments that stage Docker-less sandboxes select on it via `skipEval` instead of keeping lists of eval ids. Add the experiment-land runtime the CLI arms use: `resolveCliVersion` turns `stable`/`beta` into a concrete CLI version from npm dist-tags (lazily, memoised, overridable, never silently falling back to the pin), and `dockerAwareLocalStackRuntime({ channel, docker })` installs that version and, for `no-daemon`/`absent`, makes Docker unusable in layers the agent cannot undo — DOCKER_HOST pointed at an unbound port on every command, a root-owned `supabase` shim that re-exports it, the real docker binaries removed, and (no-daemon) a `docker` shim that still answers --version so the CLI's managed-stack runtime probe selects Docker as it would on a real host. CI's sandbox makes the socket world-writable, so permissions alone cannot stage this. Built only from `@supabase-evals/sandbox` exports. Experiments: codex-gpt-5.6-luna-cli-{stable,beta,nodaemon,absent} in the `cli` suite, and the pinned Codex Luna experiment tagged `cli` as the baseline column. No evals yet; the first is stacked on this branch. Refs: https://linear.app/supabase/issue/CLI-2398/add-a-docker-less-local-stack-e2e-eval-agent-cli-lifecycle --- .github/CODEOWNERS | 3 + .github/workflows/append-gh-pages-history.yml | 4 +- .github/workflows/eval-refresh.yml | 11 +- CONTRIBUTING.md | 12 + README.md | 4 +- experiments/_lib/cli-channel.test.ts | 187 +++++++++++ experiments/_lib/cli-channel.ts | 80 +++++ .../_lib/docker-aware-local-stack.test.ts | 75 +++++ experiments/_lib/docker-aware-local-stack.ts | 308 ++++++++++++++++++ experiments/codex-gpt-5.6-luna-cli-absent.ts | 26 ++ experiments/codex-gpt-5.6-luna-cli-beta.ts | 24 ++ .../codex-gpt-5.6-luna-cli-nodaemon.ts | 26 ++ experiments/codex-gpt-5.6-luna-cli-stable.ts | 24 ++ experiments/codex-gpt-5.6-luna.ts | 3 +- packages/core/src/eval-metadata.ts | 12 + 15 files changed, 791 insertions(+), 8 deletions(-) create mode 100644 experiments/_lib/cli-channel.test.ts create mode 100644 experiments/_lib/cli-channel.ts create mode 100644 experiments/_lib/docker-aware-local-stack.test.ts create mode 100644 experiments/_lib/docker-aware-local-stack.ts create mode 100644 experiments/codex-gpt-5.6-luna-cli-absent.ts create mode 100644 experiments/codex-gpt-5.6-luna-cli-beta.ts create mode 100644 experiments/codex-gpt-5.6-luna-cli-nodaemon.ts create mode 100644 experiments/codex-gpt-5.6-luna-cli-stable.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 26f4dc44..fe40841c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,3 +2,6 @@ /evals/docs/ @supabase/docs /apps/web/src/data/docs-eval-results.json @supabase/docs + +/evals/cli/ @supabase/cli +/apps/web/src/data/cli-eval-results.json @supabase/cli diff --git a/.github/workflows/append-gh-pages-history.yml b/.github/workflows/append-gh-pages-history.yml index 8061dab3..04e96e15 100644 --- a/.github/workflows/append-gh-pages-history.yml +++ b/.github/workflows/append-gh-pages-history.yml @@ -7,6 +7,7 @@ on: - apps/web/src/data/eval-results.json - apps/web/src/data/regression-eval-results.json - apps/web/src/data/docs-eval-results.json + - apps/web/src/data/cli-eval-results.json workflow_dispatch: permissions: @@ -58,8 +59,9 @@ jobs: append_if_changed apps/web/src/data/eval-results.json results.jsonl append_if_changed apps/web/src/data/regression-eval-results.json regression-results.jsonl append_if_changed apps/web/src/data/docs-eval-results.json docs-results.jsonl + append_if_changed apps/web/src/data/cli-eval-results.json cli-results.jsonl - for history in results.jsonl regression-results.jsonl docs-results.jsonl; do + for history in results.jsonl regression-results.jsonl docs-results.jsonl cli-results.jsonl; do [ -f "$history" ] && git add "$history" done if git diff --cached --quiet; then diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 4b738fa8..59bf7146 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -98,16 +98,16 @@ jobs: elif [ "${{ github.event_name }}" = "schedule" ]; then experiments_override="" eval_id="" - suite="regression" - experiment_suite="regression" + suite="regression,cli" + experiment_suite="regression,cli" runs="3" timeout_sec="720" sandbox_concurrency="250" else experiments_override="" eval_id="" - suite="benchmark,regression,docs" - experiment_suite="benchmark,no-skills,regression,docs" + suite="benchmark,regression,docs,cli" + experiment_suite="benchmark,no-skills,regression,docs,cli" runs="3" timeout_sec="720" sandbox_concurrency="250" @@ -224,6 +224,7 @@ jobs: benchmark) experiment_suites=(benchmark no-skills) ;; regression) experiment_suites=(regression) ;; docs) experiment_suites=(docs) ;; + cli) experiment_suites=(cli) ;; *) continue ;; esac @@ -462,7 +463,7 @@ jobs: pnpm --filter @supabase-evals/framework export-results -- "${export_args[@]}" fi - for eval_suite in regression docs; do + for eval_suite in regression docs cli; do if jq -e --arg s "$eval_suite" 'any(.[]; .eval_suite == $s)' <<< "$pairs" > /dev/null; then export_args=(--suite "$eval_suite" --runs "${{ needs.prepare.outputs.runs }}" --output "apps/web/src/data/${eval_suite}-eval-results.json") if [ "${{ needs.prepare.outputs.do_merge }}" = "true" ]; then diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d0918bc3..e5ff163a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,6 +9,7 @@ First, determine the eval suite for your scenario: - **Regression** evals are suitable for most scenarios. If we notice agents make a narrow mistake, we track it here to reproduce the issue, verify a fix, and monitor for regression. These scenarios are not included in the benchmark so they don't inflate scores. - **Benchmark** evals are scenarios we've intentionally selected for the published benchmark report. These should be representative of the user journey on Supabase to cover a breadth of dimensions. - **Docs** evals are owned by docs team for their own analysis of how agents interpret docs pages, refreshed as-needed. +- **CLI** evals are owned by the CLI team: one scenario run unchanged across forced CLI environments (Docker available / daemon unreachable / absent; pinned, stable and beta CLI) via the `cli` experiment suite. Then add a folder under `evals//` containing: @@ -68,3 +69,14 @@ Common workflows: - **Add or change a docs eval.** Add the scenario under `evals/docs//` (see [Adding an eval](#adding-an-eval)), open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview. - **Refresh every docs eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: docs` and `experiment_suite: docs`. It opens a draft PR with the updated `docs-eval-results.json` for you to review and merge. - **Analyze results over time.** Every merge that changes `docs-eval-results.json` appends a snapshot to [`docs-results.jsonl`](https://supabase.github.io/evals/docs-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl) and [regression](https://supabase.github.io/evals/regression-results.jsonl) histories. + +## CLI evals + +The CLI team owns `evals/cli/` and its results. CLI evals run on the pinned-CLI baseline (`codex-gpt-5.6-luna`) plus `codex-gpt-5.6-luna-cli-{stable,beta,nodaemon,absent}`, which install the latest stable or beta CLI and force Docker-less sandboxes to compare the same scenario across CLI environments. + +Common workflows: + +- **Add or change a CLI eval.** Add the scenario under `evals/cli//` (see [Adding an eval](#adding-an-eval)); set `needsDocker: false` in its `PROMPT.md` frontmatter if it can run without Docker, open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview. +- **Refresh every CLI eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: cli` and `experiment_suite: cli`. It opens a draft PR with the updated `cli-eval-results.json` for you to review and merge. +- **Analyze results over time.** Every merge that changes `cli-eval-results.json` appends a snapshot to [`cli-results.jsonl`](https://supabase.github.io/evals/cli-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl), [regression](https://supabase.github.io/evals/regression-results.jsonl), and [docs](https://supabase.github.io/evals/docs-results.jsonl) histories. +- **Run the unit tests.** `pnpm --filter @supabase-evals/framework exec vitest run --root ../.. experiments/_lib evals/cli` (the CLI runtime helpers plus every CLI eval's scorer tests; these paths sit outside the package `test` scripts, so `pnpm check` does not run them) diff --git a/README.md b/README.md index 52a84349..c705cbfc 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Agent-backed runs require the relevant provider key in `.env` (e.g. `OPENAI_API_ - An **eval** is one scenario under `evals///`. It contains the prompt, scorer, and optional starting state for the two environments: `remote/` (the hosted project) and `local/` (the agent's working files). - An **experiment** is one agent/runtime/model setup under `experiments/.ts`. -- An **eval suite** is a named set of evals to run together. An eval's suite is its parent folder under `evals/` (`benchmark`, `regression`, `docs`, or `other`). +- An **eval suite** is a named set of evals to run together. An eval's suite is its parent folder under `evals/` (`benchmark`, `regression`, `docs`, `cli`, or `other`). - An **experiment suite** is a named set of experiments with related configurations, for head to head comparisons. - An **agent** is the model driver that receives the eval prompt and calls the configured tools. - A **runtime** is the local Supabase-like environment and tool surface an experiment gives to the agent. @@ -147,6 +147,8 @@ Set `cliVersion: 2.109.1` in an eval's frontmatter when it requires a specific S Scorers check what the agent produced, never what the harness provisioned: with `projectRunning: true` (the default) the running stack and the seeded `local/` workspace are setup, so score only the deltas the agent made on top; with `projectRunning: false` the agent creates that state itself, so depending on it is fair game. +`needsDocker` defaults to `true`; set it `false` when the scenario can run, and is meaningful, without a Docker daemon (e.g. starting the stack is the agent's own job), so Docker-less experiments pick it up. + Test the sandbox plumbing without an agent run (Docker required, not part of `pnpm check`): ```bash diff --git a/experiments/_lib/cli-channel.test.ts b/experiments/_lib/cli-channel.test.ts new file mode 100644 index 00000000..0c01c795 --- /dev/null +++ b/experiments/_lib/cli-channel.test.ts @@ -0,0 +1,187 @@ +// Run: pnpm --filter @supabase-evals/framework exec vitest run --root ../.. experiments/_lib +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const STABLE_ENV = 'SUPABASE_CLI_STABLE_VERSION'; +const BETA_ENV = 'SUPABASE_CLI_BETA_VERSION'; +const DIST_TAGS_URL = 'https://registry.npmjs.org/-/package/supabase/dist-tags'; + +function distTagsResponse( + body: unknown, + init: { ok?: boolean; status?: number; statusText?: string } = {} +) { + return { + ok: init.ok ?? true, + status: init.status ?? 200, + statusText: init.statusText ?? 'OK', + json: async () => body, + }; +} + +beforeEach(() => { + vi.resetModules(); + delete process.env[STABLE_ENV]; + delete process.env[BETA_ENV]; +}); + +afterEach(() => { + vi.unstubAllGlobals(); + delete process.env[STABLE_ENV]; + delete process.env[BETA_ENV]; +}); + +describe('resolveCliVersion', () => { + it('resolves the stable channel from the "latest" dist-tag', async () => { + const fetchMock = vi + .fn() + .mockResolvedValue( + distTagsResponse({ latest: '1.2.3', beta: '1.3.0-rc.1' }) + ); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('./cli-channel.js'); + + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + + expect(fetchMock).toHaveBeenCalledTimes(1); + const [url, options] = fetchMock.mock.calls[0]; + expect(url).toBe(DIST_TAGS_URL); + expect(options.signal).toBeInstanceOf(AbortSignal); + }); + + it('resolves the beta channel from the "beta" dist-tag', async () => { + const fetchMock = vi + .fn() + .mockResolvedValue( + distTagsResponse({ latest: '1.2.3', beta: '1.3.0-rc.1' }) + ); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('./cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('1.3.0-rc.1'); + }); + + it('prefers the env override over the network and strips a leading v', async () => { + process.env[STABLE_ENV] = 'v9.9.9'; + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('./cli-channel.js'); + + await expect(resolveCliVersion('stable')).resolves.toBe('9.9.9'); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('trims surrounding whitespace from the env override', async () => { + process.env[BETA_ENV] = ' 1.4.0-rc.2 '; + vi.stubGlobal('fetch', vi.fn()); + const { resolveCliVersion } = await import('./cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('1.4.0-rc.2'); + }); + + it('throws naming the env var when the override is not a valid version', async () => { + process.env[BETA_ENV] = 'not-a-version'; + vi.stubGlobal('fetch', vi.fn()); + const { resolveCliVersion } = await import('./cli-channel.js'); + + await expect(resolveCliVersion('beta')).rejects.toThrow( + `${BETA_ENV}="not-a-version" is not a valid Supabase CLI version` + ); + }); + + it('throws with the HTTP status when the registry request fails', async () => { + const fetchMock = vi.fn().mockResolvedValue( + distTagsResponse(undefined, { + ok: false, + status: 500, + statusText: 'Internal Server Error', + }) + ); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('./cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow( + `${DIST_TAGS_URL} -> 500 Internal Server Error` + ); + }); + + it('throws when the requested dist-tag is missing from the response', async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(distTagsResponse({ beta: '1.0.0' })); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('./cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow( + /did not have a valid "latest" version for the stable channel/ + ); + }); + + it('throws when the dist-tag value is not a valid version string', async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(distTagsResponse({ latest: 'not-a-version' })); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('./cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow( + /did not have a valid "latest" version for the stable channel/ + ); + }); + + it('memoises a successful resolution so a second call does not refetch', async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(distTagsResponse({ latest: '1.2.3', beta: '1.3.0' })); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('./cli-channel.js'); + + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('clears the cache entry on rejection so a later call refetches', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + distTagsResponse(undefined, { + ok: false, + status: 500, + statusText: 'Internal Server Error', + }) + ) + .mockResolvedValueOnce( + distTagsResponse({ latest: '1.2.3', beta: '1.3.0' }) + ); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('./cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow('500'); + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it('does not let a beta rejection clear the stable cache entry', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + distTagsResponse({ latest: '1.2.3', beta: '1.3.0' }) + ) + .mockResolvedValueOnce( + distTagsResponse(undefined, { + ok: false, + status: 500, + statusText: 'Internal Server Error', + }) + ); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('./cli-channel.js'); + + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + await expect(resolveCliVersion('beta')).rejects.toThrow('500'); + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + + expect(fetchMock).toHaveBeenCalledTimes(2); + }); +}); diff --git a/experiments/_lib/cli-channel.ts b/experiments/_lib/cli-channel.ts new file mode 100644 index 00000000..b3ea2e58 --- /dev/null +++ b/experiments/_lib/cli-channel.ts @@ -0,0 +1,80 @@ +/** + * Resolves "latest stable" / "latest beta" Supabase CLI versions from npm's + * dist-tags for the `supabase` package — no auth, no rate-limit exposure + * (unlike the GitHub releases API), which matters because CI's sandbox only + * forwards the model API keys, never a GITHUB_TOKEN. + */ + +export type CliChannel = 'stable' | 'beta'; + +const NPM_DIST_TAGS_URL = + 'https://registry.npmjs.org/-/package/supabase/dist-tags'; + +// npm's beta dist-tag can carry a prerelease suffix like -rc.1, not just -beta.N. +const VERSION_RE = /^\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?$/; + +const ENV_OVERRIDE: Record = { + stable: 'SUPABASE_CLI_STABLE_VERSION', + beta: 'SUPABASE_CLI_BETA_VERSION', +}; + +const DIST_TAG: Record = { + stable: 'latest', + beta: 'beta', +}; + +const versionCache = new Map>(); + +/** + * Resolve a channel to a concrete Supabase CLI version. Memoised per channel + * so a single nightly run only hits the registry once per channel; the cache + * entry is cleared on rejection so a later retry can hit the network again. + * Never falls back to the pinned SUPABASE_CLI_VERSION on failure — that would + * silently mislabel data — so callers must let the throw propagate. + */ +export async function resolveCliVersion(channel: CliChannel): Promise { + const cached = versionCache.get(channel); + if (cached) return cached; + + const promise = resolveCliVersionUncached(channel); + versionCache.set(channel, promise); + promise.catch(() => versionCache.delete(channel)); + return promise; +} + +async function resolveCliVersionUncached(channel: CliChannel): Promise { + const envVar = ENV_OVERRIDE[channel]; + const override = process.env[envVar]?.trim().replace(/^v/, ''); + if (override) { + if (!VERSION_RE.test(override)) { + throw new Error( + `${envVar}=${JSON.stringify(override)} is not a valid Supabase CLI version` + ); + } + return override; + } + + const response = await fetch(NPM_DIST_TAGS_URL, { + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) { + throw new Error( + `failed to resolve the latest ${channel} Supabase CLI version: ` + + `GET ${NPM_DIST_TAGS_URL} -> ${response.status} ${response.statusText}` + ); + } + const tags = await response.json(); + const distTag = DIST_TAG[channel]; + const version = isRecord(tags) ? tags[distTag] : undefined; + if (typeof version !== 'string' || !VERSION_RE.test(version)) { + throw new Error( + `npm dist-tags for "supabase" did not have a valid "${distTag}" version ` + + `for the ${channel} channel: ${JSON.stringify(version)}` + ); + } + return version; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} diff --git a/experiments/_lib/docker-aware-local-stack.test.ts b/experiments/_lib/docker-aware-local-stack.test.ts new file mode 100644 index 00000000..2eebf341 --- /dev/null +++ b/experiments/_lib/docker-aware-local-stack.test.ts @@ -0,0 +1,75 @@ +// Run: pnpm --filter @supabase-evals/framework exec vitest run --root ../.. experiments/_lib +import { describe, expect, it } from 'vitest'; +import type { SupabaseService } from '@supabase-evals/sandbox'; +import { + buildDockerDaemonShimScript, + buildSupabaseShimScript, + dockerAwareLocalStackRuntime, +} from './docker-aware-local-stack.js'; + +describe('dockerAwareLocalStackRuntime', () => { + it('ids as `local-stack-cli-` when docker is unset (available)', () => { + expect(dockerAwareLocalStackRuntime({ channel: 'stable' }).id).toBe( + 'local-stack-cli-stable' + ); + }); + + it('ids as `local-stack-cli--` when docker is a non-available state', () => { + expect( + dockerAwareLocalStackRuntime({ channel: 'beta', docker: 'no-daemon' }).id + ).toBe('local-stack-cli-beta-no-daemon'); + }); +}); + +describe('buildSupabaseShimScript', () => { + it('emits DOCKER_HOST, the -x start branch for excluded services, and a passthrough exec', () => { + const excluded: SupabaseService[] = ['gotrue', 'kong']; + expect(buildSupabaseShimScript('/usr/bin/supabase', excluded)).toEqual( + [ + '#!/bin/bash', + 'export DOCKER_HOST=tcp://127.0.0.1:1', + 'REAL="/usr/bin/supabase"', + 'if [ "$1" = "start" ]; then shift; exec "$REAL" start "$@" -x gotrue,kong; fi', + 'exec "$REAL" "$@"', + ].join('\n') + ); + }); + + it('omits the start branch entirely when no services are excluded', () => { + expect( + buildSupabaseShimScript('/usr/bin/supabase', []) + ).toMatchInlineSnapshot(` + "#!/bin/bash + export DOCKER_HOST=tcp://127.0.0.1:1 + REAL="/usr/bin/supabase" + exec "$REAL" "$@"" + `); + }); + + it('always ends with exec "$REAL" "$@" regardless of exclusions', () => { + const excluded: SupabaseService[] = ['gotrue', 'kong']; + const script = buildSupabaseShimScript('/usr/bin/supabase', excluded); + expect(script.endsWith('exec "$REAL" "$@"')).toBe(true); + }); + + it('JSON-quotes a realBin containing a single quote safely for bash double-quoting', () => { + const script = buildSupabaseShimScript("/usr/local/it's/bin/supabase", []); + expect(script).toContain(`REAL="/usr/local/it's/bin/supabase"`); + expect(script).not.toContain('REAL=/usr/local/it'); + }); +}); + +describe('buildDockerDaemonShimScript', () => { + const dockerVersion = 'Docker version 20.10.24+dfsg1, build 297e128'; + + it('echoes the captured version string verbatim for --version/-v and exits 0, otherwise fails as unreachable', () => { + expect(buildDockerDaemonShimScript(dockerVersion)).toMatchInlineSnapshot(` + "#!/bin/bash + case "$1" in + --version|-v) echo "Docker version 20.10.24+dfsg1, build 297e128"; exit 0 ;; + esac + echo "Cannot connect to the Docker daemon at tcp://127.0.0.1:1. Is the docker daemon running?" >&2 + exit 1" + `); + }); +}); diff --git a/experiments/_lib/docker-aware-local-stack.ts b/experiments/_lib/docker-aware-local-stack.ts new file mode 100644 index 00000000..fcd48e7a --- /dev/null +++ b/experiments/_lib/docker-aware-local-stack.ts @@ -0,0 +1,308 @@ +/** + * A `LocalStackRuntime` that installs the latest stable/beta Supabase CLI + * (instead of the pinned default) and, when the calling experiment passes a + * `docker` option, stages a sandbox where the Docker daemon is unreachable + * or the `docker` binary is absent entirely — while still offering the agent + * the exact same tools, MCP wiring, and prompt as the stock + * `localStackRuntime()` for a normal (Docker-available) session. Which arm + * runs is chosen by the experiment, not read from any eval-side marker. + */ + +import { + supabaseMcpServer, + type LocalStackRuntime, + type LocalStackSession, + type LocalStackSessionArgs, +} from '@supabase-evals/core'; +import { + buildLocalStackScoringContext, + buildLocalStackTools, + buildSkillsPrompt, + buildToolSurfaceAddendum, + computeExcludedServices, + DockerSandbox, + ensureSupabaseSandboxImage, + installSkills, + installSupabaseCli, + localStackRuntime, + teardownSupabaseProject, + toAgentSandbox, + type SupabaseService, +} from '@supabase-evals/sandbox'; +import { resolveCliVersion, type CliChannel } from './cli-channel.js'; + +export type DockerState = 'available' | 'no-daemon' | 'absent'; + +// CLI eval scorers duplicate this schema rather than import it, so evals stay +// self-contained; keep them in sync. Scorers should only read `channel` and +// `sessionStartedMs` — never `docker` (what the experiment staged) to decide +// pass/fail, or the eval would be grading against its own environment. +export const RUNTIME_MARKER_PATH = '/tmp/supabase-eval-runtime.json'; + +export type RuntimeMarker = { + runtime: 'docker-aware-local-stack'; + channel: CliChannel; + cliVersion: string; + docker: DockerState; + sessionStartedMs: number; +}; + +const SUPABASE_SHIM_PATH = '/usr/local/sbin/supabase'; +const DOCKER_SHIM_PATH = '/usr/local/sbin/docker'; + +// Port 1 is never bound (the CLI's own e2e suite reserves it for exactly this +// purpose); not 2375, which Docker Desktop can legitimately expose. +const UNREACHABLE_DOCKER_HOST = 'tcp://127.0.0.1:1'; + +export function dockerAwareLocalStackRuntime(options: { + channel: CliChannel; + docker?: DockerState; +}): LocalStackRuntime { + const { channel } = options; + const docker = options.docker ?? 'available'; + return { + id: + docker === 'available' + ? `local-stack-cli-${channel}` + : `local-stack-cli-${channel}-${docker}`, + async startSession( + args: LocalStackSessionArgs + ): Promise { + // Stamped before setup so it's comparable with the scorer's PID-1 fallback. + const sessionStartedMs = Date.now(); + const state = docker; + // An eval's own `cliVersion:` frontmatter still wins over the channel, + // same precedence as the stock local-stack runtime. + const version = args.cliVersion ?? (await resolveCliVersion(channel)); + + if (state === 'available') { + const session = await localStackRuntime({ + cliVersion: version, + }).startSession(args); + try { + await writeRuntimeMarker( + (command) => session.scoringContext.exec(command), + buildRuntimeMarker(channel, version, state, sessionStartedMs) + ); + } catch (err) { + await session.close(); + throw err; + } + return session; + } + + if (args.projectRunning !== false) { + throw new Error( + 'docker-less sandbox evals must set `projectRunning: false`; the harness cannot pre-start a stack without Docker' + ); + } + if (args.hosted) { + throw new Error( + 'docker-less sandbox evals cannot link a hosted project — set hostedProject: false' + ); + } + + const image = await ensureSupabaseSandboxImage(); + const sandbox = await DockerSandbox.create({ + image, + network: 'host', + mounts: args.mounts, + }); + + try { + if (!args.skipCliInstall) { + await installSupabaseCli(sandbox, version); + } + + // Deliberately no socket-group grant here: CI's sandbox already ends + // its Docker setup with `chmod 666 /var/run/docker.sock`, so the grant + // would be a no-op there — DOCKER_HOST below is the real mechanism. + sandbox.extraEnv = { + ...sandbox.extraEnv, + DOCKER_HOST: UNREACHABLE_DOCKER_HOST, + }; + + if (!args.skipCliInstall) { + await installSupabaseShim(sandbox, args.includeServices); + } + + // Captured before removal: no-daemon's shim echoes this for `docker + // --version` so the CLI's #6563 runtime probe still picks Docker. + let dockerVersion = ''; + if (state === 'no-daemon') { + dockerVersion = ( + await sandbox.runShellAsRoot('docker --version') + ).stdout.trim(); + if (!dockerVersion) { + throw new Error( + 'failed to capture `docker --version` before removing the real binary' + ); + } + } + + // Assert the postcondition instead of trusting `rm -f`, which always exits 0. + const removal = await sandbox.runShellAsRoot( + 'for b in docker dockerd docker-proxy; do p="$(command -v "$b" 2>/dev/null)" && rm -f "$p"; done; ! command -v docker >/dev/null 2>&1' + ); + if (!removal.ok) { + throw new Error( + `docker is still on PATH after removal: ${removal.stderr || removal.stdout}` + ); + } + + if (state === 'no-daemon') { + await installDockerDaemonShim(sandbox, dockerVersion); + } + + if (args.localDir) { + await sandbox.copyToContainer(args.localDir, sandbox.workdir); + } + const skills = await installSkills(sandbox, args.skills ?? []); + + const ping = await sandbox.runShell( + 'curl -sf --unix-socket /var/run/docker.sock http://localhost/_ping' + ); + if (ping.ok) { + console.warn( + '[docker-aware-local-stack] raw docker socket is reachable by the sandbox user; relying on DOCKER_HOST + shims' + ); + } + + await writeRuntimeMarker( + (command) => sandbox.runShell(command), + buildRuntimeMarker(channel, version, state, sessionStartedMs) + ); + + return { + tools: buildLocalStackTools(sandbox), + sandbox: toAgentSandbox(sandbox), + mcpServers: { + supabase: ( + await supabaseMcpServer({ features: ['docs'] }).createConfig( + undefined + ) + ).config, + }, + promptAddendum: [ + buildToolSurfaceAddendum(args.agent, { + skipCliInstall: args.skipCliInstall, + }), + buildSkillsPrompt(args.agent, skills), + ] + .filter(Boolean) + .join('\n\n'), + scoringContext: buildLocalStackScoringContext(sandbox), + exportWorkspace: (hostDir: string) => + sandbox.copyToHost(sandbox.workdir, hostDir), + // Nothing to restore without Docker; the `available` path above + // delegates to the stock session, which has its own ensureReady. + ensureReady: async () => {}, + close: async () => { + await teardownSupabaseProject(sandbox); + await sandbox.stop(); + }, + }; + } catch (err) { + await sandbox.stop(); + throw err; + } + }, + }; +} + +function buildRuntimeMarker( + channel: CliChannel, + cliVersion: string, + docker: DockerState, + sessionStartedMs: number +): RuntimeMarker { + return { + runtime: 'docker-aware-local-stack', + channel, + cliVersion, + docker, + sessionStartedMs, + }; +} + +async function writeRuntimeMarker( + exec: ( + command: string + ) => Promise<{ ok: boolean; stdout: string; stderr: string }>, + marker: RuntimeMarker +): Promise { + // base64 transport sidesteps quoting the JSON payload through the shell. + const encoded = Buffer.from(JSON.stringify(marker), 'utf-8').toString( + 'base64' + ); + const result = await exec( + `echo ${encoded} | base64 -d > ${RUNTIME_MARKER_PATH}` + ); + if (!result.ok) { + throw new Error( + `failed to write the runtime marker: ${result.stderr || result.stdout}` + ); + } +} + +async function installSupabaseShim( + sandbox: DockerSandbox, + includeServices: readonly string[] | undefined +): Promise { + const real = ( + await sandbox.runShellAsRoot('command -v supabase') + ).stdout.trim(); + if (!real || real === SUPABASE_SHIM_PATH) { + throw new Error( + `could not resolve the real supabase binary before installing the CLI shim (got ${JSON.stringify(real)})` + ); + } + const excluded = computeExcludedServices(includeServices); + await sandbox.writeRootFile( + SUPABASE_SHIM_PATH, + buildSupabaseShimScript(real, excluded), + '0755' + ); +} + +export function buildSupabaseShimScript( + realBin: string, + excluded: readonly SupabaseService[] +): string { + const lines = [ + '#!/bin/bash', + `export DOCKER_HOST=${UNREACHABLE_DOCKER_HOST}`, + `REAL=${JSON.stringify(realBin)}`, + ]; + if (excluded.length > 0) { + lines.push( + `if [ "$1" = "start" ]; then shift; exec "$REAL" start "$@" -x ${excluded.join(',')}; fi` + ); + } + lines.push('exec "$REAL" "$@"'); + return lines.join('\n'); +} + +async function installDockerDaemonShim( + sandbox: DockerSandbox, + dockerVersion: string +): Promise { + await sandbox.writeRootFile( + DOCKER_SHIM_PATH, + buildDockerDaemonShimScript(dockerVersion), + '0755' + ); +} + +export function buildDockerDaemonShimScript(dockerVersion: string): string { + return [ + '#!/bin/bash', + 'case "$1" in', + // --version must keep working so the CLI's new managed stack still + // *chooses* Docker as its runtime (per supabase/cli#6563's probe). + ' --version|-v) echo ' + JSON.stringify(dockerVersion) + '; exit 0 ;;', + 'esac', + `echo "Cannot connect to the Docker daemon at ${UNREACHABLE_DOCKER_HOST}. Is the docker daemon running?" >&2`, + 'exit 1', + ].join('\n'); +} diff --git a/experiments/codex-gpt-5.6-luna-cli-absent.ts b/experiments/codex-gpt-5.6-luna-cli-absent.ts new file mode 100644 index 00000000..37b12af4 --- /dev/null +++ b/experiments/codex-gpt-5.6-luna-cli-absent.ts @@ -0,0 +1,26 @@ +import { + codexAgent, + defineExperiment, + platformLiteRuntime, + supabaseMcpServer, +} from '@supabase-evals/core'; +import { dockerAwareLocalStackRuntime } from './_lib/docker-aware-local-stack.js'; + +export default defineExperiment({ + suite: ['cli'], + agent: codexAgent({ + model: 'gpt-5.6-luna', + reasoningEffort: 'medium', + }), + runtime: platformLiteRuntime({ + mcpServers: [supabaseMcpServer()], + }), + // beta: the Docker-less path only exists in the managed stack, which ships in beta. + localStack: dockerAwareLocalStackRuntime({ + channel: 'beta', + docker: 'absent', + }), + skills: ['supabase', 'supabase-postgres-best-practices'], + // A Docker-less sandbox can only run evals that declare they don't need Docker. + skipEval: (ev) => ev.metadata.needsDocker !== false, +}); diff --git a/experiments/codex-gpt-5.6-luna-cli-beta.ts b/experiments/codex-gpt-5.6-luna-cli-beta.ts new file mode 100644 index 00000000..b363b32c --- /dev/null +++ b/experiments/codex-gpt-5.6-luna-cli-beta.ts @@ -0,0 +1,24 @@ +import { + codexAgent, + defineExperiment, + platformLiteRuntime, + supabaseMcpServer, +} from '@supabase-evals/core'; +import { dockerAwareLocalStackRuntime } from './_lib/docker-aware-local-stack.js'; + +export default defineExperiment({ + suite: ['cli'], + agent: codexAgent({ + model: 'gpt-5.6-luna', + reasoningEffort: 'medium', + }), + runtime: platformLiteRuntime({ + mcpServers: [supabaseMcpServer()], + }), + localStack: dockerAwareLocalStackRuntime({ channel: 'beta' }), + skills: ['supabase', 'supabase-postgres-best-practices'], + // Only CLI evals exercise the installed CLI version; hosted evals seed .temp + // version files pinned to the baseline CLI's service versions. + skipEval: (ev) => + ev.metadata.interface !== 'cli' || ev.metadata.hostedProject === true, +}); diff --git a/experiments/codex-gpt-5.6-luna-cli-nodaemon.ts b/experiments/codex-gpt-5.6-luna-cli-nodaemon.ts new file mode 100644 index 00000000..60acc4b6 --- /dev/null +++ b/experiments/codex-gpt-5.6-luna-cli-nodaemon.ts @@ -0,0 +1,26 @@ +import { + codexAgent, + defineExperiment, + platformLiteRuntime, + supabaseMcpServer, +} from '@supabase-evals/core'; +import { dockerAwareLocalStackRuntime } from './_lib/docker-aware-local-stack.js'; + +export default defineExperiment({ + suite: ['cli'], + agent: codexAgent({ + model: 'gpt-5.6-luna', + reasoningEffort: 'medium', + }), + runtime: platformLiteRuntime({ + mcpServers: [supabaseMcpServer()], + }), + // beta: the Docker-less path only exists in the managed stack, which ships in beta. + localStack: dockerAwareLocalStackRuntime({ + channel: 'beta', + docker: 'no-daemon', + }), + skills: ['supabase', 'supabase-postgres-best-practices'], + // A Docker-less sandbox can only run evals that declare they don't need Docker. + skipEval: (ev) => ev.metadata.needsDocker !== false, +}); diff --git a/experiments/codex-gpt-5.6-luna-cli-stable.ts b/experiments/codex-gpt-5.6-luna-cli-stable.ts new file mode 100644 index 00000000..237dc161 --- /dev/null +++ b/experiments/codex-gpt-5.6-luna-cli-stable.ts @@ -0,0 +1,24 @@ +import { + codexAgent, + defineExperiment, + platformLiteRuntime, + supabaseMcpServer, +} from '@supabase-evals/core'; +import { dockerAwareLocalStackRuntime } from './_lib/docker-aware-local-stack.js'; + +export default defineExperiment({ + suite: ['cli'], + agent: codexAgent({ + model: 'gpt-5.6-luna', + reasoningEffort: 'medium', + }), + runtime: platformLiteRuntime({ + mcpServers: [supabaseMcpServer()], + }), + localStack: dockerAwareLocalStackRuntime({ channel: 'stable' }), + skills: ['supabase', 'supabase-postgres-best-practices'], + // Only CLI evals exercise the installed CLI version; hosted evals seed .temp + // version files pinned to the baseline CLI's service versions. + skipEval: (ev) => + ev.metadata.interface !== 'cli' || ev.metadata.hostedProject === true, +}); diff --git a/experiments/codex-gpt-5.6-luna.ts b/experiments/codex-gpt-5.6-luna.ts index d4e18052..a8f7e227 100644 --- a/experiments/codex-gpt-5.6-luna.ts +++ b/experiments/codex-gpt-5.6-luna.ts @@ -7,7 +7,8 @@ import { import { localStackRuntime } from '@supabase-evals/sandbox'; export default defineExperiment({ - suite: ['benchmark', 'regression'], + // cli: the pinned-CLI baseline column for CLI-team evals. + suite: ['benchmark', 'regression', 'cli'], agent: codexAgent({ model: 'gpt-5.6-luna', reasoningEffort: 'medium', diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index 78644602..82f6924d 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -44,6 +44,7 @@ export const evalSuiteSchema = z.enum([ 'benchmark', 'regression', 'docs', + 'cli', 'other', ]); export const EVAL_SUITES = evalSuiteSchema.options; @@ -54,6 +55,7 @@ export const experimentSuiteSchema = z.enum([ 'no-skills', 'regression', 'docs', + 'cli', ]); export const EXPERIMENT_SUITES = experimentSuiteSchema.options; export type ExperimentSuite = z.infer; @@ -152,6 +154,14 @@ export type EvalMetadata = { * with `projectRunning: false`. */ skipCliInstall?: boolean; + /** + * Whether this scenario needs a working Docker daemon (sandbox evals + * only). Defaults to true. Set false when the scenario can run, and is + * meaningful, in a sandbox without Docker (e.g. starting the stack is the + * agent's own job) — Docker-less experiments filter on this to decide + * which evals they can run. + */ + needsDocker?: boolean; }; export type ParsedEvalMarkdown = { @@ -173,6 +183,7 @@ export const evalMetadataSchema = z.object({ hostedProject: z.union([z.boolean(), z.stringbool()]).optional(), skills: z.array(z.string().min(1)).optional(), skipCliInstall: z.union([z.boolean(), z.stringbool()]).optional(), + needsDocker: z.union([z.boolean(), z.stringbool()]).optional(), }); // Collapse a YAML scalar into a comparable token: trim, lowercase, and fold @@ -251,6 +262,7 @@ export const evalFrontmatterSchema = z.preprocess((raw) => { ? toIdentifierList(data.skills) : undefined, skipCliInstall: data.skipCliInstall, + needsDocker: data.needsDocker, }; }, evalMetadataSchema); From 8fe4e2acef134fc70fa4bea469ec5050d4761aa1 Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Fri, 18 Sep 2026 10:00:53 +0100 Subject: [PATCH 2/2] fix(framework): harden cli arms after review and the first CI run The first CI run of the cli suite lost every beta-channel pair: npm's `beta` dist-tag pointed at 2.118.0-beta.52 while its GitHub release was still a draft, so the .deb download 404'd before the agent started. The resolver now HEAD-checks the release asset and, for beta only, walks back to the newest published `-beta.N`; a missing stable asset throws. The workflow resolves both channels once in `prepare` (overridable via workflow_dispatch inputs) and forwards the pins into the Vercel sandbox, so one nightly never mixes CLI versions across pairs. Make the Docker-less arms structurally Docker-less: `DockerSandboxOptions` gains `mountDockerSocket` (default true) and the arms create their sandbox without the socket, then assert it is absent and that no `docker` resolves on the agent's PATH. DOCKER_HOST and the shims stay as defence in depth. Gate the new code in CI: `apps/framework/tsconfig.json`'s `experiments` include pointed at a nonexistent directory, so experiments were never typechecked; `pnpm check` now typechecks them and runs the `_lib` and `evals/cli` unit tests (`test:cli-lib`). Also: shared `skipUnlessCli`/`skipUnlessDockerless` predicates so mis-tagged evals skip instead of erroring; root-owned read-only runtime marker; single-quote shell escaping in the shims; resolved version logged per session; `needsDocker` frontmatter parse tests; CODEOWNERS for the CLI runtime and arms; CONTRIBUTING spells out which evals each arm picks up. --- .github/CODEOWNERS | 2 + .github/workflows/eval-refresh.yml | 41 +++ CONTRIBUTING.md | 9 +- apps/framework/package.json | 3 +- .../scripts/run-vercel-evals.test.ts | 49 +++- apps/framework/scripts/run-vercel-evals.ts | 13 +- apps/framework/tsconfig.json | 5 +- experiments/_lib/cli-channel.test.ts | 242 +++++++++++++----- experiments/_lib/cli-channel.ts | 131 +++++++++- .../_lib/docker-aware-local-stack.test.ts | 116 ++++++++- experiments/_lib/docker-aware-local-stack.ts | 82 ++++-- experiments/codex-gpt-5.6-luna-cli-absent.ts | 8 +- experiments/codex-gpt-5.6-luna-cli-beta.ts | 10 +- .../codex-gpt-5.6-luna-cli-nodaemon.ts | 8 +- experiments/codex-gpt-5.6-luna-cli-stable.ts | 12 +- packages/sandbox/src/docker-sandbox.ts | 14 +- packages/sandbox/test/unit.test.ts | 37 +++ 17 files changed, 673 insertions(+), 109 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index fe40841c..e799e6e9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,3 +5,5 @@ /evals/cli/ @supabase/cli /apps/web/src/data/cli-eval-results.json @supabase/cli +/experiments/_lib/ @supabase/cli +/experiments/codex-gpt-5.6-luna-cli-*.ts @supabase/cli diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 59bf7146..18f5f8a2 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -41,6 +41,14 @@ on: type: boolean required: false default: false + cli_stable_version: + description: "Pin the cli suite's stable CLI version instead of resolving npm's latest dist-tag (blank to resolve)" + required: false + default: "" + cli_beta_version: + description: "Pin the cli suite's beta CLI version instead of resolving npm's beta dist-tag (blank to resolve)" + required: false + default: "" schedule: - cron: '15 6 * * *' pull_request: @@ -80,6 +88,8 @@ jobs: filter_changed: ${{ steps.inputs.outputs.filter_changed }} do_merge: ${{ steps.inputs.outputs.do_merge }} suite: ${{ steps.inputs.outputs.suite }} + cli_stable_version: ${{ steps.cli_versions.outputs.cli_stable_version }} + cli_beta_version: ${{ steps.cli_versions.outputs.cli_beta_version }} steps: - name: Prepare inputs id: inputs @@ -259,6 +269,33 @@ jobs: echo "pairs=$pairs" >> "$GITHUB_OUTPUT" + - name: Resolve CLI channel versions + id: cli_versions + env: + # `inputs.*` is only populated for workflow_dispatch; other event + # types see it as null, which the `||` below turns into "". + CLI_STABLE_OVERRIDE: ${{ github.event_name == 'workflow_dispatch' && inputs.cli_stable_version || '' }} + CLI_BETA_OVERRIDE: ${{ github.event_name == 'workflow_dispatch' && inputs.cli_beta_version || '' }} + shell: bash + run: | + set -euo pipefail + + if ! jq -e 'index("cli") != null' <<< '${{ steps.inputs.outputs.suite }}' > /dev/null; then + { + echo "cli_stable_version=" + echo "cli_beta_version=" + } >> "$GITHUB_OUTPUT" + exit 0 + fi + + pnpm --filter @supabase-evals/framework exec node --import tsx/esm -e " + import { resolveCliVersion } from '../../experiments/_lib/cli-channel.ts'; + const stable = process.env.CLI_STABLE_OVERRIDE || (await resolveCliVersion('stable')); + const beta = process.env.CLI_BETA_OVERRIDE || (await resolveCliVersion('beta')); + console.log('cli_stable_version=' + stable); + console.log('cli_beta_version=' + beta); + " >> "$GITHUB_OUTPUT" + run-evals: needs: prepare if: needs.prepare.outputs.pairs != '[]' @@ -273,6 +310,8 @@ jobs: VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} + SUPABASE_CLI_STABLE_VERSION: ${{ needs.prepare.outputs.cli_stable_version }} + SUPABASE_CLI_BETA_VERSION: ${{ needs.prepare.outputs.cli_beta_version }} steps: - name: Checkout uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 @@ -300,6 +339,8 @@ jobs: echo "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}" echo "OPENAI_API_KEY=${OPENAI_API_KEY}" echo "AI_GATEWAY_API_KEY=${AI_GATEWAY_API_KEY}" + echo "SUPABASE_CLI_STABLE_VERSION=${SUPABASE_CLI_STABLE_VERSION}" + echo "SUPABASE_CLI_BETA_VERSION=${SUPABASE_CLI_BETA_VERSION}" } > .env - name: Run evals diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e5ff163a..290f8497 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,9 +74,14 @@ Common workflows: The CLI team owns `evals/cli/` and its results. CLI evals run on the pinned-CLI baseline (`codex-gpt-5.6-luna`) plus `codex-gpt-5.6-luna-cli-{stable,beta,nodaemon,absent}`, which install the latest stable or beta CLI and force Docker-less sandboxes to compare the same scenario across CLI environments. +Which evals each arm picks up: + +- **pinned, stable, beta** run every `interface: cli` eval that isn't `hostedProject: true`. +- **nodaemon, absent** additionally only run evals that also set `needsDocker: false` and `projectRunning: false` — the harness cannot pre-start a stack, or link a hosted project, without Docker. + Common workflows: - **Add or change a CLI eval.** Add the scenario under `evals/cli//` (see [Adding an eval](#adding-an-eval)); set `needsDocker: false` in its `PROMPT.md` frontmatter if it can run without Docker, open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview. -- **Refresh every CLI eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: cli` and `experiment_suite: cli`. It opens a draft PR with the updated `cli-eval-results.json` for you to review and merge. +- **Refresh every CLI eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: cli` and `experiment_suite: cli`. It opens a draft PR with the updated `cli-eval-results.json` for you to review and merge. Leave `cli_stable_version`/`cli_beta_version` blank to resolve npm's latest dist-tags, or pin them to reproduce a specific run. - **Analyze results over time.** Every merge that changes `cli-eval-results.json` appends a snapshot to [`cli-results.jsonl`](https://supabase.github.io/evals/cli-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl), [regression](https://supabase.github.io/evals/regression-results.jsonl), and [docs](https://supabase.github.io/evals/docs-results.jsonl) histories. -- **Run the unit tests.** `pnpm --filter @supabase-evals/framework exec vitest run --root ../.. experiments/_lib evals/cli` (the CLI runtime helpers plus every CLI eval's scorer tests; these paths sit outside the package `test` scripts, so `pnpm check` does not run them) +- **Run the unit tests.** `pnpm --filter @supabase-evals/framework test:cli-lib` (the CLI runtime helpers plus every CLI eval's scorer tests) — also part of `pnpm check`. diff --git a/apps/framework/package.json b/apps/framework/package.json index 7f73e481..c2c6311a 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -4,7 +4,7 @@ "version": "0.0.1", "type": "module", "scripts": { - "check": "pnpm typecheck && pnpm test && pnpm test:framework && pnpm test:vercel-runner", + "check": "pnpm typecheck && pnpm test && pnpm test:framework && pnpm test:vercel-runner && pnpm test:cli-lib", "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts", "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry", "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke", @@ -13,6 +13,7 @@ "test": "vitest run harness", "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts lib/cli-args.test.ts lib/sample-sets.test.ts", + "test:cli-lib": "vitest run --root ../.. experiments/_lib evals/cli", "export-results": "node --import tsx/esm scripts/export-results.ts", "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts", "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts" diff --git a/apps/framework/scripts/run-vercel-evals.test.ts b/apps/framework/scripts/run-vercel-evals.test.ts index e7faf593..622480bd 100644 --- a/apps/framework/scripts/run-vercel-evals.test.ts +++ b/apps/framework/scripts/run-vercel-evals.test.ts @@ -1,6 +1,7 @@ import { APIError } from '@vercel/sandbox'; -import { describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { + agentEnvironment, isRetryableSandboxCreateError, isTerminalSandboxCreateError, parsePairs, @@ -9,6 +10,52 @@ import { expandJobs, } from './run-vercel-evals.js'; +const FORWARDED_ENV_NAMES = [ + 'ANTHROPIC_API_KEY', + 'OPENAI_API_KEY', + 'AI_GATEWAY_API_KEY', + 'SUPABASE_CLI_STABLE_VERSION', + 'SUPABASE_CLI_BETA_VERSION', +]; + +describe('agentEnvironment', () => { + const originalValues = new Map(); + + beforeEach(() => { + for (const name of FORWARDED_ENV_NAMES) { + originalValues.set(name, process.env[name]); + delete process.env[name]; + } + }); + + afterEach(() => { + for (const [name, value] of originalValues) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + }); + + it('forwards every configured name when set', () => { + for (const name of FORWARDED_ENV_NAMES) { + process.env[name] = `${name}-value`; + } + + const lines = agentEnvironment().split('\n'); + for (const name of FORWARDED_ENV_NAMES) { + expect(lines).toContain(`${name}=${name}-value`); + } + }); + + it('omits the CLI channel pins when unset', () => { + process.env.ANTHROPIC_API_KEY = 'anthropic-value'; + + const env = agentEnvironment(); + expect(env).toBe('ANTHROPIC_API_KEY=anthropic-value'); + expect(env).not.toContain('SUPABASE_CLI_STABLE_VERSION'); + expect(env).not.toContain('SUPABASE_CLI_BETA_VERSION'); + }); +}); + describe('Vercel eval controller', () => { it('bounds concurrent work and lets independent failures settle', async () => { let active = 0; diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index ebd03a51..73e4ff9c 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -14,10 +14,15 @@ const ROOT = fileURLToPath(new URL('../../../', import.meta.url)); /** Base for sandbox URLs printed during runs */ const SANDBOX_DASHBOARD_URL = 'https://vercel.com/supabase/evals-runner/sandboxes'; -const AGENT_ENV_NAMES = [ +const FORWARDED_ENV_NAMES = [ 'ANTHROPIC_API_KEY', 'OPENAI_API_KEY', 'AI_GATEWAY_API_KEY', + // Pins the CLI channel versions the prepare job resolved for this run, so + // every sandbox job in the run scores against the same version instead of + // each independently re-resolving "latest" and drifting mid-run. + 'SUPABASE_CLI_STABLE_VERSION', + 'SUPABASE_CLI_BETA_VERSION', ]; /** * Slack for the non-agent work inside `pnpm eval` (supabase start, resets, @@ -543,10 +548,10 @@ function vercelCredentialsFromEnv(): { }; } -/** Serializes configured provider keys into the repo-root `.env` file. */ -function agentEnvironment(): string { +/** Serializes configured provider keys and CLI channel pins into the sandbox's `.env` file. */ +export function agentEnvironment(): string { const lines: string[] = []; - for (const name of AGENT_ENV_NAMES) { + for (const name of FORWARDED_ENV_NAMES) { const value = process.env[name]; if (value) lines.push(`${name}=${value}`); } diff --git a/apps/framework/tsconfig.json b/apps/framework/tsconfig.json index 0b21c008..6c6f48e7 100644 --- a/apps/framework/tsconfig.json +++ b/apps/framework/tsconfig.json @@ -1,10 +1,11 @@ { "extends": "../../tsconfig.base.json", "include": [ - "experiments", + "../../experiments", "harness", "shims", "scripts", "../../evals/**/EVAL.ts" - ] + ], + "exclude": ["../../experiments/**/*.test.ts"] } diff --git a/experiments/_lib/cli-channel.test.ts b/experiments/_lib/cli-channel.test.ts index 0c01c795..db61dafc 100644 --- a/experiments/_lib/cli-channel.test.ts +++ b/experiments/_lib/cli-channel.test.ts @@ -4,8 +4,18 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const STABLE_ENV = 'SUPABASE_CLI_STABLE_VERSION'; const BETA_ENV = 'SUPABASE_CLI_BETA_VERSION'; const DIST_TAGS_URL = 'https://registry.npmjs.org/-/package/supabase/dist-tags'; +const PACKUMENT_URL = 'https://registry.npmjs.org/supabase'; +const INSTALL_V1_ACCEPT = 'application/vnd.npm.install-v1+json'; -function distTagsResponse( +// Mirrors hostDebArch()'s mapping so this test's expected URLs match +// whatever host architecture actually runs it. +const HOST_ARCH = process.arch === 'arm64' ? 'arm64' : 'amd64'; + +function debUrl(version: string): string { + return `https://github.com/supabase/cli/releases/download/v${version}/supabase_${version}_linux_${HOST_ARCH}.deb`; +} + +function jsonResponse( body: unknown, init: { ok?: boolean; status?: number; statusText?: string } = {} ) { @@ -17,6 +27,39 @@ function distTagsResponse( }; } +function headResponse(ok: boolean) { + return { ok, status: ok ? 200 : 404, statusText: ok ? 'OK' : 'Not Found' }; +} + +/** + * Routes a single stubbed `fetch` by method + URL, mirroring the real + * mixture of GET (dist-tags, packument) and HEAD (asset check) calls + * resolveCliVersion makes. + */ +function routedFetchMock(routes: { + distTags?: unknown; + distTagsInit?: { ok?: boolean; status?: number; statusText?: string }; + assetOk?: (version: string) => boolean; + packument?: unknown; +}) { + return vi.fn(async (url: string, init?: RequestInit) => { + const method = init?.method ?? 'GET'; + if (method === 'HEAD') { + const version = url.match(/supabase_(.+)_linux_/)?.[1]; + return headResponse( + version ? (routes.assetOk?.(version) ?? false) : false + ); + } + if (url === DIST_TAGS_URL) { + return jsonResponse(routes.distTags, routes.distTagsInit); + } + if (url === PACKUMENT_URL) { + return jsonResponse(routes.packument); + } + throw new Error(`unexpected fetch: ${method} ${url}`); + }); +} + beforeEach(() => { vi.resetModules(); delete process.env[STABLE_ENV]; @@ -30,33 +73,32 @@ afterEach(() => { }); describe('resolveCliVersion', () => { - it('resolves the stable channel from the "latest" dist-tag', async () => { - const fetchMock = vi - .fn() - .mockResolvedValue( - distTagsResponse({ latest: '1.2.3', beta: '1.3.0-rc.1' }) - ); + it('resolves the stable channel from the "latest" dist-tag when its asset exists', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => true, + }); vi.stubGlobal('fetch', fetchMock); const { resolveCliVersion } = await import('./cli-channel.js'); await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); - expect(fetchMock).toHaveBeenCalledTimes(1); - const [url, options] = fetchMock.mock.calls[0]; - expect(url).toBe(DIST_TAGS_URL); - expect(options.signal).toBeInstanceOf(AbortSignal); + const headCalls = fetchMock.mock.calls.filter( + ([, init]) => init?.method === 'HEAD' + ); + expect(headCalls).toHaveLength(1); + expect(headCalls[0]?.[0]).toBe(debUrl('1.2.3')); }); - it('resolves the beta channel from the "beta" dist-tag', async () => { - const fetchMock = vi - .fn() - .mockResolvedValue( - distTagsResponse({ latest: '1.2.3', beta: '1.3.0-rc.1' }) - ); + it('resolves the beta channel from the "beta" dist-tag when its asset exists', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => true, + }); vi.stubGlobal('fetch', fetchMock); const { resolveCliVersion } = await import('./cli-channel.js'); - await expect(resolveCliVersion('beta')).resolves.toBe('1.3.0-rc.1'); + await expect(resolveCliVersion('beta')).resolves.toBe('1.3.0-beta.1'); }); it('prefers the env override over the network and strips a leading v', async () => { @@ -88,13 +130,14 @@ describe('resolveCliVersion', () => { }); it('throws with the HTTP status when the registry request fails', async () => { - const fetchMock = vi.fn().mockResolvedValue( - distTagsResponse(undefined, { + const fetchMock = routedFetchMock({ + distTags: undefined, + distTagsInit: { ok: false, status: 500, statusText: 'Internal Server Error', - }) - ); + }, + }); vi.stubGlobal('fetch', fetchMock); const { resolveCliVersion } = await import('./cli-channel.js'); @@ -104,9 +147,7 @@ describe('resolveCliVersion', () => { }); it('throws when the requested dist-tag is missing from the response', async () => { - const fetchMock = vi - .fn() - .mockResolvedValue(distTagsResponse({ beta: '1.0.0' })); + const fetchMock = routedFetchMock({ distTags: { beta: '1.0.0-beta.1' } }); vi.stubGlobal('fetch', fetchMock); const { resolveCliVersion } = await import('./cli-channel.js'); @@ -116,9 +157,9 @@ describe('resolveCliVersion', () => { }); it('throws when the dist-tag value is not a valid version string', async () => { - const fetchMock = vi - .fn() - .mockResolvedValue(distTagsResponse({ latest: 'not-a-version' })); + const fetchMock = routedFetchMock({ + distTags: { latest: 'not-a-version' }, + }); vi.stubGlobal('fetch', fetchMock); const { resolveCliVersion } = await import('./cli-channel.js'); @@ -128,60 +169,143 @@ describe('resolveCliVersion', () => { }); it('memoises a successful resolution so a second call does not refetch', async () => { - const fetchMock = vi - .fn() - .mockResolvedValue(distTagsResponse({ latest: '1.2.3', beta: '1.3.0' })); + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => true, + }); vi.stubGlobal('fetch', fetchMock); const { resolveCliVersion } = await import('./cli-channel.js'); await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); - expect(fetchMock).toHaveBeenCalledTimes(1); + // One GET (dist-tags) + one HEAD (asset check) — the second call hits the cache. + expect(fetchMock).toHaveBeenCalledTimes(2); }); it('clears the cache entry on rejection so a later call refetches', async () => { - const fetchMock = vi - .fn() - .mockResolvedValueOnce( - distTagsResponse(undefined, { - ok: false, - status: 500, - statusText: 'Internal Server Error', - }) - ) - .mockResolvedValueOnce( - distTagsResponse({ latest: '1.2.3', beta: '1.3.0' }) - ); + let failNextDistTags = true; + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + if (init?.method === 'HEAD') return headResponse(true); + if (url === DIST_TAGS_URL) { + if (failNextDistTags) { + failNextDistTags = false; + return jsonResponse(undefined, { + ok: false, + status: 500, + statusText: 'Internal Server Error', + }); + } + return jsonResponse({ latest: '1.2.3', beta: '1.3.0-beta.1' }); + } + throw new Error(`unexpected fetch: ${url}`); + }); vi.stubGlobal('fetch', fetchMock); const { resolveCliVersion } = await import('./cli-channel.js'); await expect(resolveCliVersion('stable')).rejects.toThrow('500'); await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); - expect(fetchMock).toHaveBeenCalledTimes(2); + expect(fetchMock).toHaveBeenCalledTimes(3); }); it('does not let a beta rejection clear the stable cache entry', async () => { - const fetchMock = vi - .fn() - .mockResolvedValueOnce( - distTagsResponse({ latest: '1.2.3', beta: '1.3.0' }) - ) - .mockResolvedValueOnce( - distTagsResponse(undefined, { - ok: false, - status: 500, - statusText: 'Internal Server Error', - }) - ); + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + if (init?.method === 'HEAD') return headResponse(true); + if (url === DIST_TAGS_URL) { + return jsonResponse({ latest: '1.2.3', beta: 'not-a-version' }); + } + throw new Error(`unexpected fetch: ${url}`); + }); vi.stubGlobal('fetch', fetchMock); const { resolveCliVersion } = await import('./cli-channel.js'); await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); - await expect(resolveCliVersion('beta')).rejects.toThrow('500'); + await expect(resolveCliVersion('beta')).rejects.toThrow( + /did not have a valid "beta" version for the beta channel/ + ); await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); - expect(fetchMock).toHaveBeenCalledTimes(2); + // Two dist-tags GETs (stable, beta) + one HEAD (stable's asset check + // only — beta never gets that far). + expect(fetchMock).toHaveBeenCalledTimes(3); + }); + + it('falls back to the newest published beta.N-1 when the dist-tag asset is a 404', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '2.118.0-beta.52' }, + assetOk: (version) => version !== '2.118.0-beta.52', + packument: { + versions: { + '2.118.0-beta.52': {}, + '2.118.0-beta.51': {}, + '2.118.0-beta.50': {}, + '2.117.0': {}, + }, + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('./cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.51'); + + const packumentCall = fetchMock.mock.calls.find( + ([url]) => url === PACKUMENT_URL + ); + expect(packumentCall?.[1]?.headers).toMatchObject({ + Accept: INSTALL_V1_ACCEPT, + }); + + const headUrls = fetchMock.mock.calls + .filter(([, init]) => init?.method === 'HEAD') + .map(([url]) => url); + expect(headUrls).toEqual([ + debUrl('2.118.0-beta.52'), + debUrl('2.118.0-beta.51'), + ]); + }); + + it('throws naming the unpublished versions when no published beta has a downloadable asset', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '2.118.0-beta.52' }, + assetOk: () => false, + packument: { + versions: { + '2.118.0-beta.52': {}, + '2.118.0-beta.51': {}, + }, + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('./cli-channel.js'); + + await expect(resolveCliVersion('beta')).rejects.toThrow( + /2\.118\.0-beta\.52.*2\.118\.0-beta\.51/ + ); + }); + + it('throws instead of guessing when the stable dist-tag asset is a 404', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => false, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('./cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow( + `HEAD ${debUrl('1.2.3')} was not ok` + ); + }); + + it('does not hit the network at all for an env override', async () => { + process.env[STABLE_ENV] = '9.9.9'; + process.env[BETA_ENV] = '9.9.9-beta.1'; + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('./cli-channel.js'); + + await expect(resolveCliVersion('stable')).resolves.toBe('9.9.9'); + await expect(resolveCliVersion('beta')).resolves.toBe('9.9.9-beta.1'); + expect(fetchMock).not.toHaveBeenCalled(); }); }); diff --git a/experiments/_lib/cli-channel.ts b/experiments/_lib/cli-channel.ts index b3ea2e58..8bdc0219 100644 --- a/experiments/_lib/cli-channel.ts +++ b/experiments/_lib/cli-channel.ts @@ -3,6 +3,14 @@ * dist-tags for the `supabase` package — no auth, no rate-limit exposure * (unlike the GitHub releases API), which matters because CI's sandbox only * forwards the model API keys, never a GITHUB_TOKEN. + * + * npm's dist-tag can point at a version whose GitHub release is still a + * draft, so the resolved version's `.deb` release asset is HEAD-checked + * before being trusted; for the beta channel only, a missing asset walks + * back through older published `-beta.N` versions for one that downloads + * (see resolveFallbackBetaVersion). A stable dist-tag with a missing asset + * always throws instead — guessing at a "stable" release would defeat the + * point of the channel. */ export type CliChannel = 'stable' | 'beta'; @@ -10,9 +18,24 @@ export type CliChannel = 'stable' | 'beta'; const NPM_DIST_TAGS_URL = 'https://registry.npmjs.org/-/package/supabase/dist-tags'; +// Abbreviated packument (versions + dist-tags only, no changelogs/READMEs) — +// this header is what makes npm serve the small form instead of the full one. +const NPM_PACKUMENT_URL = 'https://registry.npmjs.org/supabase'; +const NPM_INSTALL_V1_ACCEPT = 'application/vnd.npm.install-v1+json'; + // npm's beta dist-tag can carry a prerelease suffix like -rc.1, not just -beta.N. const VERSION_RE = /^\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?$/; +// Only the -beta.N shape is walkable for the "try an older published beta" +// fallback below; a -rc.N or other prerelease suffix has no defined ordering +// we can search. +const BETA_SUFFIX_RE = /^(\d+\.\d+\.\d+-beta\.)(\d+)$/; + +// The number of older published beta versions to probe for a downloadable +// asset before giving up — keeps a broken release from turning one nightly +// run into an unbounded chain of GitHub requests. +const MAX_BETA_FALLBACK_CANDIDATES = 5; + const ENV_OVERRIDE: Record = { stable: 'SUPABASE_CLI_STABLE_VERSION', beta: 'SUPABASE_CLI_BETA_VERSION', @@ -25,6 +48,29 @@ const DIST_TAG: Record = { const versionCache = new Map>(); +/** Arch suffix the CLI's own release `.deb` filenames use. */ +export function hostDebArch(): 'amd64' | 'arm64' { + return process.arch === 'arm64' ? 'arm64' : 'amd64'; +} + +/** Mirrors installSupabaseCli's download URL template in packages/sandbox/src/supabase.ts. */ +export function cliDebUrl( + version: string, + arch: 'amd64' | 'arm64' = hostDebArch() +): string { + return `https://github.com/supabase/cli/releases/download/v${version}/supabase_${version}_linux_${arch}.deb`; +} + +/** HEAD-checks that a release's `.deb` asset is actually downloadable (not behind a draft release). */ +async function debAssetExists(version: string): Promise { + const response = await fetch(cliDebUrl(version), { + method: 'HEAD', + redirect: 'follow', + signal: AbortSignal.timeout(15_000), + }); + return response.ok; +} + /** * Resolve a channel to a concrete Supabase CLI version. Memoised per channel * so a single nightly run only hits the registry once per channel; the cache @@ -51,6 +97,7 @@ async function resolveCliVersionUncached(channel: CliChannel): Promise { `${envVar}=${JSON.stringify(override)} is not a valid Supabase CLI version` ); } + // An explicit pin is trusted as-is — no asset check, network or otherwise. return override; } @@ -72,7 +119,89 @@ async function resolveCliVersionUncached(channel: CliChannel): Promise { `for the ${channel} channel: ${JSON.stringify(version)}` ); } - return version; + + if (await debAssetExists(version)) return version; + + // The dist-tag pointed at a version whose GitHub release has no + // downloadable .deb (e.g. still a draft) — this is exactly what broke the + // beta channel in CI run 35274970438. A stable release is never guessed at; + // only beta walks back to the newest published version that does have one. + if (channel === 'stable') { + throw new Error( + `npm's "latest" dist-tag for "supabase" points at ${version}, but its ` + + `release asset is missing: HEAD ${cliDebUrl(version)} was not ok` + ); + } + + return resolveFallbackBetaVersion(version); +} + +/** + * Walks back through published npm versions sharing the same `X.Y.Z-beta.` + * prefix as `unpublishedVersion`, newest first, and returns the first one + * whose release `.deb` asset actually downloads. + */ +async function resolveFallbackBetaVersion( + unpublishedVersion: string +): Promise { + const match = BETA_SUFFIX_RE.exec(unpublishedVersion); + if (!match) { + throw new Error( + `npm's "beta" dist-tag for "supabase" points at ${unpublishedVersion}, ` + + `whose release asset is missing (HEAD ${cliDebUrl(unpublishedVersion)} ` + + 'was not ok), and its version does not match the X.Y.Z-beta.N shape ' + + 'this fallback can walk back through' + ); + } + const [, prefix] = match; + + const response = await fetch(NPM_PACKUMENT_URL, { + headers: { Accept: NPM_INSTALL_V1_ACCEPT }, + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) { + throw new Error( + `failed to look up published beta versions for "supabase": ` + + `GET ${NPM_PACKUMENT_URL} -> ${response.status} ${response.statusText}` + ); + } + const packument = await response.json(); + const versions = isRecord(packument) ? packument.versions : undefined; + if (!isRecord(versions)) { + throw new Error( + `npm packument for "supabase" did not include a "versions" object` + ); + } + + const suffixRe = new RegExp(`^${escapeRegExp(prefix)}(\\d+)$`); + const candidates = Object.keys(versions) + .filter((candidate) => candidate !== unpublishedVersion) + .map((candidate) => { + const suffixMatch = suffixRe.exec(candidate); + return suffixMatch + ? { version: candidate, suffix: Number(suffixMatch[1]) } + : null; + }) + .filter((entry): entry is { version: string; suffix: number } => + Boolean(entry) + ) + .sort((a, b) => b.suffix - a.suffix) + .slice(0, MAX_BETA_FALLBACK_CANDIDATES) + .map((entry) => entry.version); + + for (const candidate of candidates) { + if (await debAssetExists(candidate)) return candidate; + } + + throw new Error( + `no published beta Supabase CLI release has a downloadable .deb asset; ` + + `checked ${[unpublishedVersion, ...candidates].join(', ')} via ` + + `${NPM_PACKUMENT_URL}` + ); +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); } function isRecord(value: unknown): value is Record { diff --git a/experiments/_lib/docker-aware-local-stack.test.ts b/experiments/_lib/docker-aware-local-stack.test.ts index 2eebf341..c7bf86cd 100644 --- a/experiments/_lib/docker-aware-local-stack.test.ts +++ b/experiments/_lib/docker-aware-local-stack.test.ts @@ -1,12 +1,22 @@ // Run: pnpm --filter @supabase-evals/framework exec vitest run --root ../.. experiments/_lib +import type { EvalMetadata } from '@supabase-evals/core'; import { describe, expect, it } from 'vitest'; import type { SupabaseService } from '@supabase-evals/sandbox'; import { buildDockerDaemonShimScript, buildSupabaseShimScript, dockerAwareLocalStackRuntime, + skipUnlessCli, + skipUnlessDockerless, } from './docker-aware-local-stack.js'; +const baseMetadata: EvalMetadata = { + stage: 'build', + product: ['database'], + topic: ['sdk'], + interface: 'cli', +}; + describe('dockerAwareLocalStackRuntime', () => { it('ids as `local-stack-cli-` when docker is unset (available)', () => { expect(dockerAwareLocalStackRuntime({ channel: 'stable' }).id).toBe( @@ -21,6 +31,80 @@ describe('dockerAwareLocalStackRuntime', () => { }); }); +describe('skipUnlessCli', () => { + it('runs a cli eval that is not hosted-linked', () => { + expect(skipUnlessCli({ id: 'e', metadata: baseMetadata })).toBe(false); + }); + + it('skips a non-cli eval', () => { + expect( + skipUnlessCli({ + id: 'e', + metadata: { ...baseMetadata, interface: 'mcp' }, + }) + ).toBe(true); + }); + + it('skips a hosted-linked eval', () => { + expect( + skipUnlessCli({ + id: 'e', + metadata: { ...baseMetadata, hostedProject: true }, + }) + ).toBe(true); + }); + + it('runs a cli eval with hostedProject explicitly false', () => { + expect( + skipUnlessCli({ + id: 'e', + metadata: { ...baseMetadata, hostedProject: false }, + }) + ).toBe(false); + }); +}); + +describe('skipUnlessDockerless', () => { + const dockerlessMetadata: EvalMetadata = { + ...baseMetadata, + needsDocker: false, + projectRunning: false, + }; + + it('runs a cli eval that needs no Docker and has no pre-started stack', () => { + expect( + skipUnlessDockerless({ id: 'e', metadata: dockerlessMetadata }) + ).toBe(false); + }); + + it('skips a non-cli eval', () => { + expect( + skipUnlessDockerless({ + id: 'e', + metadata: { ...dockerlessMetadata, interface: 'mcp' }, + }) + ).toBe(true); + }); + + it('skips an eval that needs Docker', () => { + expect( + skipUnlessDockerless({ + id: 'e', + metadata: { ...dockerlessMetadata, needsDocker: true }, + }) + ).toBe(true); + }); + + it('skips an eval whose stack is already running', () => { + expect( + skipUnlessDockerless({ + id: 'e', + metadata: { ...dockerlessMetadata, projectRunning: true }, + }) + ).toBe(true); + }); +}); + describe('buildSupabaseShimScript', () => { it('emits DOCKER_HOST, the -x start branch for excluded services, and a passthrough exec', () => { const excluded: SupabaseService[] = ['gotrue', 'kong']; @@ -28,7 +112,7 @@ describe('buildSupabaseShimScript', () => { [ '#!/bin/bash', 'export DOCKER_HOST=tcp://127.0.0.1:1', - 'REAL="/usr/bin/supabase"', + "REAL='/usr/bin/supabase'", 'if [ "$1" = "start" ]; then shift; exec "$REAL" start "$@" -x gotrue,kong; fi', 'exec "$REAL" "$@"', ].join('\n') @@ -41,7 +125,7 @@ describe('buildSupabaseShimScript', () => { ).toMatchInlineSnapshot(` "#!/bin/bash export DOCKER_HOST=tcp://127.0.0.1:1 - REAL="/usr/bin/supabase" + REAL='/usr/bin/supabase' exec "$REAL" "$@"" `); }); @@ -52,10 +136,18 @@ describe('buildSupabaseShimScript', () => { expect(script.endsWith('exec "$REAL" "$@"')).toBe(true); }); - it('JSON-quotes a realBin containing a single quote safely for bash double-quoting', () => { + it('single-quotes a realBin containing a single quote safely for the shell', () => { const script = buildSupabaseShimScript("/usr/local/it's/bin/supabase", []); - expect(script).toContain(`REAL="/usr/local/it's/bin/supabase"`); - expect(script).not.toContain('REAL=/usr/local/it'); + expect(script).toContain(`REAL='/usr/local/it'\\''s/bin/supabase'`); + }); + + it('single-quotes a realBin containing $, backtick, and " safely for the shell', () => { + const realBin = '/usr/local/$(whoami)/`id`/"bin"/supabase'; + const script = buildSupabaseShimScript(realBin, []); + const realLine = script + .split('\n') + .find((line) => line.startsWith('REAL=')); + expect(realLine).toBe('REAL=\'/usr/local/$(whoami)/`id`/"bin"/supabase\''); }); }); @@ -66,10 +158,22 @@ describe('buildDockerDaemonShimScript', () => { expect(buildDockerDaemonShimScript(dockerVersion)).toMatchInlineSnapshot(` "#!/bin/bash case "$1" in - --version|-v) echo "Docker version 20.10.24+dfsg1, build 297e128"; exit 0 ;; + --version|-v) echo 'Docker version 20.10.24+dfsg1, build 297e128'; exit 0 ;; esac echo "Cannot connect to the Docker daemon at tcp://127.0.0.1:1. Is the docker daemon running?" >&2 exit 1" `); }); + + it('single-quotes a version string containing a single quote safely for the shell', () => { + const script = buildDockerDaemonShimScript( + "Docker version 'weird', build x" + ); + const versionLine = script + .split('\n') + .find((line) => line.includes('--version|-v')); + expect(versionLine).toBe( + ` --version|-v) echo 'Docker version '\\''weird'\\'', build x'; exit 0 ;;` + ); + }); }); diff --git a/experiments/_lib/docker-aware-local-stack.ts b/experiments/_lib/docker-aware-local-stack.ts index fcd48e7a..d94c7c7b 100644 --- a/experiments/_lib/docker-aware-local-stack.ts +++ b/experiments/_lib/docker-aware-local-stack.ts @@ -10,6 +10,7 @@ import { supabaseMcpServer, + type EvalMetadata, type LocalStackRuntime, type LocalStackSession, type LocalStackSessionArgs, @@ -25,12 +26,31 @@ import { installSkills, installSupabaseCli, localStackRuntime, - teardownSupabaseProject, toAgentSandbox, type SupabaseService, } from '@supabase-evals/sandbox'; import { resolveCliVersion, type CliChannel } from './cli-channel.js'; +/** Runs only evals that exercise the real CLI (never hosted-linked ones, which seed .temp pins instead). */ +export function skipUnlessCli(ev: { + id: string; + metadata: EvalMetadata; +}): boolean { + return ev.metadata.interface !== 'cli' || ev.metadata.hostedProject === true; +} + +/** A Docker-less sandbox can only run evals that declare they need no Docker and don't expect a pre-started stack. */ +export function skipUnlessDockerless(ev: { + id: string; + metadata: EvalMetadata; +}): boolean { + return ( + skipUnlessCli(ev) || + ev.metadata.needsDocker !== false || + ev.metadata.projectRunning !== false + ); +} + export type DockerState = 'available' | 'no-daemon' | 'absent'; // CLI eval scorers duplicate this schema rather than import it, so evals stay @@ -74,12 +94,18 @@ export function dockerAwareLocalStackRuntime(options: { // An eval's own `cliVersion:` frontmatter still wins over the channel, // same precedence as the stock local-stack runtime. const version = args.cliVersion ?? (await resolveCliVersion(channel)); + console.log( + `[docker-aware-local-stack] channel=${channel} cliVersion=${version} docker=${state}` + ); if (state === 'available') { const session = await localStackRuntime({ cliVersion: version, }).startSession(args); try { + // The scoring context's exec has no root access inside the + // sandbox, so this marker is agent-writable — fine here since it's + // metrics-only, unlike the docker-less path's root-owned marker. await writeRuntimeMarker( (command) => session.scoringContext.exec(command), buildRuntimeMarker(channel, version, state, sessionStartedMs) @@ -107,6 +133,7 @@ export function dockerAwareLocalStackRuntime(options: { image, network: 'host', mounts: args.mounts, + mountDockerSocket: false, }); try { @@ -117,6 +144,9 @@ export function dockerAwareLocalStackRuntime(options: { // Deliberately no socket-group grant here: CI's sandbox already ends // its Docker setup with `chmod 666 /var/run/docker.sock`, so the grant // would be a no-op there — DOCKER_HOST below is the real mechanism. + // For `absent`, a real Docker-less host wouldn't have DOCKER_HOST set + // at all — but leaving it set here costs nothing and blocks any + // future accidental socket exposure, so it stays for both states. sandbox.extraEnv = { ...sandbox.extraEnv, DOCKER_HOST: UNREACHABLE_DOCKER_HOST, @@ -150,6 +180,18 @@ export function dockerAwareLocalStackRuntime(options: { ); } + // The root shell above has a different PATH than the agent's + // SANDBOX_PATH, so also assert the binary is gone from the PATH the + // agent actually runs commands under. + const pathCheck = await sandbox.runShell( + '! command -v docker >/dev/null 2>&1' + ); + if (!pathCheck.ok) { + throw new Error( + `docker is still on the agent's PATH after removal: ${pathCheck.stderr || pathCheck.stdout}` + ); + } + if (state === 'no-daemon') { await installDockerDaemonShim(sandbox, dockerVersion); } @@ -159,18 +201,25 @@ export function dockerAwareLocalStackRuntime(options: { } const skills = await installSkills(sandbox, args.skills ?? []); - const ping = await sandbox.runShell( - 'curl -sf --unix-socket /var/run/docker.sock http://localhost/_ping' + // With no bind mount, the socket must not exist at all — a stronger + // guarantee than the old warn-probe that merely checked reachability. + const socketAbsent = await sandbox.runShellAsRoot( + 'test ! -e /var/run/docker.sock' ); - if (ping.ok) { - console.warn( - '[docker-aware-local-stack] raw docker socket is reachable by the sandbox user; relying on DOCKER_HOST + shims' + if (!socketAbsent.ok) { + throw new Error( + 'the Docker socket unexpectedly exists in a Docker-less sandbox' ); } - await writeRuntimeMarker( - (command) => sandbox.runShell(command), - buildRuntimeMarker(channel, version, state, sessionStartedMs) + // Root-owned and read-only so the agent cannot rewrite it to fake + // the environment it's being evaluated in. + await sandbox.writeRootFile( + RUNTIME_MARKER_PATH, + JSON.stringify( + buildRuntimeMarker(channel, version, state, sessionStartedMs) + ), + '0444' ); return { @@ -197,10 +246,8 @@ export function dockerAwareLocalStackRuntime(options: { // Nothing to restore without Docker; the `available` path above // delegates to the stock session, which has its own ensureReady. ensureReady: async () => {}, - close: async () => { - await teardownSupabaseProject(sandbox); - await sandbox.stop(); - }, + // No teardownSupabaseProject: nothing can have started without Docker. + close: () => sandbox.stop(), }; } catch (err) { await sandbox.stop(); @@ -272,7 +319,7 @@ export function buildSupabaseShimScript( const lines = [ '#!/bin/bash', `export DOCKER_HOST=${UNREACHABLE_DOCKER_HOST}`, - `REAL=${JSON.stringify(realBin)}`, + `REAL=${shellQuote(realBin)}`, ]; if (excluded.length > 0) { lines.push( @@ -300,9 +347,14 @@ export function buildDockerDaemonShimScript(dockerVersion: string): string { 'case "$1" in', // --version must keep working so the CLI's new managed stack still // *chooses* Docker as its runtime (per supabase/cli#6563's probe). - ' --version|-v) echo ' + JSON.stringify(dockerVersion) + '; exit 0 ;;', + ` --version|-v) echo ${shellQuote(dockerVersion)}; exit 0 ;;`, 'esac', `echo "Cannot connect to the Docker daemon at ${UNREACHABLE_DOCKER_HOST}. Is the docker daemon running?" >&2`, 'exit 1', ].join('\n'); } + +// Same implementation as packages/sandbox/src/local-stack-runtime.ts's. +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'\\''`)}'`; +} diff --git a/experiments/codex-gpt-5.6-luna-cli-absent.ts b/experiments/codex-gpt-5.6-luna-cli-absent.ts index 37b12af4..d4677eb8 100644 --- a/experiments/codex-gpt-5.6-luna-cli-absent.ts +++ b/experiments/codex-gpt-5.6-luna-cli-absent.ts @@ -4,7 +4,10 @@ import { platformLiteRuntime, supabaseMcpServer, } from '@supabase-evals/core'; -import { dockerAwareLocalStackRuntime } from './_lib/docker-aware-local-stack.js'; +import { + dockerAwareLocalStackRuntime, + skipUnlessDockerless, +} from './_lib/docker-aware-local-stack.js'; export default defineExperiment({ suite: ['cli'], @@ -21,6 +24,5 @@ export default defineExperiment({ docker: 'absent', }), skills: ['supabase', 'supabase-postgres-best-practices'], - // A Docker-less sandbox can only run evals that declare they don't need Docker. - skipEval: (ev) => ev.metadata.needsDocker !== false, + skipEval: skipUnlessDockerless, }); diff --git a/experiments/codex-gpt-5.6-luna-cli-beta.ts b/experiments/codex-gpt-5.6-luna-cli-beta.ts index b363b32c..3035c6a8 100644 --- a/experiments/codex-gpt-5.6-luna-cli-beta.ts +++ b/experiments/codex-gpt-5.6-luna-cli-beta.ts @@ -4,7 +4,10 @@ import { platformLiteRuntime, supabaseMcpServer, } from '@supabase-evals/core'; -import { dockerAwareLocalStackRuntime } from './_lib/docker-aware-local-stack.js'; +import { + dockerAwareLocalStackRuntime, + skipUnlessCli, +} from './_lib/docker-aware-local-stack.js'; export default defineExperiment({ suite: ['cli'], @@ -17,8 +20,5 @@ export default defineExperiment({ }), localStack: dockerAwareLocalStackRuntime({ channel: 'beta' }), skills: ['supabase', 'supabase-postgres-best-practices'], - // Only CLI evals exercise the installed CLI version; hosted evals seed .temp - // version files pinned to the baseline CLI's service versions. - skipEval: (ev) => - ev.metadata.interface !== 'cli' || ev.metadata.hostedProject === true, + skipEval: skipUnlessCli, }); diff --git a/experiments/codex-gpt-5.6-luna-cli-nodaemon.ts b/experiments/codex-gpt-5.6-luna-cli-nodaemon.ts index 60acc4b6..796dc9ef 100644 --- a/experiments/codex-gpt-5.6-luna-cli-nodaemon.ts +++ b/experiments/codex-gpt-5.6-luna-cli-nodaemon.ts @@ -4,7 +4,10 @@ import { platformLiteRuntime, supabaseMcpServer, } from '@supabase-evals/core'; -import { dockerAwareLocalStackRuntime } from './_lib/docker-aware-local-stack.js'; +import { + dockerAwareLocalStackRuntime, + skipUnlessDockerless, +} from './_lib/docker-aware-local-stack.js'; export default defineExperiment({ suite: ['cli'], @@ -21,6 +24,5 @@ export default defineExperiment({ docker: 'no-daemon', }), skills: ['supabase', 'supabase-postgres-best-practices'], - // A Docker-less sandbox can only run evals that declare they don't need Docker. - skipEval: (ev) => ev.metadata.needsDocker !== false, + skipEval: skipUnlessDockerless, }); diff --git a/experiments/codex-gpt-5.6-luna-cli-stable.ts b/experiments/codex-gpt-5.6-luna-cli-stable.ts index 237dc161..0c59de7d 100644 --- a/experiments/codex-gpt-5.6-luna-cli-stable.ts +++ b/experiments/codex-gpt-5.6-luna-cli-stable.ts @@ -4,7 +4,10 @@ import { platformLiteRuntime, supabaseMcpServer, } from '@supabase-evals/core'; -import { dockerAwareLocalStackRuntime } from './_lib/docker-aware-local-stack.js'; +import { + dockerAwareLocalStackRuntime, + skipUnlessCli, +} from './_lib/docker-aware-local-stack.js'; export default defineExperiment({ suite: ['cli'], @@ -15,10 +18,9 @@ export default defineExperiment({ runtime: platformLiteRuntime({ mcpServers: [supabaseMcpServer()], }), + // Currently equal to the pin (npm `latest` == SUPABASE_CLI_VERSION); kept + // as drift insurance between pin bumps. localStack: dockerAwareLocalStackRuntime({ channel: 'stable' }), skills: ['supabase', 'supabase-postgres-best-practices'], - // Only CLI evals exercise the installed CLI version; hosted evals seed .temp - // version files pinned to the baseline CLI's service versions. - skipEval: (ev) => - ev.metadata.interface !== 'cli' || ev.metadata.hostedProject === true, + skipEval: skipUnlessCli, }); diff --git a/packages/sandbox/src/docker-sandbox.ts b/packages/sandbox/src/docker-sandbox.ts index 360cd67b..1825a463 100644 --- a/packages/sandbox/src/docker-sandbox.ts +++ b/packages/sandbox/src/docker-sandbox.ts @@ -87,6 +87,13 @@ export interface DockerSandboxOptions { * tools must execute — e.g. a local MCP server build. */ mounts?: readonly SandboxMount[]; + /** + * Bind-mount the host Docker socket into the container. Defaults to true. + * Set false for sandboxes that must be provably Docker-less — the socket + * is never exposed to the container at all, rather than merely hidden or + * blocked from inside it. + */ + mountDockerSocket?: boolean; } export interface RunCommandOptions { @@ -100,6 +107,7 @@ export class DockerSandbox { private network: string | undefined; private image: string; private mounts: readonly SandboxMount[]; + private mountDockerSocket: boolean; readonly workdir: string; /** * Env vars injected into every `runShell` (non-root) command — both the @@ -113,6 +121,7 @@ export class DockerSandbox { this.network = options.network; this.image = options.image ?? DEFAULT_IMAGE; this.mounts = options.mounts ?? []; + this.mountDockerSocket = options.mountDockerSocket !== false; this.workdir = `${WORKSPACE_BASE}-${randomUUID().slice(0, 8)}`; } @@ -141,8 +150,9 @@ export class DockerSandbox { '--rm', '--label', `${SANDBOX_CONTAINER_LABEL}=1`, - '--volume', - '/var/run/docker.sock:/var/run/docker.sock', + ...(this.mountDockerSocket + ? ['--volume', '/var/run/docker.sock:/var/run/docker.sock'] + : []), '--volume', `${this.workdir}:${this.workdir}`, // Caller-requested host mounts (e.g. a local MCP server build the diff --git a/packages/sandbox/test/unit.test.ts b/packages/sandbox/test/unit.test.ts index 029d6b8a..1aaa284d 100644 --- a/packages/sandbox/test/unit.test.ts +++ b/packages/sandbox/test/unit.test.ts @@ -636,6 +636,43 @@ describe('skipCliInstall frontmatter', () => { }); }); +describe('needsDocker frontmatter', () => { + const buildMarkdown = (extra: string) => + [ + '---', + 'stage: build', + 'interface: cli', + 'product: [database]', + 'topic: [sdk]', + extra, + '---', + 'body', + ].join('\n'); + + it('accepts a real boolean true and false', () => { + expect( + parseEvalMarkdown(buildMarkdown('needsDocker: true')).metadata.needsDocker + ).toBe(true); + expect( + parseEvalMarkdown(buildMarkdown('needsDocker: false')).metadata + .needsDocker + ).toBe(false); + }); + + it('accepts a quoted string form via z.stringbool()', () => { + expect( + parseEvalMarkdown(buildMarkdown('needsDocker: "false"')).metadata + .needsDocker + ).toBe(false); + }); + + it('defaults to undefined when omitted', () => { + expect( + parseEvalMarkdown(buildMarkdown('')).metadata.needsDocker + ).toBeUndefined(); + }); +}); + describe('resolveSandboxPath', () => { it('accepts and normalizes relative paths', () => { expect(resolveSandboxPath('a/b.txt')).toBe('a/b.txt');