From 1a4b8892884f906381f272487a2bae38ea841c15 Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Tue, 22 Sep 2026 10:16:41 +0100 Subject: [PATCH 01/12] feat(sandbox): resolve npm channel tags for the local-stack CLI version Moves the CLI channel resolver into @supabase-evals/sandbox and lets localStackRuntime's cliVersion accept 'stable'/'beta' in addition to an exact version, resolved against npm's dist-tags at session start. An eval's own cliVersion: frontmatter pin still wins. Forwards the resolved channel pins to Vercel sandbox jobs so a run scores against one version instead of each job re-resolving "latest" independently. --- README.md | 2 + .../scripts/run-vercel-evals.test.ts | 49 ++- apps/framework/scripts/run-vercel-evals.ts | 13 +- packages/sandbox/package.json | 2 +- packages/sandbox/src/cli-channel.ts | 228 ++++++++++++ packages/sandbox/src/index.ts | 2 + packages/sandbox/src/local-stack-runtime.ts | 15 +- packages/sandbox/test/cli-channel.test.ts | 352 ++++++++++++++++++ 8 files changed, 654 insertions(+), 9 deletions(-) create mode 100644 packages/sandbox/src/cli-channel.ts create mode 100644 packages/sandbox/test/cli-channel.test.ts diff --git a/README.md b/README.md index 52a84349..adbaf62a 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,8 @@ An eval's optional `local/` directory is copied into the sandbox workspace befor Set `cliVersion: 2.109.1` in an eval's frontmatter when it requires a specific Supabase CLI release. This overrides an experiment's `localStackRuntime({ cliVersion })` setting; otherwise the runtime setting or repository-wide default applies. +An experiment can instead pass `localStackRuntime({ cliVersion: 'stable' })` or `'beta'` to track npm's dist-tag for the `supabase` package rather than an exact version, resolved once at session start. An eval's own `cliVersion:` pin still wins over either form. + Scorers check what the agent produced, never what the harness provisioned: with `projectRunning: true` (the default) the running stack and the seeded `local/` workspace are setup, so score only the deltas the agent made on top; with `projectRunning: false` the agent creates that state itself, so depending on it is fair game. Test the sandbox plumbing without an agent run (Docker required, not part of `pnpm check`): diff --git a/apps/framework/scripts/run-vercel-evals.test.ts b/apps/framework/scripts/run-vercel-evals.test.ts index 1190490a..fe1b9806 100644 --- a/apps/framework/scripts/run-vercel-evals.test.ts +++ b/apps/framework/scripts/run-vercel-evals.test.ts @@ -11,8 +11,9 @@ import { } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; -import { describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { + agentEnvironment, cleanupSandbox, downloadResults, finalizeResult, @@ -25,6 +26,52 @@ import { expandJobs, } from './run-vercel-evals.js'; +const FORWARDED_ENV_NAMES = [ + 'ANTHROPIC_API_KEY', + 'OPENAI_API_KEY', + 'AI_GATEWAY_API_KEY', + 'SUPABASE_CLI_STABLE_VERSION', + 'SUPABASE_CLI_BETA_VERSION', +]; + +describe('agentEnvironment', () => { + const originalValues = new Map(); + + beforeEach(() => { + for (const name of FORWARDED_ENV_NAMES) { + originalValues.set(name, process.env[name]); + delete process.env[name]; + } + }); + + afterEach(() => { + for (const [name, value] of originalValues) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + }); + + it('forwards every configured name when set', () => { + for (const name of FORWARDED_ENV_NAMES) { + process.env[name] = `${name}-value`; + } + + const lines = agentEnvironment().split('\n'); + for (const name of FORWARDED_ENV_NAMES) { + expect(lines).toContain(`${name}=${name}-value`); + } + }); + + it('omits the CLI channel pins when unset', () => { + process.env.ANTHROPIC_API_KEY = 'anthropic-value'; + + const env = agentEnvironment(); + expect(env).toBe('ANTHROPIC_API_KEY=anthropic-value'); + expect(env).not.toContain('SUPABASE_CLI_STABLE_VERSION'); + expect(env).not.toContain('SUPABASE_CLI_BETA_VERSION'); + }); +}); + describe('Vercel eval controller', () => { it('bounds concurrent work and lets independent failures settle', async () => { let active = 0; diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index 221cdcaa..efcbcc14 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -19,11 +19,16 @@ const ROOT = fileURLToPath(new URL('../../../', import.meta.url)); /** Base for sandbox URLs printed during runs */ const SANDBOX_DASHBOARD_URL = 'https://vercel.com/supabase/evals-runner/sandboxes'; -const AGENT_ENV_NAMES = [ +const FORWARDED_ENV_NAMES = [ 'ANTHROPIC_API_KEY', 'OPENAI_API_KEY', 'AI_GATEWAY_API_KEY', 'XAI_API_KEY', + // Pins the CLI channel versions the prepare job resolved for this run, so + // every sandbox job in the run scores against the same version instead of + // each independently re-resolving "latest" and drifting mid-run. + 'SUPABASE_CLI_STABLE_VERSION', + 'SUPABASE_CLI_BETA_VERSION', ]; /** * Slack for the non-agent work inside `pnpm eval` (supabase start, resets, @@ -622,10 +627,10 @@ function vercelCredentialsFromEnv(): { }; } -/** Serializes configured provider keys into the repo-root `.env` file. */ -function agentEnvironment(): string { +/** Serializes configured provider keys and CLI channel pins into the sandbox's `.env` file. */ +export function agentEnvironment(): string { const lines: string[] = []; - for (const name of AGENT_ENV_NAMES) { + for (const name of FORWARDED_ENV_NAMES) { const value = process.env[name]; if (value) lines.push(`${name}=${value}`); } diff --git a/packages/sandbox/package.json b/packages/sandbox/package.json index 41b587a1..9da33f75 100644 --- a/packages/sandbox/package.json +++ b/packages/sandbox/package.json @@ -11,7 +11,7 @@ }, "scripts": { "typecheck": "tsc --noEmit", - "test": "vitest run test/unit.test.ts", + "test": "vitest run test", "test:docker": "SANDBOX_DOCKER_TESTS=1 vitest run test/docker.test.ts" }, "dependencies": { diff --git a/packages/sandbox/src/cli-channel.ts b/packages/sandbox/src/cli-channel.ts new file mode 100644 index 00000000..768d913c --- /dev/null +++ b/packages/sandbox/src/cli-channel.ts @@ -0,0 +1,228 @@ +/** + * Resolves "latest stable" / "latest beta" Supabase CLI versions from npm's + * dist-tags for the `supabase` package — no auth, no rate-limit exposure + * (unlike the GitHub releases API), which matters because CI's sandbox only + * forwards the model API keys, never a GITHUB_TOKEN. + * + * npm's dist-tag can point at a version whose GitHub release is still a + * draft, so the resolved version's `.deb` release asset is HEAD-checked + * before being trusted; for the beta channel only, a missing asset walks + * back through older published `-beta.N` versions for one that downloads + * (see resolveFallbackBetaVersion). A stable dist-tag with a missing asset + * always throws instead — guessing at a "stable" release would defeat the + * point of the channel. + */ + +export type CliChannel = 'stable' | 'beta'; + +const NPM_DIST_TAGS_URL = + 'https://registry.npmjs.org/-/package/supabase/dist-tags'; + +// Abbreviated packument (versions + dist-tags only, no changelogs/READMEs) — +// this header is what makes npm serve the small form instead of the full one. +const NPM_PACKUMENT_URL = 'https://registry.npmjs.org/supabase'; +const NPM_INSTALL_V1_ACCEPT = 'application/vnd.npm.install-v1+json'; + +// npm's beta dist-tag can carry a prerelease suffix like -rc.1, not just -beta.N. +const VERSION_RE = /^\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?$/; + +// Only the -beta.N shape is walkable for the "try an older published beta" +// fallback below; a -rc.N or other prerelease suffix has no defined ordering +// we can search. +const BETA_SUFFIX_RE = /^(\d+\.\d+\.\d+-beta\.)(\d+)$/; + +// The number of older published beta versions to probe for a downloadable +// asset before giving up — keeps a broken release from turning one nightly +// run into an unbounded chain of GitHub requests. +const MAX_BETA_FALLBACK_CANDIDATES = 5; + +const ENV_OVERRIDE: Record = { + stable: 'SUPABASE_CLI_STABLE_VERSION', + beta: 'SUPABASE_CLI_BETA_VERSION', +}; + +const DIST_TAG: Record = { + stable: 'latest', + beta: 'beta', +}; + +const versionCache = new Map>(); + +const CLI_CHANNELS = new Set(['stable', 'beta']); + +function isCliChannel(value: string): value is CliChannel { + return CLI_CHANNELS.has(value as CliChannel); +} + +/** Arch suffix the CLI's own release `.deb` filenames use. */ +export function hostDebArch(): 'amd64' | 'arm64' { + return process.arch === 'arm64' ? 'arm64' : 'amd64'; +} + +/** Mirrors installSupabaseCli's download URL template in packages/sandbox/src/supabase.ts. */ +export function cliDebUrl( + version: string, + arch: 'amd64' | 'arm64' = hostDebArch() +): string { + return `https://github.com/supabase/cli/releases/download/v${version}/supabase_${version}_linux_${arch}.deb`; +} + +/** HEAD-checks that a release's `.deb` asset is actually downloadable (not behind a draft release). */ +async function debAssetExists(version: string): Promise { + const response = await fetch(cliDebUrl(version), { + method: 'HEAD', + redirect: 'follow', + signal: AbortSignal.timeout(15_000), + }); + return response.ok; +} + +/** + * Resolve a channel to a concrete Supabase CLI version. Memoised per channel + * so a single nightly run only hits the registry once per channel; the cache + * entry is cleared on rejection so a later retry can hit the network again. + * Never falls back to the pinned SUPABASE_CLI_VERSION on failure — that would + * silently mislabel data — so callers must let the throw propagate. + */ +export async function resolveCliVersion(channel: CliChannel): Promise { + const cached = versionCache.get(channel); + if (cached) return cached; + + const promise = resolveCliVersionUncached(channel); + versionCache.set(channel, promise); + promise.catch(() => versionCache.delete(channel)); + return promise; +} + +/** + * Resolves `value` if it names a channel (`'stable'` | `'beta'`); an exact + * version (or `undefined`) passes through unchanged. Split out from + * localStackRuntime's startSession so the branching itself is unit-testable + * without booting a sandbox. + */ +export async function resolveCliVersionOption( + value: string | CliChannel | undefined +): Promise { + if (value === undefined) return undefined; + return isCliChannel(value) ? resolveCliVersion(value) : value; +} + +async function resolveCliVersionUncached(channel: CliChannel): Promise { + const envVar = ENV_OVERRIDE[channel]; + const override = process.env[envVar]?.trim().replace(/^v/, ''); + if (override) { + if (!VERSION_RE.test(override)) { + throw new Error( + `${envVar}=${JSON.stringify(override)} is not a valid Supabase CLI version` + ); + } + // An explicit pin is trusted as-is — no asset check, network or otherwise. + return override; + } + + const response = await fetch(NPM_DIST_TAGS_URL, { + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) { + throw new Error( + `failed to resolve the latest ${channel} Supabase CLI version: ` + + `GET ${NPM_DIST_TAGS_URL} -> ${response.status} ${response.statusText}` + ); + } + const tags = await response.json(); + const distTag = DIST_TAG[channel]; + const version = isRecord(tags) ? tags[distTag] : undefined; + if (typeof version !== 'string' || !VERSION_RE.test(version)) { + throw new Error( + `npm dist-tags for "supabase" did not have a valid "${distTag}" version ` + + `for the ${channel} channel: ${JSON.stringify(version)}` + ); + } + + if (await debAssetExists(version)) return version; + + // The dist-tag pointed at a version whose GitHub release has no + // downloadable .deb (e.g. still a draft) — this is exactly what broke the + // beta channel in CI run 35274970438. A stable release is never guessed at; + // only beta walks back to the newest published version that does have one. + if (channel === 'stable') { + throw new Error( + `npm's "latest" dist-tag for "supabase" points at ${version}, but its ` + + `release asset is missing: HEAD ${cliDebUrl(version)} was not ok` + ); + } + + return resolveFallbackBetaVersion(version); +} + +/** + * Walks back through published npm versions sharing the same `X.Y.Z-beta.` + * prefix as `unpublishedVersion`, newest first, and returns the first one + * whose release `.deb` asset actually downloads. + */ +async function resolveFallbackBetaVersion( + unpublishedVersion: string +): Promise { + const match = BETA_SUFFIX_RE.exec(unpublishedVersion); + if (!match) { + throw new Error( + `npm's "beta" dist-tag for "supabase" points at ${unpublishedVersion}, ` + + `whose release asset is missing (HEAD ${cliDebUrl(unpublishedVersion)} ` + + 'was not ok), and its version does not match the X.Y.Z-beta.N shape ' + + 'this fallback can walk back through' + ); + } + const [, prefix] = match; + + const response = await fetch(NPM_PACKUMENT_URL, { + headers: { Accept: NPM_INSTALL_V1_ACCEPT }, + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) { + throw new Error( + `failed to look up published beta versions for "supabase": ` + + `GET ${NPM_PACKUMENT_URL} -> ${response.status} ${response.statusText}` + ); + } + const packument = await response.json(); + const versions = isRecord(packument) ? packument.versions : undefined; + if (!isRecord(versions)) { + throw new Error( + `npm packument for "supabase" did not include a "versions" object` + ); + } + + const suffixRe = new RegExp(`^${escapeRegExp(prefix)}(\\d+)$`); + const candidates = Object.keys(versions) + .filter((candidate) => candidate !== unpublishedVersion) + .map((candidate) => { + const suffixMatch = suffixRe.exec(candidate); + return suffixMatch + ? { version: candidate, suffix: Number(suffixMatch[1]) } + : null; + }) + .filter((entry): entry is { version: string; suffix: number } => + Boolean(entry) + ) + .sort((a, b) => b.suffix - a.suffix) + .slice(0, MAX_BETA_FALLBACK_CANDIDATES) + .map((entry) => entry.version); + + for (const candidate of candidates) { + if (await debAssetExists(candidate)) return candidate; + } + + throw new Error( + `no published beta Supabase CLI release has a downloadable .deb asset; ` + + `checked ${[unpublishedVersion, ...candidates].join(', ')} via ` + + `${NPM_PACKUMENT_URL}` + ); +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} diff --git a/packages/sandbox/src/index.ts b/packages/sandbox/src/index.ts index 793a5e5e..5cdc23ed 100644 --- a/packages/sandbox/src/index.ts +++ b/packages/sandbox/src/index.ts @@ -1,3 +1,5 @@ +export { resolveCliVersion, hostDebArch, cliDebUrl } from './cli-channel.js'; +export type { CliChannel } from './cli-channel.js'; export { DockerSandbox, dockerCli } from './docker-sandbox.js'; export type { DockerSandboxOptions, diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index 855a3be8..879f6720 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -15,6 +15,7 @@ import { DockerSandbox } from './docker-sandbox.js'; import { createAgentEnvironment } from './agent-environment.js'; import { ensureEdgeRuntime, teardownSupabaseProject } from './supabase.js'; import { buildSkillsPrompt } from './skills.js'; +import { resolveCliVersionOption, type CliChannel } from './cli-channel.js'; const DEFAULT_BASH_TIMEOUT_SEC = 240; const MAX_BASH_TIMEOUT_SEC = 600; @@ -36,8 +37,15 @@ const STACK_CONFIG_RETRY_MS = 2_000; * profile — out of scope for now. */ export interface LocalStackRuntimeOptions { - /** Supabase CLI version baked into the sandbox image (pinned default). */ - cliVersion?: string; + /** + * Supabase CLI version baked into the sandbox image (pinned default). + * Either an exact version (e.g. `2.109.1`) or a channel tag (`'stable'` | + * `'beta'`) that tracks npm's dist-tag for the `supabase` package — + * resolved to a concrete version once, at session start. An eval's own + * `cliVersion:` frontmatter pin always wins over this option, whether this + * is an exact version or a channel tag. + */ + cliVersion?: CliChannel | (string & {}); /** * Supabase MCP feature groups to expose to the agent when the eval links to * a hosted project (`hostedProject: true`). The MCP server runs host-side and @@ -86,7 +94,8 @@ export function localStackRuntime( skipCliInstall, }) { const env = await createAgentEnvironment({ - cliVersion: cliVersion ?? options.cliVersion, + cliVersion: + cliVersion ?? (await resolveCliVersionOption(options.cliVersion)), localDir, skills, mounts, diff --git a/packages/sandbox/test/cli-channel.test.ts b/packages/sandbox/test/cli-channel.test.ts new file mode 100644 index 00000000..209086e0 --- /dev/null +++ b/packages/sandbox/test/cli-channel.test.ts @@ -0,0 +1,352 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const STABLE_ENV = 'SUPABASE_CLI_STABLE_VERSION'; +const BETA_ENV = 'SUPABASE_CLI_BETA_VERSION'; +const DIST_TAGS_URL = 'https://registry.npmjs.org/-/package/supabase/dist-tags'; +const PACKUMENT_URL = 'https://registry.npmjs.org/supabase'; +const INSTALL_V1_ACCEPT = 'application/vnd.npm.install-v1+json'; + +// Mirrors hostDebArch()'s mapping so this test's expected URLs match +// whatever host architecture actually runs it. +const HOST_ARCH = process.arch === 'arm64' ? 'arm64' : 'amd64'; + +function debUrl(version: string): string { + return `https://github.com/supabase/cli/releases/download/v${version}/supabase_${version}_linux_${HOST_ARCH}.deb`; +} + +function jsonResponse( + body: unknown, + init: { ok?: boolean; status?: number; statusText?: string } = {} +) { + return { + ok: init.ok ?? true, + status: init.status ?? 200, + statusText: init.statusText ?? 'OK', + json: async () => body, + }; +} + +function headResponse(ok: boolean) { + return { ok, status: ok ? 200 : 404, statusText: ok ? 'OK' : 'Not Found' }; +} + +/** + * Routes a single stubbed `fetch` by method + URL, mirroring the real + * mixture of GET (dist-tags, packument) and HEAD (asset check) calls + * resolveCliVersion makes. + */ +function routedFetchMock(routes: { + distTags?: unknown; + distTagsInit?: { ok?: boolean; status?: number; statusText?: string }; + assetOk?: (version: string) => boolean; + packument?: unknown; +}) { + return vi.fn(async (url: string, init?: RequestInit) => { + const method = init?.method ?? 'GET'; + if (method === 'HEAD') { + const version = url.match(/supabase_(.+)_linux_/)?.[1]; + return headResponse( + version ? (routes.assetOk?.(version) ?? false) : false + ); + } + if (url === DIST_TAGS_URL) { + return jsonResponse(routes.distTags, routes.distTagsInit); + } + if (url === PACKUMENT_URL) { + return jsonResponse(routes.packument); + } + throw new Error(`unexpected fetch: ${method} ${url}`); + }); +} + +beforeEach(() => { + vi.resetModules(); + delete process.env[STABLE_ENV]; + delete process.env[BETA_ENV]; +}); + +afterEach(() => { + vi.unstubAllGlobals(); + delete process.env[STABLE_ENV]; + delete process.env[BETA_ENV]; +}); + +describe('resolveCliVersion', () => { + it('resolves the stable channel from the "latest" dist-tag when its asset exists', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + + const headCalls = fetchMock.mock.calls.filter( + ([, init]) => init?.method === 'HEAD' + ); + expect(headCalls).toHaveLength(1); + expect(headCalls[0]?.[0]).toBe(debUrl('1.2.3')); + }); + + it('resolves the beta channel from the "beta" dist-tag when its asset exists', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('1.3.0-beta.1'); + }); + + it('prefers the env override over the network and strips a leading v', async () => { + process.env[STABLE_ENV] = 'v9.9.9'; + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).resolves.toBe('9.9.9'); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('trims surrounding whitespace from the env override', async () => { + process.env[BETA_ENV] = ' 1.4.0-rc.2 '; + vi.stubGlobal('fetch', vi.fn()); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('1.4.0-rc.2'); + }); + + it('throws naming the env var when the override is not a valid version', async () => { + process.env[BETA_ENV] = 'not-a-version'; + vi.stubGlobal('fetch', vi.fn()); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).rejects.toThrow( + `${BETA_ENV}="not-a-version" is not a valid Supabase CLI version` + ); + }); + + it('throws with the HTTP status when the registry request fails', async () => { + const fetchMock = routedFetchMock({ + distTags: undefined, + distTagsInit: { + ok: false, + status: 500, + statusText: 'Internal Server Error', + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow( + `${DIST_TAGS_URL} -> 500 Internal Server Error` + ); + }); + + it('throws when the requested dist-tag is missing from the response', async () => { + const fetchMock = routedFetchMock({ distTags: { beta: '1.0.0-beta.1' } }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow( + /did not have a valid "latest" version for the stable channel/ + ); + }); + + it('throws when the dist-tag value is not a valid version string', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: 'not-a-version' }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow( + /did not have a valid "latest" version for the stable channel/ + ); + }); + + it('memoises a successful resolution so a second call does not refetch', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + + // One GET (dist-tags) + one HEAD (asset check) — the second call hits the cache. + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it('clears the cache entry on rejection so a later call refetches', async () => { + let failNextDistTags = true; + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + if (init?.method === 'HEAD') return headResponse(true); + if (url === DIST_TAGS_URL) { + if (failNextDistTags) { + failNextDistTags = false; + return jsonResponse(undefined, { + ok: false, + status: 500, + statusText: 'Internal Server Error', + }); + } + return jsonResponse({ latest: '1.2.3', beta: '1.3.0-beta.1' }); + } + throw new Error(`unexpected fetch: ${url}`); + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow('500'); + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + + expect(fetchMock).toHaveBeenCalledTimes(3); + }); + + it('does not let a beta rejection clear the stable cache entry', async () => { + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + if (init?.method === 'HEAD') return headResponse(true); + if (url === DIST_TAGS_URL) { + return jsonResponse({ latest: '1.2.3', beta: 'not-a-version' }); + } + throw new Error(`unexpected fetch: ${url}`); + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + await expect(resolveCliVersion('beta')).rejects.toThrow( + /did not have a valid "beta" version for the beta channel/ + ); + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + + // Two dist-tags GETs (stable, beta) + one HEAD (stable's asset check + // only — beta never gets that far). + expect(fetchMock).toHaveBeenCalledTimes(3); + }); + + it('falls back to the newest published beta.N-1 when the dist-tag asset is a 404', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '2.118.0-beta.52' }, + assetOk: (version) => version !== '2.118.0-beta.52', + packument: { + versions: { + '2.118.0-beta.52': {}, + '2.118.0-beta.51': {}, + '2.118.0-beta.50': {}, + '2.117.0': {}, + }, + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.51'); + + const packumentCall = fetchMock.mock.calls.find( + ([url]) => url === PACKUMENT_URL + ); + expect(packumentCall?.[1]?.headers).toMatchObject({ + Accept: INSTALL_V1_ACCEPT, + }); + + const headUrls = fetchMock.mock.calls + .filter(([, init]) => init?.method === 'HEAD') + .map(([url]) => url); + expect(headUrls).toEqual([ + debUrl('2.118.0-beta.52'), + debUrl('2.118.0-beta.51'), + ]); + }); + + it('throws naming the unpublished versions when no published beta has a downloadable asset', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '2.118.0-beta.52' }, + assetOk: () => false, + packument: { + versions: { + '2.118.0-beta.52': {}, + '2.118.0-beta.51': {}, + }, + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).rejects.toThrow( + /2\.118\.0-beta\.52.*2\.118\.0-beta\.51/ + ); + }); + + it('throws instead of guessing when the stable dist-tag asset is a 404', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => false, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow( + `HEAD ${debUrl('1.2.3')} was not ok` + ); + }); + + it('does not hit the network at all for an env override', async () => { + process.env[STABLE_ENV] = '9.9.9'; + process.env[BETA_ENV] = '9.9.9-beta.1'; + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).resolves.toBe('9.9.9'); + await expect(resolveCliVersion('beta')).resolves.toBe('9.9.9-beta.1'); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + +describe('resolveCliVersionOption', () => { + it('passes an exact version through unchanged, without touching the network', async () => { + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionOption } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersionOption('2.109.1')).resolves.toBe('2.109.1'); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('passes undefined through unchanged, without touching the network', async () => { + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionOption } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersionOption(undefined)).resolves.toBeUndefined(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('resolves a "stable" channel tag against npm', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionOption } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersionOption('stable')).resolves.toBe('1.2.3'); + }); + + it('resolves a "beta" channel tag against npm', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionOption } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersionOption('beta')).resolves.toBe('1.3.0-beta.1'); + }); +}); From 8c7261a30a07a9d5f35416e9cf984365dc8c8866 Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Tue, 22 Sep 2026 10:29:15 +0100 Subject: [PATCH 02/12] feat(sandbox): fold docker-aware local-stack staging into @supabase-evals/sandbox Promote the CLI team's experiment-land dockerAwareLocalStackRuntime into a first-class localStackRuntime({ docker: 'no-daemon' | 'absent' }) option, so any experiment can stage a sandbox where the Docker daemon is unreachable or the docker binary is absent entirely, alongside the mountDockerSocket sandbox primitive and needsDocker eval frontmatter it depends on. --- README.md | 2 + packages/core/src/eval-metadata.ts | 10 + packages/sandbox/src/cli-channel.ts | 3 +- packages/sandbox/src/docker-sandbox.ts | 14 +- packages/sandbox/src/index.ts | 9 +- packages/sandbox/src/local-stack-runtime.ts | 464 ++++++++++++++++-- .../sandbox/test/local-stack-docker.test.ts | 141 ++++++ packages/sandbox/test/unit.test.ts | 37 ++ 8 files changed, 630 insertions(+), 50 deletions(-) create mode 100644 packages/sandbox/test/local-stack-docker.test.ts diff --git a/README.md b/README.md index adbaf62a..a972a75e 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,8 @@ Set `cliVersion: 2.109.1` in an eval's frontmatter when it requires a specific S An experiment can instead pass `localStackRuntime({ cliVersion: 'stable' })` or `'beta'` to track npm's dist-tag for the `supabase` package rather than an exact version, resolved once at session start. An eval's own `cliVersion:` pin still wins over either form. +An experiment can pass `localStackRuntime({ docker: 'no-daemon' })` or `'absent'` to stage a sandbox where the Docker daemon is unreachable or the `docker` binary is missing entirely, instead of the default `'available'`. `needsDocker` defaults to `true`; set it `false` in an eval's frontmatter when the scenario can run, and is meaningful, without a Docker daemon (e.g. starting the stack is the agent's own job) — that's what lets a Docker-less experiment pick the eval up. + Scorers check what the agent produced, never what the harness provisioned: with `projectRunning: true` (the default) the running stack and the seeded `local/` workspace are setup, so score only the deltas the agent made on top; with `projectRunning: false` the agent creates that state itself, so depending on it is fair game. Test the sandbox plumbing without an agent run (Docker required, not part of `pnpm check`): diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index 328a9d60..fa699c88 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -154,6 +154,14 @@ export type EvalMetadata = { * with `projectRunning: false`. */ skipCliInstall?: boolean; + /** + * Whether this scenario needs a working Docker daemon (sandbox evals + * only). Defaults to true. Set false when the scenario can run, and is + * meaningful, in a sandbox without Docker (e.g. starting the stack is the + * agent's own job) — Docker-less experiments filter on this to decide + * which evals they can run. + */ + needsDocker?: boolean; }; export type ParsedEvalMarkdown = { @@ -175,6 +183,7 @@ export const evalMetadataSchema = z.object({ hostedProject: z.union([z.boolean(), z.stringbool()]).optional(), skills: z.array(z.string().min(1)).optional(), skipCliInstall: z.union([z.boolean(), z.stringbool()]).optional(), + needsDocker: z.union([z.boolean(), z.stringbool()]).optional(), }); // Collapse a YAML scalar into a comparable token: trim, lowercase, and fold @@ -253,6 +262,7 @@ export const evalFrontmatterSchema = z.preprocess((raw) => { ? toIdentifierList(data.skills) : undefined, skipCliInstall: data.skipCliInstall, + needsDocker: data.needsDocker, }; }, evalMetadataSchema); diff --git a/packages/sandbox/src/cli-channel.ts b/packages/sandbox/src/cli-channel.ts index 768d913c..4dfc40d2 100644 --- a/packages/sandbox/src/cli-channel.ts +++ b/packages/sandbox/src/cli-channel.ts @@ -50,7 +50,8 @@ const versionCache = new Map>(); const CLI_CHANNELS = new Set(['stable', 'beta']); -function isCliChannel(value: string): value is CliChannel { +/** Whether `value` names a channel (`'stable'` | `'beta'`) rather than an exact version. */ +export function isCliChannel(value: string): value is CliChannel { return CLI_CHANNELS.has(value as CliChannel); } diff --git a/packages/sandbox/src/docker-sandbox.ts b/packages/sandbox/src/docker-sandbox.ts index 360cd67b..1825a463 100644 --- a/packages/sandbox/src/docker-sandbox.ts +++ b/packages/sandbox/src/docker-sandbox.ts @@ -87,6 +87,13 @@ export interface DockerSandboxOptions { * tools must execute — e.g. a local MCP server build. */ mounts?: readonly SandboxMount[]; + /** + * Bind-mount the host Docker socket into the container. Defaults to true. + * Set false for sandboxes that must be provably Docker-less — the socket + * is never exposed to the container at all, rather than merely hidden or + * blocked from inside it. + */ + mountDockerSocket?: boolean; } export interface RunCommandOptions { @@ -100,6 +107,7 @@ export class DockerSandbox { private network: string | undefined; private image: string; private mounts: readonly SandboxMount[]; + private mountDockerSocket: boolean; readonly workdir: string; /** * Env vars injected into every `runShell` (non-root) command — both the @@ -113,6 +121,7 @@ export class DockerSandbox { this.network = options.network; this.image = options.image ?? DEFAULT_IMAGE; this.mounts = options.mounts ?? []; + this.mountDockerSocket = options.mountDockerSocket !== false; this.workdir = `${WORKSPACE_BASE}-${randomUUID().slice(0, 8)}`; } @@ -141,8 +150,9 @@ export class DockerSandbox { '--rm', '--label', `${SANDBOX_CONTAINER_LABEL}=1`, - '--volume', - '/var/run/docker.sock:/var/run/docker.sock', + ...(this.mountDockerSocket + ? ['--volume', '/var/run/docker.sock:/var/run/docker.sock'] + : []), '--volume', `${this.workdir}:${this.workdir}`, // Caller-requested host mounts (e.g. a local MCP server build the diff --git a/packages/sandbox/src/index.ts b/packages/sandbox/src/index.ts index 5cdc23ed..3206257d 100644 --- a/packages/sandbox/src/index.ts +++ b/packages/sandbox/src/index.ts @@ -19,13 +19,20 @@ export { } from './supabase.js'; export type { SetupSupabaseSandboxOptions } from './supabase.js'; export { + buildDockerDaemonShimScript, buildLocalStackScoringContext, buildLocalStackTools, + buildSupabaseShimScript, buildToolSurfaceAddendum, + LOCAL_STACK_MARKER_PATH, localStackRuntime, toAgentSandbox, } from './local-stack-runtime.js'; -export type { LocalStackRuntimeOptions } from './local-stack-runtime.js'; +export type { + DockerState, + LocalStackEnvironmentMarker, + LocalStackRuntimeOptions, +} from './local-stack-runtime.js'; export { SKILLS_CLI_VERSION, SKILLS_INSTALL_AGENTS, diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index 879f6720..a6499fc2 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -13,9 +13,21 @@ import { } from '@supabase-evals/core'; import { DockerSandbox } from './docker-sandbox.js'; import { createAgentEnvironment } from './agent-environment.js'; -import { ensureEdgeRuntime, teardownSupabaseProject } from './supabase.js'; -import { buildSkillsPrompt } from './skills.js'; -import { resolveCliVersionOption, type CliChannel } from './cli-channel.js'; +import { + computeExcludedServices, + ensureEdgeRuntime, + ensureSupabaseSandboxImage, + installSupabaseCli, + SUPABASE_CLI_VERSION, + teardownSupabaseProject, +} from './supabase.js'; +import { buildSkillsPrompt, installSkills } from './skills.js'; +import { + isCliChannel, + resolveCliVersionOption, + type CliChannel, +} from './cli-channel.js'; +import type { SupabaseService } from './types.js'; const DEFAULT_BASH_TIMEOUT_SEC = 240; const MAX_BASH_TIMEOUT_SEC = 600; @@ -63,8 +75,65 @@ export interface LocalStackRuntimeOptions { * sandbox; pass `{}` to disable MCP altogether. */ mcpServers?: Record; + /** + * Docker availability to stage in the sandbox (default `'available'`). + * `'no-daemon'` and `'absent'` stage a sandbox with no Docker at all — the + * socket is never bind-mounted and `DOCKER_HOST` points at an unreachable + * address — so an eval can exercise the Supabase CLI's behavior when + * Docker is missing, rather than merely simulated from inside a working + * Docker sandbox. `'no-daemon'` additionally shims the `docker` binary so + * `docker --version` keeps working (the CLI's runtime probe, + * supabase/cli#6563, must still *choose* Docker before discovering it + * can't reach it); `'absent'` removes the binary outright. Docker-less + * sessions require `projectRunning: false` and no hosted project link — + * the harness cannot pre-start a stack or link a hosted project without + * Docker. + */ + docker?: DockerState; } +/** + * Docker's availability inside a local-stack sandbox session. See + * {@link LocalStackRuntimeOptions.docker}. + */ +export type DockerState = 'available' | 'no-daemon' | 'absent'; + +/** + * Path of the marker each local-stack session writes recording the + * environment it staged, for scorers to *report* — never to decide + * pass/fail, since branching on `docker` there would grade an eval against + * its own environment. On the `'available'` path this is written through + * the session's own `scoringContext.exec`, which has no root access inside + * the sandbox, so the marker is agent-writable there — acceptable since + * it's metrics-only. On the Docker-less paths it's written root-owned and + * read-only (mode 0444), so the agent cannot rewrite it to fake the + * environment it's being graded in. + */ +export const LOCAL_STACK_MARKER_PATH = '/tmp/supabase-eval-runtime.json'; + +export type LocalStackEnvironmentMarker = { + runtime: 'local-stack'; + /** + * The channel `cliVersion` was resolved from, when the runtime option + * named one (`'stable'` | `'beta'`) and no per-eval `cliVersion:` + * frontmatter pin overrode it. Undefined for an exact-version pin, whether + * from the runtime option or the per-eval override. + */ + channel?: CliChannel; + cliVersion: string; + docker: DockerState; + sessionStartedMs: number; +}; + +// Shadow the real `docker`/`supabase` binaries from the first entry on the +// sandbox PATH (see SANDBOX_PATH in docker-sandbox.ts). +const SUPABASE_SHIM_PATH = '/usr/local/sbin/supabase'; +const DOCKER_SHIM_PATH = '/usr/local/sbin/docker'; + +// Port 1 is never bound (the CLI's own e2e suite reserves it for exactly this +// purpose); not 2375, which Docker Desktop can legitimately expose. +const UNREACHABLE_DOCKER_HOST = 'tcp://127.0.0.1:1'; + /** * Supabase MCP feature groups exposed by default: `docs` only. The sandbox has * no web tools, so `search_docs` is the one capability the agent otherwise @@ -81,7 +150,7 @@ export function localStackRuntime( options: LocalStackRuntimeOptions = {} ): LocalStackRuntime { return { - id: 'local-stack', + id: buildRuntimeId(options), async startSession({ agent, cliVersion, @@ -93,53 +162,356 @@ export function localStackRuntime( mounts, skipCliInstall, }) { - const env = await createAgentEnvironment({ - cliVersion: - cliVersion ?? (await resolveCliVersionOption(options.cliVersion)), - localDir, - skills, + // Stamped before setup so it's comparable with the scorer's PID-1 fallback. + const sessionStartedMs = Date.now(); + const docker = options.docker ?? 'available'; + // An eval's own `cliVersion:` pin always wins over the runtime option + // (whether that option is an exact version or a channel), so only an + // unpinned eval can inherit a channel for the marker below. + const channel = + cliVersion === undefined && + options.cliVersion !== undefined && + isCliChannel(options.cliVersion) + ? options.cliVersion + : undefined; + const version = + cliVersion ?? + (await resolveCliVersionOption(options.cliVersion)) ?? + SUPABASE_CLI_VERSION; + + if (docker === 'available') { + const env = await createAgentEnvironment({ + cliVersion: version, + localDir, + skills, + mounts, + localStack: { + includeServices, + projectRunning, + hosted: hosted + ? { + port: hosted.port, + pgPort: hosted.pgPort, + ref: hosted.ref, + accessToken: hosted.accessToken, + } + : undefined, + skipCliInstall, + }, + }); + const sandbox = env.sandbox; + + const mcpServers = await resolveMcpServers(options, hosted); + + const session = { + tools: buildLocalStackTools(sandbox), + sandbox: toAgentSandbox(sandbox), + mcpServers, + promptAddendum: [ + buildToolSurfaceAddendum(agent, { skipCliInstall }), + buildSkillsPrompt(agent, env.skills), + ] + .filter(Boolean) + .join('\n\n'), + scoringContext: buildLocalStackScoringContext(sandbox, hosted), + ensureReady: () => ensureEdgeRuntime(sandbox, includeServices), + exportWorkspace: (hostDir: string) => + sandbox.copyToHost(sandbox.workdir, hostDir), + close: async () => { + await teardownSupabaseProject(sandbox); + await env.close(); + }, + }; + + // The scoring context's exec has no root access inside the sandbox, + // so this marker is agent-writable on this path — fine here since + // it's metrics-only, unlike the Docker-less path's root-owned marker + // below. Best-effort for the same reason: this is the path every + // experiment takes, and nothing about a default session is worth + // failing a whole run over. The Docker-less path *does* fail hard, + // because there the marker is the evidence that the environment was + // staged as claimed. + try { + await writeLocalStackMarkerViaExec( + (command) => session.scoringContext.exec(command), + buildLocalStackMarker(docker, version, sessionStartedMs, channel) + ); + } catch (err) { + console.warn( + `[local-stack] could not write ${LOCAL_STACK_MARKER_PATH}: ${ + err instanceof Error ? err.message : String(err) + }` + ); + } + return session; + } + + // Docker-less staging (docker === 'no-daemon' | 'absent'): the harness + // cannot pre-start a stack or link a hosted project without Docker. + if (projectRunning !== false) { + throw new Error( + 'docker-less sandbox evals must set `projectRunning: false`; the harness cannot pre-start a stack without Docker' + ); + } + if (hosted) { + throw new Error( + 'docker-less sandbox evals cannot link a hosted project — set hostedProject: false' + ); + } + + const image = await ensureSupabaseSandboxImage(); + const sandbox = await DockerSandbox.create({ + image, + network: 'host', mounts, - localStack: { - includeServices, - projectRunning, - hosted: hosted - ? { - port: hosted.port, - pgPort: hosted.pgPort, - ref: hosted.ref, - accessToken: hosted.accessToken, - } - : undefined, - skipCliInstall, - }, + mountDockerSocket: false, }); - const sandbox = env.sandbox; - - const mcpServers = await resolveMcpServers(options, hosted); - - return { - tools: buildLocalStackTools(sandbox), - sandbox: toAgentSandbox(sandbox), - mcpServers, - promptAddendum: [ - buildToolSurfaceAddendum(agent, { skipCliInstall }), - buildSkillsPrompt(agent, env.skills), - ] - .filter(Boolean) - .join('\n\n'), - scoringContext: buildLocalStackScoringContext(sandbox, hosted), - ensureReady: () => ensureEdgeRuntime(sandbox, includeServices), - exportWorkspace: (hostDir: string) => - sandbox.copyToHost(sandbox.workdir, hostDir), - close: async () => { - await teardownSupabaseProject(sandbox); - await env.close(); - }, - }; + + try { + if (!skipCliInstall) { + await installSupabaseCli(sandbox, version); + } + + // Deliberately no socket-group grant here (unlike setupSupabaseSandbox): + // CI's sandbox already ends its Docker setup with `chmod 666 + // /var/run/docker.sock`, so the grant would be a no-op there — + // DOCKER_HOST below is the real mechanism. For `absent`, a real + // Docker-less host wouldn't have DOCKER_HOST set at all — but leaving + // it set here costs nothing and blocks any future accidental socket + // exposure, so it stays for both states. + sandbox.extraEnv = { + ...sandbox.extraEnv, + DOCKER_HOST: UNREACHABLE_DOCKER_HOST, + }; + + if (!skipCliInstall) { + await installSupabaseShim(sandbox, includeServices); + } + + // Captured before removal: no-daemon's shim echoes this for `docker + // --version` so the CLI's #6563 runtime probe still picks Docker. + let dockerVersion = ''; + if (docker === 'no-daemon') { + dockerVersion = ( + await sandbox.runShellAsRoot('docker --version') + ).stdout.trim(); + if (!dockerVersion) { + throw new Error( + 'failed to capture `docker --version` before removing the real binary' + ); + } + } + + // Assert the postcondition instead of trusting `rm -f`, which always exits 0. + const removal = await sandbox.runShellAsRoot( + 'for b in docker dockerd docker-proxy; do p="$(command -v "$b" 2>/dev/null)" && rm -f "$p"; done; ! command -v docker >/dev/null 2>&1' + ); + if (!removal.ok) { + throw new Error( + `docker is still on PATH after removal: ${removal.stderr || removal.stdout}` + ); + } + + // The root shell above has a different PATH than the agent's + // SANDBOX_PATH, so also assert the binary is gone from the PATH the + // agent actually runs commands under. + const pathCheck = await sandbox.runShell( + '! command -v docker >/dev/null 2>&1' + ); + if (!pathCheck.ok) { + throw new Error( + `docker is still on the agent's PATH after removal: ${pathCheck.stderr || pathCheck.stdout}` + ); + } + + if (docker === 'no-daemon') { + await installDockerDaemonShim(sandbox, dockerVersion); + } + + if (localDir) { + await sandbox.copyToContainer(localDir, sandbox.workdir); + } + const installedSkills = await installSkills(sandbox, skills ?? []); + + // With no bind mount, the socket must not exist at all — a stronger + // guarantee than merely checking reachability from inside the container. + const socketAbsent = await sandbox.runShellAsRoot( + 'test ! -e /var/run/docker.sock' + ); + if (!socketAbsent.ok) { + throw new Error( + 'the Docker socket unexpectedly exists in a Docker-less sandbox' + ); + } + + // Root-owned and read-only so the agent cannot rewrite it to fake + // the environment it's being evaluated in. + await sandbox.writeRootFile( + LOCAL_STACK_MARKER_PATH, + JSON.stringify( + buildLocalStackMarker(docker, version, sessionStartedMs, channel) + ), + '0444' + ); + + const dockerlessMcpServers = await resolveMcpServers( + options, + undefined + ); + + return { + tools: buildLocalStackTools(sandbox), + sandbox: toAgentSandbox(sandbox), + // Same wiring as the `available` path: an experiment's explicit + // `mcpServers`/`mcpFeatures` must not be silently dropped just + // because Docker is missing. `hosted` is always undefined here + // (guarded above), so this resolves to the platform-independent + // docs server unless the experiment asked for something else. + mcpServers: dockerlessMcpServers, + promptAddendum: [ + buildToolSurfaceAddendum(agent, { skipCliInstall }), + buildSkillsPrompt(agent, installedSkills), + ] + .filter(Boolean) + .join('\n\n'), + scoringContext: buildLocalStackScoringContext(sandbox), + exportWorkspace: (hostDir: string) => + sandbox.copyToHost(sandbox.workdir, hostDir), + // Nothing to restore without Docker; the `available` path above + // delegates to its own session's ensureReady. + ensureReady: async () => {}, + // No teardownSupabaseProject: nothing can have started without Docker. + close: () => sandbox.stop(), + }; + } catch (err) { + await sandbox.stop(); + throw err; + } }, }; } +/** + * The runtime's log-line id (see run-eval.ts's PLAN line): plain + * `'local-stack'` for default options, otherwise the non-default bits + * appended so e.g. a beta + absent runtime reads as + * `local-stack-beta-absent` in the run log. + */ +function buildRuntimeId(options: LocalStackRuntimeOptions): string { + const bits: string[] = []; + if (options.cliVersion !== undefined) bits.push(options.cliVersion); + const docker = options.docker ?? 'available'; + if (docker !== 'available') bits.push(docker); + return bits.length > 0 ? `local-stack-${bits.join('-')}` : 'local-stack'; +} + +function buildLocalStackMarker( + docker: DockerState, + cliVersion: string, + sessionStartedMs: number, + channel?: CliChannel +): LocalStackEnvironmentMarker { + return { + runtime: 'local-stack', + channel, + cliVersion, + docker, + sessionStartedMs, + }; +} + +async function writeLocalStackMarkerViaExec( + exec: ( + command: string + ) => Promise<{ ok: boolean; stdout: string; stderr: string }>, + marker: LocalStackEnvironmentMarker +): Promise { + // base64 transport sidesteps quoting the JSON payload through the shell. + const encoded = Buffer.from(JSON.stringify(marker), 'utf-8').toString( + 'base64' + ); + const result = await exec( + `echo ${encoded} | base64 -d > ${LOCAL_STACK_MARKER_PATH}` + ); + if (!result.ok) { + throw new Error( + `failed to write the local-stack environment marker: ${result.stderr || result.stdout}` + ); + } +} + +async function installSupabaseShim( + sandbox: DockerSandbox, + includeServices: readonly string[] | undefined +): Promise { + const real = ( + await sandbox.runShellAsRoot('command -v supabase') + ).stdout.trim(); + if (!real || real === SUPABASE_SHIM_PATH) { + throw new Error( + `could not resolve the real supabase binary before installing the CLI shim (got ${JSON.stringify(real)})` + ); + } + const excluded = computeExcludedServices(includeServices); + await sandbox.writeRootFile( + SUPABASE_SHIM_PATH, + buildSupabaseShimScript(real, excluded), + '0755' + ); +} + +/** + * Shim that shadows `supabase` on PATH in a Docker-less sandbox. On the + * `'available'` path the *harness* runs `supabase start` itself and applies + * `services:` exclusions via `buildSupabaseStartCommand`; without Docker the + * harness cannot pre-start anything, so the *agent* runs `supabase start` + * instead, and this shim is the only seam left through which the eval's + * `includeServices` (`services:` frontmatter) still gets honored — it + * injects the same `-x ` flag (`computeExcludedServices`) into + * whatever `supabase start` the agent types. + */ +export function buildSupabaseShimScript( + realBin: string, + excluded: readonly SupabaseService[] +): string { + const lines = [ + '#!/bin/bash', + `export DOCKER_HOST=${UNREACHABLE_DOCKER_HOST}`, + `REAL=${shellQuote(realBin)}`, + ]; + if (excluded.length > 0) { + lines.push( + `if [ "$1" = "start" ]; then shift; exec "$REAL" start "$@" -x ${excluded.join(',')}; fi` + ); + } + lines.push('exec "$REAL" "$@"'); + return lines.join('\n'); +} + +async function installDockerDaemonShim( + sandbox: DockerSandbox, + dockerVersion: string +): Promise { + await sandbox.writeRootFile( + DOCKER_SHIM_PATH, + buildDockerDaemonShimScript(dockerVersion), + '0755' + ); +} + +export function buildDockerDaemonShimScript(dockerVersion: string): string { + return [ + '#!/bin/bash', + 'case "$1" in', + // --version must keep working so the CLI's runtime probe + // (supabase/cli#6563) still *chooses* Docker as its runtime. + ` --version|-v) echo ${shellQuote(dockerVersion)}; exit 0 ;;`, + 'esac', + `echo "Cannot connect to the Docker daemon at ${UNREACHABLE_DOCKER_HOST}. Is the docker daemon running?" >&2`, + 'exit 1', + ].join('\n'); +} + /** * Describes the session's tool surface: the binaries installed in the workspace * and the in-process `bash`/`files_*` tools from `buildLocalStackTools`. @@ -175,7 +547,7 @@ export function buildToolSurfaceAddendum( * hosted project there's no platform to talk to, so fall back to the * platform-independent docs server (`search_docs`). */ -async function resolveMcpServers( +export async function resolveMcpServers( options: LocalStackRuntimeOptions, hosted?: HostedLink ): Promise> { diff --git a/packages/sandbox/test/local-stack-docker.test.ts b/packages/sandbox/test/local-stack-docker.test.ts new file mode 100644 index 00000000..720d4d87 --- /dev/null +++ b/packages/sandbox/test/local-stack-docker.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, it } from 'vitest'; +import { + buildDockerDaemonShimScript, + buildSupabaseShimScript, + localStackRuntime, + resolveMcpServers, +} from '../src/local-stack-runtime.js'; +import type { SupabaseService } from '../src/types.js'; + +describe('localStackRuntime id', () => { + it('ids as `local-stack` with no options', () => { + expect(localStackRuntime().id).toBe('local-stack'); + }); + + it('ids as `local-stack` when docker is explicitly the default', () => { + expect(localStackRuntime({ docker: 'available' }).id).toBe('local-stack'); + }); + + it('appends the docker state when it is non-default', () => { + expect(localStackRuntime({ docker: 'no-daemon' }).id).toBe( + 'local-stack-no-daemon' + ); + expect(localStackRuntime({ docker: 'absent' }).id).toBe( + 'local-stack-absent' + ); + }); + + it('appends the cliVersion option when set', () => { + expect(localStackRuntime({ cliVersion: 'beta' }).id).toBe( + 'local-stack-beta' + ); + expect(localStackRuntime({ cliVersion: '2.109.1' }).id).toBe( + 'local-stack-2.109.1' + ); + }); + + it('combines a non-default cliVersion and docker state', () => { + expect(localStackRuntime({ cliVersion: 'beta', docker: 'absent' }).id).toBe( + 'local-stack-beta-absent' + ); + }); +}); + +describe('buildSupabaseShimScript', () => { + it('emits DOCKER_HOST, the -x start branch for excluded services, and a passthrough exec', () => { + const excluded: SupabaseService[] = ['gotrue', 'kong']; + expect(buildSupabaseShimScript('/usr/bin/supabase', excluded)).toEqual( + [ + '#!/bin/bash', + 'export DOCKER_HOST=tcp://127.0.0.1:1', + "REAL='/usr/bin/supabase'", + 'if [ "$1" = "start" ]; then shift; exec "$REAL" start "$@" -x gotrue,kong; fi', + 'exec "$REAL" "$@"', + ].join('\n') + ); + }); + + it('omits the start branch entirely when no services are excluded', () => { + expect( + buildSupabaseShimScript('/usr/bin/supabase', []) + ).toMatchInlineSnapshot(` + "#!/bin/bash + export DOCKER_HOST=tcp://127.0.0.1:1 + REAL='/usr/bin/supabase' + exec "$REAL" "$@"" + `); + }); + + it('always ends with exec "$REAL" "$@" regardless of exclusions', () => { + const excluded: SupabaseService[] = ['gotrue', 'kong']; + const script = buildSupabaseShimScript('/usr/bin/supabase', excluded); + expect(script.endsWith('exec "$REAL" "$@"')).toBe(true); + }); + + it('single-quotes a realBin containing a single quote safely for the shell', () => { + const script = buildSupabaseShimScript("/usr/local/it's/bin/supabase", []); + expect(script).toContain(`REAL='/usr/local/it'\\''s/bin/supabase'`); + }); + + it('single-quotes a realBin containing $, backtick, and " safely for the shell', () => { + const realBin = '/usr/local/$(whoami)/`id`/"bin"/supabase'; + const script = buildSupabaseShimScript(realBin, []); + const realLine = script + .split('\n') + .find((line) => line.startsWith('REAL=')); + expect(realLine).toBe('REAL=\'/usr/local/$(whoami)/`id`/"bin"/supabase\''); + }); +}); + +describe('buildDockerDaemonShimScript', () => { + const dockerVersion = 'Docker version 20.10.24+dfsg1, build 297e128'; + + it('echoes the captured version string verbatim for --version/-v and exits 0, otherwise fails as unreachable', () => { + expect(buildDockerDaemonShimScript(dockerVersion)).toMatchInlineSnapshot(` + "#!/bin/bash + case "$1" in + --version|-v) echo 'Docker version 20.10.24+dfsg1, build 297e128'; exit 0 ;; + esac + echo "Cannot connect to the Docker daemon at tcp://127.0.0.1:1. Is the docker daemon running?" >&2 + exit 1" + `); + }); + + it('single-quotes a version string containing a single quote safely for the shell', () => { + const script = buildDockerDaemonShimScript( + "Docker version 'weird', build x" + ); + const versionLine = script + .split('\n') + .find((line) => line.includes('--version|-v')); + expect(versionLine).toBe( + ` --version|-v) echo 'Docker version '\\''weird'\\'', build x'; exit 0 ;;` + ); + }); +}); + +describe('resolveMcpServers', () => { + // The Docker-less session path resolves its MCP map through this same + // helper rather than hardcoding a docs-only server, so an experiment's + // explicit wiring survives in a sandbox with no Docker. `hosted` is always + // undefined there, which is the case pinned here. + it('returns an explicit mcpServers map untouched', async () => { + const explicit = { + custom: { command: 'node', args: ['server.js'] }, + }; + await expect( + resolveMcpServers({ mcpServers: explicit }, undefined) + ).resolves.toBe(explicit); + }); + + it('honours an explicit empty map, disabling MCP entirely', async () => { + await expect( + resolveMcpServers({ mcpServers: {} }, undefined) + ).resolves.toEqual({}); + }); + + it('falls back to a single docs-only supabase server', async () => { + const servers = await resolveMcpServers({}, undefined); + expect(Object.keys(servers)).toEqual(['supabase']); + }); +}); diff --git a/packages/sandbox/test/unit.test.ts b/packages/sandbox/test/unit.test.ts index 9f15aa00..6b3e8c2a 100644 --- a/packages/sandbox/test/unit.test.ts +++ b/packages/sandbox/test/unit.test.ts @@ -643,6 +643,43 @@ describe('skipCliInstall frontmatter', () => { }); }); +describe('needsDocker frontmatter', () => { + const buildMarkdown = (extra: string) => + [ + '---', + 'stage: build', + 'interface: cli', + 'product: [database]', + 'topic: [sdk]', + extra, + '---', + 'body', + ].join('\n'); + + it('accepts a real boolean true and false', () => { + expect( + parseEvalMarkdown(buildMarkdown('needsDocker: true')).metadata.needsDocker + ).toBe(true); + expect( + parseEvalMarkdown(buildMarkdown('needsDocker: false')).metadata + .needsDocker + ).toBe(false); + }); + + it('accepts a quoted string form via z.stringbool()', () => { + expect( + parseEvalMarkdown(buildMarkdown('needsDocker: "false"')).metadata + .needsDocker + ).toBe(false); + }); + + it('defaults to undefined when omitted', () => { + expect( + parseEvalMarkdown(buildMarkdown('')).metadata.needsDocker + ).toBeUndefined(); + }); +}); + describe('resolveSandboxPath', () => { it('accepts and normalizes relative paths', () => { expect(resolveSandboxPath('a/b.txt')).toBe('a/b.txt'); From f6753427d4428789ad012ee09f989c3a1c3a7f23 Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Tue, 22 Sep 2026 12:09:46 +0100 Subject: [PATCH 03/12] fix(sandbox): address PR #324 review findings for CLI channel resolution - Export FORWARDED_ENV_NAMES from run-vercel-evals.ts and import it in the test instead of a drifted local copy, so a leaked XAI_API_KEY can no longer break agentEnvironment()'s assertions. - Make cliDebUrl's arch param required and check both amd64 and arm64 release assets, since the resolver's host arch can differ from the sandbox container's; delete the now-unused hostDebArch. - Generalise the beta walk-back to every published X.Y.Z-beta.N version older than the unpublished one (not just the same minor), ordered with a numeric compareBetaVersionsDesc comparator instead of a string compare. - Treat a thrown error for one walk-back candidate as "unavailable" and keep probing the rest, instead of aborting the whole walk-back; the initial dist-tag asset check still fails loud. - Import isRecord from @supabase-evals/core/json instead of a local copy that dropped the array guard. - Guard the version cache's rejection cleanup with an identity check so a stale promise can't evict a newer cached one. - Drop cliDebUrl/hostDebArch from the sandbox barrel (no external consumers) and correct local-stack-runtime's doc comment: channel resolution is memoised per process, not per session. --- .../scripts/run-vercel-evals.test.ts | 9 +- apps/framework/scripts/run-vercel-evals.ts | 2 +- packages/core/package.json | 3 +- packages/core/src/json.ts | 7 +- packages/sandbox/src/cli-channel.ts | 147 +++++++---- packages/sandbox/src/index.ts | 2 +- packages/sandbox/src/local-stack-runtime.ts | 6 +- packages/sandbox/test/cli-channel.test.ts | 242 ++++++++++++++++-- 8 files changed, 336 insertions(+), 82 deletions(-) diff --git a/apps/framework/scripts/run-vercel-evals.test.ts b/apps/framework/scripts/run-vercel-evals.test.ts index fe1b9806..7da08d92 100644 --- a/apps/framework/scripts/run-vercel-evals.test.ts +++ b/apps/framework/scripts/run-vercel-evals.test.ts @@ -17,6 +17,7 @@ import { cleanupSandbox, downloadResults, finalizeResult, + FORWARDED_ENV_NAMES, isRetryableSandboxCreateError, isTerminalSandboxCreateError, packWorkspaceScript, @@ -26,14 +27,6 @@ import { expandJobs, } from './run-vercel-evals.js'; -const FORWARDED_ENV_NAMES = [ - 'ANTHROPIC_API_KEY', - 'OPENAI_API_KEY', - 'AI_GATEWAY_API_KEY', - 'SUPABASE_CLI_STABLE_VERSION', - 'SUPABASE_CLI_BETA_VERSION', -]; - describe('agentEnvironment', () => { const originalValues = new Map(); diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index efcbcc14..4ceccb2b 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -19,7 +19,7 @@ const ROOT = fileURLToPath(new URL('../../../', import.meta.url)); /** Base for sandbox URLs printed during runs */ const SANDBOX_DASHBOARD_URL = 'https://vercel.com/supabase/evals-runner/sandboxes'; -const FORWARDED_ENV_NAMES = [ +export const FORWARDED_ENV_NAMES = [ 'ANTHROPIC_API_KEY', 'OPENAI_API_KEY', 'AI_GATEWAY_API_KEY', diff --git a/packages/core/package.json b/packages/core/package.json index 51ed3f4d..6c41ea2e 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -8,7 +8,8 @@ "exports": { ".": "./src/index.ts", "./eval-metadata": "./src/eval-metadata.ts", - "./eval-markdown": "./src/eval-markdown.ts" + "./eval-markdown": "./src/eval-markdown.ts", + "./json": "./src/json.ts" }, "scripts": { "test": "vitest run" diff --git a/packages/core/src/json.ts b/packages/core/src/json.ts index b8da5d0d..8f7c0369 100644 --- a/packages/core/src/json.ts +++ b/packages/core/src/json.ts @@ -1,4 +1,9 @@ -/** Shared JSON helpers used across the core package. */ +/** + * Shared JSON helpers, imported both from within the core package and, via + * the `@supabase-evals/core/json` subpath, by other packages (e.g. + * `@supabase-evals/sandbox`'s CLI channel resolution) that want them without + * pulling in core's much larger `index.ts`. + */ export function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); diff --git a/packages/sandbox/src/cli-channel.ts b/packages/sandbox/src/cli-channel.ts index 768d913c..ffe87024 100644 --- a/packages/sandbox/src/cli-channel.ts +++ b/packages/sandbox/src/cli-channel.ts @@ -13,6 +13,8 @@ * point of the channel. */ +import { isRecord } from '@supabase-evals/core/json'; + export type CliChannel = 'stable' | 'beta'; const NPM_DIST_TAGS_URL = @@ -29,7 +31,12 @@ const VERSION_RE = /^\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?$/; // Only the -beta.N shape is walkable for the "try an older published beta" // fallback below; a -rc.N or other prerelease suffix has no defined ordering // we can search. -const BETA_SUFFIX_RE = /^(\d+\.\d+\.\d+-beta\.)(\d+)$/; +const BETA_SUFFIX_RE = /^\d+\.\d+\.\d+-beta\.\d+$/; + +// Matches any published "X.Y.Z-beta.N" version (any major/minor/patch), +// captured for the numeric tuple comparison in compareBetaVersionsDesc — +// unlike BETA_SUFFIX_RE, it isn't anchored to one specific version's prefix. +const BETA_VERSION_RE = /^(\d+)\.(\d+)\.(\d+)-beta\.(\d+)$/; // The number of older published beta versions to probe for a downloadable // asset before giving up — keeps a broken release from turning one nightly @@ -54,22 +61,18 @@ function isCliChannel(value: string): value is CliChannel { return CLI_CHANNELS.has(value as CliChannel); } -/** Arch suffix the CLI's own release `.deb` filenames use. */ -export function hostDebArch(): 'amd64' | 'arm64' { - return process.arch === 'arm64' ? 'arm64' : 'amd64'; -} - /** Mirrors installSupabaseCli's download URL template in packages/sandbox/src/supabase.ts. */ -export function cliDebUrl( - version: string, - arch: 'amd64' | 'arm64' = hostDebArch() -): string { +export function cliDebUrl(version: string, arch: 'amd64' | 'arm64'): string { return `https://github.com/supabase/cli/releases/download/v${version}/supabase_${version}_linux_${arch}.deb`; } -/** HEAD-checks that a release's `.deb` asset is actually downloadable (not behind a draft release). */ -async function debAssetExists(version: string): Promise { - const response = await fetch(cliDebUrl(version), { +/** The GitHub release page for a version, named in errors instead of a single arch's asset URL. */ +function releaseTagUrl(version: string): string { + return `https://github.com/supabase/cli/releases/tag/v${version}`; +} + +async function debAssetHeadOk(url: string): Promise { + const response = await fetch(url, { method: 'HEAD', redirect: 'follow', signal: AbortSignal.timeout(15_000), @@ -77,10 +80,30 @@ async function debAssetExists(version: string): Promise { return response.ok; } +/** + * HEAD-checks that a release's `.deb` assets are actually downloadable (not + * behind a draft release) for both architectures the sandbox installs onto — + * the host running this resolver and the sandbox container it targets can + * differ (e.g. an Apple Silicon host building a linux/amd64 sandbox image), + * so checking only one arch could pass while the other 404s. This also + * catches a partially-uploaded release that a single-arch probe would miss. + * Two HEAD requests per candidate, capped at MAX_BETA_FALLBACK_CANDIDATES + * candidates in the walk-back below, is an acceptable request budget. + */ +async function debAssetExists(version: string): Promise { + const [amd64, arm64] = await Promise.all([ + debAssetHeadOk(cliDebUrl(version, 'amd64')), + debAssetHeadOk(cliDebUrl(version, 'arm64')), + ]); + return amd64 && arm64; +} + /** * Resolve a channel to a concrete Supabase CLI version. Memoised per channel * so a single nightly run only hits the registry once per channel; the cache - * entry is cleared on rejection so a later retry can hit the network again. + * entry is cleared on rejection so a later retry can hit the network again — + * guarded by an identity check so a stale rejection can never evict a + * different (newer) promise that has since taken its place in the cache. * Never falls back to the pinned SUPABASE_CLI_VERSION on failure — that would * silently mislabel data — so callers must let the throw propagate. */ @@ -90,7 +113,9 @@ export async function resolveCliVersion(channel: CliChannel): Promise { const promise = resolveCliVersionUncached(channel); versionCache.set(channel, promise); - promise.catch(() => versionCache.delete(channel)); + promise.catch(() => { + if (versionCache.get(channel) === promise) versionCache.delete(channel); + }); return promise; } @@ -139,6 +164,9 @@ async function resolveCliVersionUncached(channel: CliChannel): Promise { ); } + // Deliberately not caught: a transient failure here (DNS blip, timeout) + // must throw loud rather than silently walk back to an older beta, or + // (for stable) never walk back at all. if (await debAssetExists(version)) return version; // The dist-tag pointed at a version whose GitHub release has no @@ -148,7 +176,8 @@ async function resolveCliVersionUncached(channel: CliChannel): Promise { if (channel === 'stable') { throw new Error( `npm's "latest" dist-tag for "supabase" points at ${version}, but its ` + - `release asset is missing: HEAD ${cliDebUrl(version)} was not ok` + `release asset is missing (checked amd64 and arm64 .deb assets at ` + + `${releaseTagUrl(version)})` ); } @@ -156,23 +185,30 @@ async function resolveCliVersionUncached(channel: CliChannel): Promise { } /** - * Walks back through published npm versions sharing the same `X.Y.Z-beta.` - * prefix as `unpublishedVersion`, newest first, and returns the first one - * whose release `.deb` asset actually downloads. + * Walks back through every published npm version matching the `X.Y.Z-beta.N` + * shape that is strictly older than `unpublishedVersion` — not just versions + * sharing its exact minor — newest first, and returns the first one whose + * release `.deb` assets actually download. This lets a brand-new minor's + * first beta (e.g. `2.119.0-beta.1`, still a draft) fall back across the + * minor boundary to the previous minor's newest published beta (e.g. + * `2.118.0-beta.60`). A candidate newer than `unpublishedVersion` is never + * considered — if npm's dist-tag skipped it, it's likelier to be a draft too. + * + * A transient error probing one candidate (network blip, timeout) is logged + * and skipped rather than aborting the whole walk-back, so one bad candidate + * doesn't waste the rest of the probe budget. */ async function resolveFallbackBetaVersion( unpublishedVersion: string ): Promise { - const match = BETA_SUFFIX_RE.exec(unpublishedVersion); - if (!match) { + if (!BETA_SUFFIX_RE.test(unpublishedVersion)) { throw new Error( `npm's "beta" dist-tag for "supabase" points at ${unpublishedVersion}, ` + - `whose release asset is missing (HEAD ${cliDebUrl(unpublishedVersion)} ` + - 'was not ok), and its version does not match the X.Y.Z-beta.N shape ' + - 'this fallback can walk back through' + `whose release asset is missing (checked amd64 and arm64 .deb assets ` + + `at ${releaseTagUrl(unpublishedVersion)}), and its version does not ` + + 'match the X.Y.Z-beta.N shape this fallback can walk back through' ); } - const [, prefix] = match; const response = await fetch(NPM_PACKUMENT_URL, { headers: { Accept: NPM_INSTALL_V1_ACCEPT }, @@ -192,24 +228,23 @@ async function resolveFallbackBetaVersion( ); } - const suffixRe = new RegExp(`^${escapeRegExp(prefix)}(\\d+)$`); const candidates = Object.keys(versions) - .filter((candidate) => candidate !== unpublishedVersion) - .map((candidate) => { - const suffixMatch = suffixRe.exec(candidate); - return suffixMatch - ? { version: candidate, suffix: Number(suffixMatch[1]) } - : null; - }) - .filter((entry): entry is { version: string; suffix: number } => - Boolean(entry) + .filter((candidate) => BETA_VERSION_RE.test(candidate)) + .filter( + (candidate) => compareBetaVersionsDesc(candidate, unpublishedVersion) > 0 ) - .sort((a, b) => b.suffix - a.suffix) - .slice(0, MAX_BETA_FALLBACK_CANDIDATES) - .map((entry) => entry.version); + .sort(compareBetaVersionsDesc) + .slice(0, MAX_BETA_FALLBACK_CANDIDATES); for (const candidate of candidates) { - if (await debAssetExists(candidate)) return candidate; + try { + if (await debAssetExists(candidate)) return candidate; + } catch (error) { + console.warn( + `[cli-channel] beta fallback candidate ${candidate} could not be checked, skipping: ` + + (error instanceof Error ? error.message : String(error)) + ); + } } throw new Error( @@ -219,10 +254,36 @@ async function resolveFallbackBetaVersion( ); } -function escapeRegExp(value: string): string { - return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +function parseBetaVersion( + version: string +): [major: number, minor: number, patch: number, beta: number] | undefined { + const match = BETA_VERSION_RE.exec(version); + if (!match) return undefined; + return [ + Number(match[1]), + Number(match[2]), + Number(match[3]), + Number(match[4]), + ]; } -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null; +/** + * Orders two "X.Y.Z-beta.N" version strings newest-first — usable directly as + * an `Array#sort` comparator for descending order. Compares each component + * numerically so e.g. `2.118.0-beta.10` correctly sorts ahead of + * `2.118.0-beta.9`; a plain string compare would invert that. Throws if + * either string isn't a parseable `X.Y.Z-beta.N` version. + */ +export function compareBetaVersionsDesc(a: string, b: string): number { + const tupleA = parseBetaVersion(a); + const tupleB = parseBetaVersion(b); + if (!tupleA || !tupleB) { + throw new Error( + `compareBetaVersionsDesc expected "X.Y.Z-beta.N" versions, got ${JSON.stringify(a)} and ${JSON.stringify(b)}` + ); + } + for (let index = 0; index < tupleA.length; index += 1) { + if (tupleA[index] !== tupleB[index]) return tupleB[index] - tupleA[index]; + } + return 0; } diff --git a/packages/sandbox/src/index.ts b/packages/sandbox/src/index.ts index 5cdc23ed..52d68e09 100644 --- a/packages/sandbox/src/index.ts +++ b/packages/sandbox/src/index.ts @@ -1,4 +1,4 @@ -export { resolveCliVersion, hostDebArch, cliDebUrl } from './cli-channel.js'; +export { resolveCliVersion } from './cli-channel.js'; export type { CliChannel } from './cli-channel.js'; export { DockerSandbox, dockerCli } from './docker-sandbox.js'; export type { diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index 879f6720..392d4dad 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -40,8 +40,10 @@ export interface LocalStackRuntimeOptions { /** * Supabase CLI version baked into the sandbox image (pinned default). * Either an exact version (e.g. `2.109.1`) or a channel tag (`'stable'` | - * `'beta'`) that tracks npm's dist-tag for the `supabase` package — - * resolved to a concrete version once, at session start. An eval's own + * `'beta'`) that tracks npm's dist-tag for the `supabase` package — a + * channel tag's resolution is memoised for the lifetime of the process + * (see resolveCliVersion), not per session, so a later session in the same + * run will not pick up a newly published version. An eval's own * `cliVersion:` frontmatter pin always wins over this option, whether this * is an exact version or a channel tag. */ diff --git a/packages/sandbox/test/cli-channel.test.ts b/packages/sandbox/test/cli-channel.test.ts index 209086e0..8bc1353a 100644 --- a/packages/sandbox/test/cli-channel.test.ts +++ b/packages/sandbox/test/cli-channel.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { cliDebUrl } from '../src/cli-channel.js'; const STABLE_ENV = 'SUPABASE_CLI_STABLE_VERSION'; const BETA_ENV = 'SUPABASE_CLI_BETA_VERSION'; @@ -6,14 +7,6 @@ const DIST_TAGS_URL = 'https://registry.npmjs.org/-/package/supabase/dist-tags'; const PACKUMENT_URL = 'https://registry.npmjs.org/supabase'; const INSTALL_V1_ACCEPT = 'application/vnd.npm.install-v1+json'; -// Mirrors hostDebArch()'s mapping so this test's expected URLs match -// whatever host architecture actually runs it. -const HOST_ARCH = process.arch === 'arm64' ? 'arm64' : 'amd64'; - -function debUrl(version: string): string { - return `https://github.com/supabase/cli/releases/download/v${version}/supabase_${version}_linux_${HOST_ARCH}.deb`; -} - function jsonResponse( body: unknown, init: { ok?: boolean; status?: number; statusText?: string } = {} @@ -32,21 +25,24 @@ function headResponse(ok: boolean) { /** * Routes a single stubbed `fetch` by method + URL, mirroring the real - * mixture of GET (dist-tags, packument) and HEAD (asset check) calls - * resolveCliVersion makes. + * mixture of GET (dist-tags, packument) and HEAD (amd64 + arm64 asset check) + * calls resolveCliVersion makes. */ function routedFetchMock(routes: { distTags?: unknown; distTagsInit?: { ok?: boolean; status?: number; statusText?: string }; - assetOk?: (version: string) => boolean; + assetOk?: (version: string, arch: 'amd64' | 'arm64') => boolean; packument?: unknown; }) { return vi.fn(async (url: string, init?: RequestInit) => { const method = init?.method ?? 'GET'; if (method === 'HEAD') { - const version = url.match(/supabase_(.+)_linux_/)?.[1]; + const match = url.match(/supabase_(.+)_linux_(amd64|arm64)\.deb$/); + const [, version, arch] = match ?? []; return headResponse( - version ? (routes.assetOk?.(version) ?? false) : false + version && arch + ? (routes.assetOk?.(version, arch as 'amd64' | 'arm64') ?? false) + : false ); } if (url === DIST_TAGS_URL) { @@ -85,8 +81,10 @@ describe('resolveCliVersion', () => { const headCalls = fetchMock.mock.calls.filter( ([, init]) => init?.method === 'HEAD' ); - expect(headCalls).toHaveLength(1); - expect(headCalls[0]?.[0]).toBe(debUrl('1.2.3')); + expect(headCalls.map(([url]) => url)).toEqual([ + cliDebUrl('1.2.3', 'amd64'), + cliDebUrl('1.2.3', 'arm64'), + ]); }); it('resolves the beta channel from the "beta" dist-tag when its asset exists', async () => { @@ -167,6 +165,16 @@ describe('resolveCliVersion', () => { ); }); + it('treats an array dist-tags response as invalid rather than a record', async () => { + const fetchMock = routedFetchMock({ distTags: ['not', 'a', 'record'] }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow( + /did not have a valid "latest" version for the stable channel/ + ); + }); + it('memoises a successful resolution so a second call does not refetch', async () => { const fetchMock = routedFetchMock({ distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, @@ -178,11 +186,12 @@ describe('resolveCliVersion', () => { await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); - // One GET (dist-tags) + one HEAD (asset check) — the second call hits the cache. - expect(fetchMock).toHaveBeenCalledTimes(2); + // One GET (dist-tags) + two HEAD (amd64+arm64 asset check) — the second + // call hits the cache. + expect(fetchMock).toHaveBeenCalledTimes(3); }); - it('clears the cache entry on rejection so a later call refetches', async () => { + it('clears the cache entry on rejection so a later call refetches, and keeps the retry memoised', async () => { let failNextDistTags = true; const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { if (init?.method === 'HEAD') return headResponse(true); @@ -204,8 +213,11 @@ describe('resolveCliVersion', () => { await expect(resolveCliVersion('stable')).rejects.toThrow('500'); await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + // A third call must still hit the cache populated by the successful + // retry — the rejected first promise's cleanup must not evict it. + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); - expect(fetchMock).toHaveBeenCalledTimes(3); + expect(fetchMock).toHaveBeenCalledTimes(4); }); it('does not let a beta rejection clear the stable cache entry', async () => { @@ -225,9 +237,9 @@ describe('resolveCliVersion', () => { ); await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); - // Two dist-tags GETs (stable, beta) + one HEAD (stable's asset check - // only — beta never gets that far). - expect(fetchMock).toHaveBeenCalledTimes(3); + // Two dist-tags GETs (stable, beta) + two HEAD (stable's amd64+arm64 + // asset check only — beta never gets that far). + expect(fetchMock).toHaveBeenCalledTimes(4); }); it('falls back to the newest published beta.N-1 when the dist-tag asset is a 404', async () => { @@ -259,8 +271,152 @@ describe('resolveCliVersion', () => { .filter(([, init]) => init?.method === 'HEAD') .map(([url]) => url); expect(headUrls).toEqual([ - debUrl('2.118.0-beta.52'), - debUrl('2.118.0-beta.51'), + cliDebUrl('2.118.0-beta.52', 'amd64'), + cliDebUrl('2.118.0-beta.52', 'arm64'), + cliDebUrl('2.118.0-beta.51', 'amd64'), + cliDebUrl('2.118.0-beta.51', 'arm64'), + ]); + }); + + it("falls back across a minor version boundary to the previous minor's newest beta", async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '2.118.0', beta: '2.119.0-beta.1' }, + assetOk: (version) => version === '2.118.0-beta.60', + packument: { + versions: { + '2.119.0-beta.1': {}, + '2.118.0-beta.60': {}, + '2.118.0-beta.59': {}, + '2.117.0': {}, + }, + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.60'); + }); + + it('orders beta.10 ahead of beta.9 during the walk-back (not a string compare)', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '2.118.0-beta.11' }, + // Only the unpublished dist-tag version (11) 404s; both walk-back + // candidates would succeed, so probe order is what decides the result. + assetOk: (version) => version !== '2.118.0-beta.11', + packument: { + versions: { + '2.118.0-beta.11': {}, + '2.118.0-beta.10': {}, + '2.118.0-beta.9': {}, + }, + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.10'); + + const headUrls = fetchMock.mock.calls + .filter(([, init]) => init?.method === 'HEAD') + .map(([url]) => url); + // A naive string compare would sort "beta.9" ahead of "beta.10" (since + // '9' > '1' character-wise), probing 9 before 10. + expect(headUrls).toEqual([ + cliDebUrl('2.118.0-beta.11', 'amd64'), + cliDebUrl('2.118.0-beta.11', 'arm64'), + cliDebUrl('2.118.0-beta.10', 'amd64'), + cliDebUrl('2.118.0-beta.10', 'arm64'), + ]); + }); + + it('never selects a beta candidate newer than the unpublished dist-tag version', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '2.118.0-beta.50' }, + // Every candidate downloads except the unpublished one — if the newer + // candidate (51) were ever probed, this would resolve to beta.51 + // instead of walking further back to beta.49. + assetOk: (version) => version !== '2.118.0-beta.50', + packument: { + versions: { + '2.118.0-beta.51': {}, + '2.118.0-beta.50': {}, + '2.118.0-beta.49': {}, + }, + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.49'); + }); + + it('treats a walk-back candidate whose asset check throws as unavailable and continues to the next', async () => { + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + if (init?.method === 'HEAD') { + if (url.includes('2.118.0-beta.52')) return headResponse(false); + if (url.includes('2.118.0-beta.51')) throw new Error('network blip'); + if (url.includes('2.118.0-beta.50')) return headResponse(true); + return headResponse(false); + } + if (url === DIST_TAGS_URL) { + return jsonResponse({ latest: '1.2.3', beta: '2.118.0-beta.52' }); + } + if (url === PACKUMENT_URL) { + return jsonResponse({ + versions: { + '2.118.0-beta.52': {}, + '2.118.0-beta.51': {}, + '2.118.0-beta.50': {}, + }, + }); + } + throw new Error(`unexpected fetch: ${url}`); + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.50'); + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining('2.118.0-beta.51') + ); + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining('network blip') + ); + + warnSpy.mockRestore(); + }); + + it('caps the beta walk-back at MAX_BETA_FALLBACK_CANDIDATES, probing newest-first', async () => { + const versions: Record = {}; + for (let n = 50; n <= 59; n += 1) versions[`2.118.0-beta.${n}`] = {}; + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '2.118.0-beta.60' }, + assetOk: () => false, + packument: { versions }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).rejects.toThrow(); + + const probedVersions = [ + ...new Set( + fetchMock.mock.calls + .filter(([, init]) => init?.method === 'HEAD') + .map(([url]) => url.match(/supabase_(.+)_linux_/)?.[1]) + ), + ]; + // The unpublished dist-tag version itself is checked first (outside the + // walk-back budget), followed by exactly MAX_BETA_FALLBACK_CANDIDATES (5) + // older published betas, newest-first. + expect(probedVersions).toEqual([ + '2.118.0-beta.60', + '2.118.0-beta.59', + '2.118.0-beta.58', + '2.118.0-beta.57', + '2.118.0-beta.56', + '2.118.0-beta.55', ]); }); @@ -292,7 +448,8 @@ describe('resolveCliVersion', () => { const { resolveCliVersion } = await import('../src/cli-channel.js'); await expect(resolveCliVersion('stable')).rejects.toThrow( - `HEAD ${debUrl('1.2.3')} was not ok` + 'checked amd64 and arm64 .deb assets at ' + + 'https://github.com/supabase/cli/releases/tag/v1.2.3' ); }); @@ -350,3 +507,38 @@ describe('resolveCliVersionOption', () => { await expect(resolveCliVersionOption('beta')).resolves.toBe('1.3.0-beta.1'); }); }); + +describe('compareBetaVersionsDesc', () => { + it('sorts newer beta numbers before older ones within the same minor', async () => { + const { compareBetaVersionsDesc } = await import('../src/cli-channel.js'); + + expect( + compareBetaVersionsDesc('2.118.0-beta.10', '2.118.0-beta.9') + ).toBeLessThan(0); + expect( + compareBetaVersionsDesc('2.118.0-beta.9', '2.118.0-beta.10') + ).toBeGreaterThan(0); + }); + + it('sorts a newer minor before an older minor regardless of beta number', async () => { + const { compareBetaVersionsDesc } = await import('../src/cli-channel.js'); + + expect( + compareBetaVersionsDesc('2.119.0-beta.1', '2.118.0-beta.60') + ).toBeLessThan(0); + }); + + it('treats equal versions as equal', async () => { + const { compareBetaVersionsDesc } = await import('../src/cli-channel.js'); + + expect(compareBetaVersionsDesc('2.118.0-beta.1', '2.118.0-beta.1')).toBe(0); + }); + + it('throws for a non "X.Y.Z-beta.N" version', async () => { + const { compareBetaVersionsDesc } = await import('../src/cli-channel.js'); + + expect(() => + compareBetaVersionsDesc('2.118.0-rc.1', '2.118.0-beta.1') + ).toThrow(); + }); +}); From 1518af50ea3246366c151385bfa2f4227de878ff Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Tue, 22 Sep 2026 12:30:36 +0100 Subject: [PATCH 04/12] chore(sandbox): trim narration and provenance comments in CLI channel resolution Strips agent-added comment cruft (module docblock essay, CI-run-ID call-out, code-shape narration) back to the invariants and external quirks the code can't express on its own; no behaviour change. --- apps/framework/scripts/run-vercel-evals.ts | 4 +- packages/sandbox/src/cli-channel.ts | 99 ++++++--------------- packages/sandbox/src/local-stack-runtime.ts | 12 +-- 3 files changed, 32 insertions(+), 83 deletions(-) diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index 4ceccb2b..ee231c2b 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -24,9 +24,7 @@ export const FORWARDED_ENV_NAMES = [ 'OPENAI_API_KEY', 'AI_GATEWAY_API_KEY', 'XAI_API_KEY', - // Pins the CLI channel versions the prepare job resolved for this run, so - // every sandbox job in the run scores against the same version instead of - // each independently re-resolving "latest" and drifting mid-run. + // Pins the CLI channel version resolved for this run across sandbox jobs. 'SUPABASE_CLI_STABLE_VERSION', 'SUPABASE_CLI_BETA_VERSION', ]; diff --git a/packages/sandbox/src/cli-channel.ts b/packages/sandbox/src/cli-channel.ts index ffe87024..4990a4e0 100644 --- a/packages/sandbox/src/cli-channel.ts +++ b/packages/sandbox/src/cli-channel.ts @@ -1,16 +1,7 @@ /** - * Resolves "latest stable" / "latest beta" Supabase CLI versions from npm's - * dist-tags for the `supabase` package — no auth, no rate-limit exposure - * (unlike the GitHub releases API), which matters because CI's sandbox only - * forwards the model API keys, never a GITHUB_TOKEN. - * - * npm's dist-tag can point at a version whose GitHub release is still a - * draft, so the resolved version's `.deb` release asset is HEAD-checked - * before being trusted; for the beta channel only, a missing asset walks - * back through older published `-beta.N` versions for one that downloads - * (see resolveFallbackBetaVersion). A stable dist-tag with a missing asset - * always throws instead — guessing at a "stable" release would defeat the - * point of the channel. + * Resolves "latest stable"/"latest beta" Supabase CLI versions from npm's + * dist-tags. npm can point at a still-draft release with no downloadable + * asset, so the resolved version's `.deb` is verified first. */ import { isRecord } from '@supabase-evals/core/json'; @@ -20,27 +11,20 @@ export type CliChannel = 'stable' | 'beta'; const NPM_DIST_TAGS_URL = 'https://registry.npmjs.org/-/package/supabase/dist-tags'; -// Abbreviated packument (versions + dist-tags only, no changelogs/READMEs) — -// this header is what makes npm serve the small form instead of the full one. +// Makes npm return the abbreviated packument instead of the full one. const NPM_PACKUMENT_URL = 'https://registry.npmjs.org/supabase'; const NPM_INSTALL_V1_ACCEPT = 'application/vnd.npm.install-v1+json'; // npm's beta dist-tag can carry a prerelease suffix like -rc.1, not just -beta.N. const VERSION_RE = /^\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?$/; -// Only the -beta.N shape is walkable for the "try an older published beta" -// fallback below; a -rc.N or other prerelease suffix has no defined ordering -// we can search. +// Only the -beta.N shape has a defined ordering the walk-back below can search. const BETA_SUFFIX_RE = /^\d+\.\d+\.\d+-beta\.\d+$/; -// Matches any published "X.Y.Z-beta.N" version (any major/minor/patch), -// captured for the numeric tuple comparison in compareBetaVersionsDesc — -// unlike BETA_SUFFIX_RE, it isn't anchored to one specific version's prefix. +// Feeds compareBetaVersionsDesc's numeric comparison; matches any version, unlike BETA_SUFFIX_RE. const BETA_VERSION_RE = /^(\d+)\.(\d+)\.(\d+)-beta\.(\d+)$/; -// The number of older published beta versions to probe for a downloadable -// asset before giving up — keeps a broken release from turning one nightly -// run into an unbounded chain of GitHub requests. +// Caps the walk-back so a broken release can't chain into unbounded GitHub requests. const MAX_BETA_FALLBACK_CANDIDATES = 5; const ENV_OVERRIDE: Record = { @@ -61,12 +45,12 @@ function isCliChannel(value: string): value is CliChannel { return CLI_CHANNELS.has(value as CliChannel); } -/** Mirrors installSupabaseCli's download URL template in packages/sandbox/src/supabase.ts. */ +/** Must match the download URL installSupabaseCli() uses in supabase.ts. */ export function cliDebUrl(version: string, arch: 'amd64' | 'arm64'): string { return `https://github.com/supabase/cli/releases/download/v${version}/supabase_${version}_linux_${arch}.deb`; } -/** The GitHub release page for a version, named in errors instead of a single arch's asset URL. */ +/** GitHub release page for a version, used in error messages. */ function releaseTagUrl(version: string): string { return `https://github.com/supabase/cli/releases/tag/v${version}`; } @@ -80,16 +64,7 @@ async function debAssetHeadOk(url: string): Promise { return response.ok; } -/** - * HEAD-checks that a release's `.deb` assets are actually downloadable (not - * behind a draft release) for both architectures the sandbox installs onto — - * the host running this resolver and the sandbox container it targets can - * differ (e.g. an Apple Silicon host building a linux/amd64 sandbox image), - * so checking only one arch could pass while the other 404s. This also - * catches a partially-uploaded release that a single-arch probe would miss. - * Two HEAD requests per candidate, capped at MAX_BETA_FALLBACK_CANDIDATES - * candidates in the walk-back below, is an acceptable request budget. - */ +/** HEAD-checks both the amd64 and arm64 `.deb` assets, since the resolver's host architecture need not match the sandbox's. */ async function debAssetExists(version: string): Promise { const [amd64, arm64] = await Promise.all([ debAssetHeadOk(cliDebUrl(version, 'amd64')), @@ -99,13 +74,10 @@ async function debAssetExists(version: string): Promise { } /** - * Resolve a channel to a concrete Supabase CLI version. Memoised per channel - * so a single nightly run only hits the registry once per channel; the cache - * entry is cleared on rejection so a later retry can hit the network again — - * guarded by an identity check so a stale rejection can never evict a - * different (newer) promise that has since taken its place in the cache. - * Never falls back to the pinned SUPABASE_CLI_VERSION on failure — that would - * silently mislabel data — so callers must let the throw propagate. + * Resolves a channel to a concrete CLI version, memoised per channel for the + * process lifetime. A rejection clears the cache entry so the next call + * retries; never falls back to the pinned SUPABASE_CLI_VERSION, since that + * would silently mislabel data. */ export async function resolveCliVersion(channel: CliChannel): Promise { const cached = versionCache.get(channel); @@ -119,12 +91,7 @@ export async function resolveCliVersion(channel: CliChannel): Promise { return promise; } -/** - * Resolves `value` if it names a channel (`'stable'` | `'beta'`); an exact - * version (or `undefined`) passes through unchanged. Split out from - * localStackRuntime's startSession so the branching itself is unit-testable - * without booting a sandbox. - */ +/** Resolves `value`: a channel tag (`'stable'` | `'beta'`) resolves against npm; an exact version or `undefined` passes through unchanged. */ export async function resolveCliVersionOption( value: string | CliChannel | undefined ): Promise { @@ -164,15 +131,13 @@ async function resolveCliVersionUncached(channel: CliChannel): Promise { ); } - // Deliberately not caught: a transient failure here (DNS blip, timeout) - // must throw loud rather than silently walk back to an older beta, or - // (for stable) never walk back at all. + // A transient failure here throws rather than silently walking back to an + // older beta (or never walking back, for stable). if (await debAssetExists(version)) return version; - // The dist-tag pointed at a version whose GitHub release has no - // downloadable .deb (e.g. still a draft) — this is exactly what broke the - // beta channel in CI run 35274970438. A stable release is never guessed at; - // only beta walks back to the newest published version that does have one. + // npm's dist-tag can point at a version whose GitHub release is still a + // draft with no downloadable .deb. Only beta walks back to an older + // published version; stable is never guessed at. if (channel === 'stable') { throw new Error( `npm's "latest" dist-tag for "supabase" points at ${version}, but its ` + @@ -185,18 +150,10 @@ async function resolveCliVersionUncached(channel: CliChannel): Promise { } /** - * Walks back through every published npm version matching the `X.Y.Z-beta.N` - * shape that is strictly older than `unpublishedVersion` — not just versions - * sharing its exact minor — newest first, and returns the first one whose - * release `.deb` assets actually download. This lets a brand-new minor's - * first beta (e.g. `2.119.0-beta.1`, still a draft) fall back across the - * minor boundary to the previous minor's newest published beta (e.g. - * `2.118.0-beta.60`). A candidate newer than `unpublishedVersion` is never - * considered — if npm's dist-tag skipped it, it's likelier to be a draft too. - * - * A transient error probing one candidate (network blip, timeout) is logged - * and skipped rather than aborting the whole walk-back, so one bad candidate - * doesn't waste the rest of the probe budget. + * Walks back through published `X.Y.Z-beta.N` versions strictly older than + * `unpublishedVersion`, newest first, for one with a downloadable `.deb`. + * Newer candidates are skipped too (likelier to be drafts); a transient + * probe error is logged and skipped rather than aborting the walk-back. */ async function resolveFallbackBetaVersion( unpublishedVersion: string @@ -268,11 +225,9 @@ function parseBetaVersion( } /** - * Orders two "X.Y.Z-beta.N" version strings newest-first — usable directly as - * an `Array#sort` comparator for descending order. Compares each component - * numerically so e.g. `2.118.0-beta.10` correctly sorts ahead of - * `2.118.0-beta.9`; a plain string compare would invert that. Throws if - * either string isn't a parseable `X.Y.Z-beta.N` version. + * Orders two `X.Y.Z-beta.N` versions newest-first, usable as an + * `Array#sort` comparator; compares components numerically so `beta.10` + * sorts ahead of `beta.9`. Throws if either string isn't parseable. */ export function compareBetaVersionsDesc(a: string, b: string): number { const tupleA = parseBetaVersion(a); diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index 392d4dad..626632ac 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -38,14 +38,10 @@ const STACK_CONFIG_RETRY_MS = 2_000; */ export interface LocalStackRuntimeOptions { /** - * Supabase CLI version baked into the sandbox image (pinned default). - * Either an exact version (e.g. `2.109.1`) or a channel tag (`'stable'` | - * `'beta'`) that tracks npm's dist-tag for the `supabase` package — a - * channel tag's resolution is memoised for the lifetime of the process - * (see resolveCliVersion), not per session, so a later session in the same - * run will not pick up a newly published version. An eval's own - * `cliVersion:` frontmatter pin always wins over this option, whether this - * is an exact version or a channel tag. + * Supabase CLI version baked into the sandbox image: an exact version + * (e.g. `2.109.1`) or a channel tag (`'stable'` | `'beta'`) resolved + * against npm's dist-tag and memoised per process. An eval's own + * `cliVersion:` frontmatter pin always wins over this option. */ cliVersion?: CliChannel | (string & {}); /** From 81c56211cf5a478b96103097d0dfd049e2387448 Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Tue, 22 Sep 2026 12:38:56 +0100 Subject: [PATCH 05/12] chore(sandbox): trim narration in docker-availability comments Condense multi-paragraph comments in local-stack-runtime.ts down to the non-obvious fact each one exists to preserve, dropping restated code sequences and step-by-step narration. --- packages/sandbox/src/local-stack-runtime.ts | 107 +++++++------------- 1 file changed, 36 insertions(+), 71 deletions(-) diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index c0120a9f..5e67e455 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -75,17 +75,10 @@ export interface LocalStackRuntimeOptions { mcpServers?: Record; /** * Docker availability to stage in the sandbox (default `'available'`). - * `'no-daemon'` and `'absent'` stage a sandbox with no Docker at all — the - * socket is never bind-mounted and `DOCKER_HOST` points at an unreachable - * address — so an eval can exercise the Supabase CLI's behavior when - * Docker is missing, rather than merely simulated from inside a working - * Docker sandbox. `'no-daemon'` additionally shims the `docker` binary so - * `docker --version` keeps working (the CLI's runtime probe, - * supabase/cli#6563, must still *choose* Docker before discovering it - * can't reach it); `'absent'` removes the binary outright. Docker-less - * sessions require `projectRunning: false` and no hosted project link — - * the harness cannot pre-start a stack or link a hosted project without - * Docker. + * `'no-daemon'` leaves `docker --version` working but unreachable; + * `'absent'` removes the binary entirely. Both require + * `projectRunning: false` and no hosted project link, since the harness + * cannot pre-start a stack or link a hosted project without Docker. */ docker?: DockerState; } @@ -97,25 +90,19 @@ export interface LocalStackRuntimeOptions { export type DockerState = 'available' | 'no-daemon' | 'absent'; /** - * Path of the marker each local-stack session writes recording the - * environment it staged, for scorers to *report* — never to decide - * pass/fail, since branching on `docker` there would grade an eval against - * its own environment. On the `'available'` path this is written through - * the session's own `scoringContext.exec`, which has no root access inside - * the sandbox, so the marker is agent-writable there — acceptable since - * it's metrics-only. On the Docker-less paths it's written root-owned and - * read-only (mode 0444), so the agent cannot rewrite it to fake the - * environment it's being graded in. + * Path of the marker each session writes recording its staged environment, + * for scorers to report, never to gate pass/fail. Agent-writable on the + * `'available'` path (metrics-only); root-owned and read-only (0444) on the + * Docker-less paths, so the agent can't fake the environment it's graded in. */ export const LOCAL_STACK_MARKER_PATH = '/tmp/supabase-eval-runtime.json'; export type LocalStackEnvironmentMarker = { runtime: 'local-stack'; /** - * The channel `cliVersion` was resolved from, when the runtime option - * named one (`'stable'` | `'beta'`) and no per-eval `cliVersion:` - * frontmatter pin overrode it. Undefined for an exact-version pin, whether - * from the runtime option or the per-eval override. + * The channel `cliVersion` resolved from, when the runtime option named + * one and no per-eval `cliVersion:` pin overrode it. Undefined for any + * exact-version pin. */ channel?: CliChannel; cliVersion: string; @@ -128,8 +115,8 @@ export type LocalStackEnvironmentMarker = { const SUPABASE_SHIM_PATH = '/usr/local/sbin/supabase'; const DOCKER_SHIM_PATH = '/usr/local/sbin/docker'; -// Port 1 is never bound (the CLI's own e2e suite reserves it for exactly this -// purpose); not 2375, which Docker Desktop can legitimately expose. +// Port 1 is never bound (the CLI's own e2e suite reserves it for this too); +// not 2375, which Docker Desktop can legitimately expose. const UNREACHABLE_DOCKER_HOST = 'tcp://127.0.0.1:1'; /** @@ -163,9 +150,7 @@ export function localStackRuntime( // Stamped before setup so it's comparable with the scorer's PID-1 fallback. const sessionStartedMs = Date.now(); const docker = options.docker ?? 'available'; - // An eval's own `cliVersion:` pin always wins over the runtime option - // (whether that option is an exact version or a channel), so only an - // unpinned eval can inherit a channel for the marker below. + // Only an unpinned eval inherits a channel; an eval's own pin always wins. const channel = cliVersion === undefined && options.cliVersion !== undefined && @@ -221,14 +206,8 @@ export function localStackRuntime( }, }; - // The scoring context's exec has no root access inside the sandbox, - // so this marker is agent-writable on this path — fine here since - // it's metrics-only, unlike the Docker-less path's root-owned marker - // below. Best-effort for the same reason: this is the path every - // experiment takes, and nothing about a default session is worth - // failing a whole run over. The Docker-less path *does* fail hard, - // because there the marker is the evidence that the environment was - // staged as claimed. + // Metrics-only and best-effort here; the Docker-less path's marker + // is root-owned and fails hard instead. try { await writeLocalStackMarkerViaExec( (command) => session.scoringContext.exec(command), @@ -244,8 +223,8 @@ export function localStackRuntime( return session; } - // Docker-less staging (docker === 'no-daemon' | 'absent'): the harness - // cannot pre-start a stack or link a hosted project without Docker. + // Docker-less staging: the harness cannot pre-start a stack or link a + // hosted project without Docker. if (projectRunning !== false) { throw new Error( 'docker-less sandbox evals must set `projectRunning: false`; the harness cannot pre-start a stack without Docker' @@ -270,13 +249,11 @@ export function localStackRuntime( await installSupabaseCli(sandbox, version); } - // Deliberately no socket-group grant here (unlike setupSupabaseSandbox): - // CI's sandbox already ends its Docker setup with `chmod 666 - // /var/run/docker.sock`, so the grant would be a no-op there — - // DOCKER_HOST below is the real mechanism. For `absent`, a real - // Docker-less host wouldn't have DOCKER_HOST set at all — but leaving - // it set here costs nothing and blocks any future accidental socket - // exposure, so it stays for both states. + // No socket-group grant (unlike setupSupabaseSandbox): CI's sandbox + // setup ends with `chmod 666 /var/run/docker.sock`, which would make + // any permission-based restriction a no-op — DOCKER_HOST below is + // what actually blocks access. Left set for `absent` too since it's + // harmless and guards against future socket exposure. sandbox.extraEnv = { ...sandbox.extraEnv, DOCKER_HOST: UNREACHABLE_DOCKER_HOST, @@ -310,9 +287,8 @@ export function localStackRuntime( ); } - // The root shell above has a different PATH than the agent's - // SANDBOX_PATH, so also assert the binary is gone from the PATH the - // agent actually runs commands under. + // The root shell's PATH differs from the agent's SANDBOX_PATH, so + // also assert the binary is gone from the PATH the agent runs under. const pathCheck = await sandbox.runShell( '! command -v docker >/dev/null 2>&1' ); @@ -331,8 +307,8 @@ export function localStackRuntime( } const installedSkills = await installSkills(sandbox, skills ?? []); - // With no bind mount, the socket must not exist at all — a stronger - // guarantee than merely checking reachability from inside the container. + // No bind mount means the socket must not exist at all — stronger + // than checking reachability from inside the container. const socketAbsent = await sandbox.runShellAsRoot( 'test ! -e /var/run/docker.sock' ); @@ -360,11 +336,8 @@ export function localStackRuntime( return { tools: buildLocalStackTools(sandbox), sandbox: toAgentSandbox(sandbox), - // Same wiring as the `available` path: an experiment's explicit - // `mcpServers`/`mcpFeatures` must not be silently dropped just - // because Docker is missing. `hosted` is always undefined here - // (guarded above), so this resolves to the platform-independent - // docs server unless the experiment asked for something else. + // Same wiring as the `available` path; `hosted` is always + // undefined here (guarded above), so this falls back to `docs`. mcpServers: dockerlessMcpServers, promptAddendum: [ buildToolSurfaceAddendum(agent, { skipCliInstall }), @@ -389,12 +362,9 @@ export function localStackRuntime( }; } -/** - * The runtime's log-line id (see run-eval.ts's PLAN line): plain - * `'local-stack'` for default options, otherwise the non-default bits - * appended so e.g. a beta + absent runtime reads as - * `local-stack-beta-absent` in the run log. - */ +// Log-line id (see run-eval.ts's PLAN line): default options read as +// 'local-stack', otherwise the non-default bits are appended, e.g. +// 'local-stack-beta-absent'. function buildRuntimeId(options: LocalStackRuntimeOptions): string { const bits: string[] = []; if (options.cliVersion !== undefined) bits.push(options.cliVersion); @@ -459,14 +429,9 @@ async function installSupabaseShim( } /** - * Shim that shadows `supabase` on PATH in a Docker-less sandbox. On the - * `'available'` path the *harness* runs `supabase start` itself and applies - * `services:` exclusions via `buildSupabaseStartCommand`; without Docker the - * harness cannot pre-start anything, so the *agent* runs `supabase start` - * instead, and this shim is the only seam left through which the eval's - * `includeServices` (`services:` frontmatter) still gets honored — it - * injects the same `-x ` flag (`computeExcludedServices`) into - * whatever `supabase start` the agent types. + * Shim that shadows `supabase` on PATH in a Docker-less sandbox. Here the + * agent, not the harness, runs `supabase start`, so this is the only route + * by which the eval's `includeServices` still gets applied (`-x `). */ export function buildSupabaseShimScript( realBin: string, @@ -502,7 +467,7 @@ export function buildDockerDaemonShimScript(dockerVersion: string): string { '#!/bin/bash', 'case "$1" in', // --version must keep working so the CLI's runtime probe - // (supabase/cli#6563) still *chooses* Docker as its runtime. + // (supabase/cli#6563) still chooses Docker as its runtime. ` --version|-v) echo ${shellQuote(dockerVersion)}; exit 0 ;;`, 'esac', `echo "Cannot connect to the Docker daemon at ${UNREACHABLE_DOCKER_HOST}. Is the docker daemon running?" >&2`, From 945d77b692ae52d6e2c76a8702ab9c1aac89473f Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Tue, 22 Sep 2026 16:45:53 +0100 Subject: [PATCH 06/12] fix(sandbox): treat GitHub HEAD errors as failures and pin CLI channels once per run debAssetHeadOk conflated a GitHub 429/5xx with a missing asset (response.ok false for both), which could downgrade a beta release during a GitHub outage and record results under the wrong version. Only a 404 now means "asset not published"; any other non-2xx throws. The beta walk-back loop is updated to match: a 404 still advances to the next candidate, but a thrown error aborts the walk-back and propagates instead of being caught and skipped. Separately, nothing set SUPABASE_CLI_STABLE_VERSION/SUPABASE_CLI_BETA_VERSION, so each sandbox job resolved its own CLI channel version independently, risking two different versions in one fan-out if a release landed mid-run. runPairs now resolves each channel pin once via resolveChannelPins() before runBounded and threads the same pins into every job's .env write. --- .../scripts/run-vercel-evals.test.ts | 93 ++++++++++++++++++- apps/framework/scripts/run-vercel-evals.ts | 41 +++++++- packages/sandbox/src/cli-channel.ts | 20 ++-- packages/sandbox/test/cli-channel.test.ts | 91 +++++++++++++++--- 4 files changed, 217 insertions(+), 28 deletions(-) diff --git a/apps/framework/scripts/run-vercel-evals.test.ts b/apps/framework/scripts/run-vercel-evals.test.ts index 7da08d92..2161f956 100644 --- a/apps/framework/scripts/run-vercel-evals.test.ts +++ b/apps/framework/scripts/run-vercel-evals.test.ts @@ -1,4 +1,5 @@ import { APIError } from '@vercel/sandbox'; +import { resolveCliVersion } from '@supabase-evals/sandbox'; import { execFileSync } from 'node:child_process'; import { copyFileSync, @@ -11,7 +12,7 @@ import { } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { agentEnvironment, cleanupSandbox, @@ -22,11 +23,14 @@ import { isTerminalSandboxCreateError, packWorkspaceScript, parsePairs, + resolveChannelPins, runBounded, tagValue, expandJobs, } from './run-vercel-evals.js'; +vi.mock('@supabase-evals/sandbox', () => ({ resolveCliVersion: vi.fn() })); + describe('agentEnvironment', () => { const originalValues = new Map(); @@ -63,6 +67,93 @@ describe('agentEnvironment', () => { expect(env).not.toContain('SUPABASE_CLI_STABLE_VERSION'); expect(env).not.toContain('SUPABASE_CLI_BETA_VERSION'); }); + + it('prefers an explicit pin over the same-named process.env value', () => { + process.env.SUPABASE_CLI_STABLE_VERSION = 'env-value'; + + const env = agentEnvironment({ + SUPABASE_CLI_STABLE_VERSION: 'pinned-value', + }); + + expect(env).toContain('SUPABASE_CLI_STABLE_VERSION=pinned-value'); + expect(env).not.toContain('env-value'); + }); + + it('shares one pin value across multiple .env writes, simulating a two-job fan-out', () => { + const pins = { + SUPABASE_CLI_STABLE_VERSION: '2.117.0', + SUPABASE_CLI_BETA_VERSION: '2.118.0-beta.5', + }; + + const jobOneEnv = agentEnvironment(pins); + const jobTwoEnv = agentEnvironment(pins); + + expect(jobOneEnv).toBe(jobTwoEnv); + expect(jobOneEnv).toContain('SUPABASE_CLI_STABLE_VERSION=2.117.0'); + expect(jobOneEnv).toContain('SUPABASE_CLI_BETA_VERSION=2.118.0-beta.5'); + }); +}); + +describe('resolveChannelPins', () => { + const STABLE_ENV = 'SUPABASE_CLI_STABLE_VERSION'; + const BETA_ENV = 'SUPABASE_CLI_BETA_VERSION'; + const originalValues = new Map(); + + beforeEach(() => { + for (const name of [STABLE_ENV, BETA_ENV]) { + originalValues.set(name, process.env[name]); + delete process.env[name]; + } + vi.mocked(resolveCliVersion).mockReset(); + }); + + afterEach(() => { + for (const [name, value] of originalValues) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + }); + + it('resolves each channel once and returns a pin every job can share', async () => { + vi.mocked(resolveCliVersion).mockImplementation(async (channel) => + channel === 'stable' ? '2.117.0' : '2.118.0-beta.5' + ); + + const pins = await resolveChannelPins(); + + expect(resolveCliVersion).toHaveBeenCalledTimes(2); + expect(resolveCliVersion).toHaveBeenCalledWith('stable'); + expect(resolveCliVersion).toHaveBeenCalledWith('beta'); + + // Two fanned-out jobs writing their own .env from the same pins object + // must get the identical value the resolver was called once for. + const jobOneEnv = agentEnvironment(pins); + const jobTwoEnv = agentEnvironment(pins); + expect(jobOneEnv).toBe(jobTwoEnv); + expect(jobOneEnv).toContain(`${STABLE_ENV}=2.117.0`); + expect(jobOneEnv).toContain(`${BETA_ENV}=2.118.0-beta.5`); + }); + + it('uses an already-set env var verbatim without calling the resolver', async () => { + process.env[STABLE_ENV] = '9.9.9'; + vi.mocked(resolveCliVersion).mockImplementation( + async () => '2.118.0-beta.5' + ); + + const pins = await resolveChannelPins(); + + expect(pins[STABLE_ENV]).toBe('9.9.9'); + expect(resolveCliVersion).toHaveBeenCalledTimes(1); + expect(resolveCliVersion).toHaveBeenCalledWith('beta'); + }); + + it('propagates a resolution failure rather than swallowing it', async () => { + vi.mocked(resolveCliVersion).mockRejectedValue( + new Error('npm unreachable') + ); + + await expect(resolveChannelPins()).rejects.toThrow('npm unreachable'); + }); }); describe('Vercel eval controller', () => { diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index ee231c2b..e06666fa 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -6,6 +6,7 @@ import { sandboxUsageSchema, type SandboxUsage, } from '@supabase-evals/core/eval-metadata'; +import { resolveCliVersion, type CliChannel } from '@supabase-evals/sandbox'; import { execFileSync } from 'node:child_process'; import { readFileSync, renameSync, writeFileSync } from 'node:fs'; import { join, resolve } from 'node:path'; @@ -28,6 +29,10 @@ export const FORWARDED_ENV_NAMES = [ 'SUPABASE_CLI_STABLE_VERSION', 'SUPABASE_CLI_BETA_VERSION', ]; +const CLI_CHANNEL_ENV: Record = { + stable: 'SUPABASE_CLI_STABLE_VERSION', + beta: 'SUPABASE_CLI_BETA_VERSION', +}; /** * Slack for the non-agent work inside `pnpm eval` (supabase start, resets, * scoring, export). Cold image pulls alone can take ~10 min. @@ -75,6 +80,29 @@ interface PairOptions extends RunnerOptions { pair: EvalPair; run: number; attempt: number; + pins: Record; +} + +/** + * Resolves each CLI channel's pin once, so every sandbox job in a fan-out + * runs against the same concrete version rather than each resolving + * independently and risking a mid-run release landing between them. An + * already-set env var is used verbatim, matching the workflow/manual + * override path in resolveCliVersion. + */ +export async function resolveChannelPins(): Promise> { + const resolved = await Promise.all( + (Object.entries(CLI_CHANNEL_ENV) as [CliChannel, string][]).map( + async ([channel, envVar]) => { + const override = process.env[envVar]; + return [ + envVar, + override ?? (await resolveCliVersion(channel)), + ] as const; + } + ) + ); + return Object.fromEntries(resolved); } interface SandboxCommandOptions { @@ -113,6 +141,10 @@ async function runPairs(options: RunnerOptions): Promise { `max ${options.concurrency} at a time` ); + // Resolved once so every job below writes the same pin, rather than each + // sandbox resolving its own channel version independently. + const pins = await resolveChannelPins(); + const results = await runBounded( jobs, options.concurrency, @@ -126,6 +158,7 @@ async function runPairs(options: RunnerOptions): Promise { pair, run, attempt, + pins, }, credentials ), @@ -269,7 +302,7 @@ async function runPairOnce( await sandbox.writeFiles([ { path: '.env', - content: `${agentEnvironment()}\n`, + content: `${agentEnvironment(options.pins)}\n`, }, ]); console.log(`${label} run eval`); @@ -625,11 +658,11 @@ function vercelCredentialsFromEnv(): { }; } -/** Serializes configured provider keys and CLI channel pins into the sandbox's `.env` file. */ -export function agentEnvironment(): string { +/** Serializes configured provider keys and CLI channel pins into the sandbox's `.env` file, preferring an explicit pin over the same-named process.env value. */ +export function agentEnvironment(pins: Record = {}): string { const lines: string[] = []; for (const name of FORWARDED_ENV_NAMES) { - const value = process.env[name]; + const value = pins[name] ?? process.env[name]; if (value) lines.push(`${name}=${value}`); } return lines.join('\n'); diff --git a/packages/sandbox/src/cli-channel.ts b/packages/sandbox/src/cli-channel.ts index 4990a4e0..7bc45c67 100644 --- a/packages/sandbox/src/cli-channel.ts +++ b/packages/sandbox/src/cli-channel.ts @@ -55,13 +55,16 @@ function releaseTagUrl(version: string): string { return `https://github.com/supabase/cli/releases/tag/v${version}`; } +/** Only a 404 means the asset is absent; any other non-2xx (e.g. a GitHub 429/5xx) is an error, not a missing-asset signal. */ async function debAssetHeadOk(url: string): Promise { const response = await fetch(url, { method: 'HEAD', redirect: 'follow', signal: AbortSignal.timeout(15_000), }); - return response.ok; + if (response.ok) return true; + if (response.status === 404) return false; + throw new Error(`HEAD ${url} -> ${response.status} ${response.statusText}`); } /** HEAD-checks both the amd64 and arm64 `.deb` assets, since the resolver's host architecture need not match the sandbox's. */ @@ -152,8 +155,10 @@ async function resolveCliVersionUncached(channel: CliChannel): Promise { /** * Walks back through published `X.Y.Z-beta.N` versions strictly older than * `unpublishedVersion`, newest first, for one with a downloadable `.deb`. - * Newer candidates are skipped too (likelier to be drafts); a transient - * probe error is logged and skipped rather than aborting the walk-back. + * Newer candidates are skipped too (likelier to be drafts). Only a 404 + * advances to the next candidate; any other probe error aborts the + * walk-back, since it means GitHub is unhealthy rather than that the + * candidate is absent. */ async function resolveFallbackBetaVersion( unpublishedVersion: string @@ -194,14 +199,7 @@ async function resolveFallbackBetaVersion( .slice(0, MAX_BETA_FALLBACK_CANDIDATES); for (const candidate of candidates) { - try { - if (await debAssetExists(candidate)) return candidate; - } catch (error) { - console.warn( - `[cli-channel] beta fallback candidate ${candidate} could not be checked, skipping: ` + - (error instanceof Error ? error.message : String(error)) - ); - } + if (await debAssetExists(candidate)) return candidate; } throw new Error( diff --git a/packages/sandbox/test/cli-channel.test.ts b/packages/sandbox/test/cli-channel.test.ts index 8bc1353a..95a34cbd 100644 --- a/packages/sandbox/test/cli-channel.test.ts +++ b/packages/sandbox/test/cli-channel.test.ts @@ -19,8 +19,12 @@ function jsonResponse( }; } -function headResponse(ok: boolean) { - return { ok, status: ok ? 200 : 404, statusText: ok ? 'OK' : 'Not Found' }; +function headResponse(ok: boolean, status?: number, statusText?: string) { + return { + ok, + status: status ?? (ok ? 200 : 404), + statusText: statusText ?? (ok ? 'OK' : 'Not Found'), + }; } /** @@ -350,12 +354,13 @@ describe('resolveCliVersion', () => { await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.49'); }); - it('treats a walk-back candidate whose asset check throws as unavailable and continues to the next', async () => { - const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + it('aborts the walk-back and propagates when a candidate probe throws, rather than continuing to the next candidate', async () => { const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { if (init?.method === 'HEAD') { if (url.includes('2.118.0-beta.52')) return headResponse(false); - if (url.includes('2.118.0-beta.51')) throw new Error('network blip'); + if (url.includes('2.118.0-beta.51')) { + return headResponse(false, 500, 'Internal Server Error'); + } if (url.includes('2.118.0-beta.50')) return headResponse(true); return headResponse(false); } @@ -376,15 +381,50 @@ describe('resolveCliVersion', () => { vi.stubGlobal('fetch', fetchMock); const { resolveCliVersion } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.50'); - expect(warnSpy).toHaveBeenCalledWith( - expect.stringContaining('2.118.0-beta.51') - ); - expect(warnSpy).toHaveBeenCalledWith( - expect.stringContaining('network blip') + await expect(resolveCliVersion('beta')).rejects.toThrow( + /2\.118\.0-beta\.51.*-> 500 Internal Server Error/ ); - warnSpy.mockRestore(); + const headUrls = fetchMock.mock.calls + .filter(([, init]) => init?.method === 'HEAD') + .map(([url]) => url); + // beta.50 is never probed — the throw on beta.51 aborts the walk-back. + expect(headUrls).toEqual([ + cliDebUrl('2.118.0-beta.52', 'amd64'), + cliDebUrl('2.118.0-beta.52', 'arm64'), + cliDebUrl('2.118.0-beta.51', 'amd64'), + cliDebUrl('2.118.0-beta.51', 'arm64'), + ]); + }); + + it('advances to the second walk-back candidate when the first is a 404', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '2.118.0-beta.52' }, + assetOk: (version) => version === '2.118.0-beta.50', + packument: { + versions: { + '2.118.0-beta.52': {}, + '2.118.0-beta.51': {}, + '2.118.0-beta.50': {}, + }, + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.50'); + + const headUrls = fetchMock.mock.calls + .filter(([, init]) => init?.method === 'HEAD') + .map(([url]) => url); + expect(headUrls).toEqual([ + cliDebUrl('2.118.0-beta.52', 'amd64'), + cliDebUrl('2.118.0-beta.52', 'arm64'), + cliDebUrl('2.118.0-beta.51', 'amd64'), + cliDebUrl('2.118.0-beta.51', 'arm64'), + cliDebUrl('2.118.0-beta.50', 'amd64'), + cliDebUrl('2.118.0-beta.50', 'arm64'), + ]); }); it('caps the beta walk-back at MAX_BETA_FALLBACK_CANDIDATES, probing newest-first', async () => { @@ -453,6 +493,33 @@ describe('resolveCliVersion', () => { ); }); + it('throws naming the URL and status when the dist-tag asset HEAD is a GitHub error, without fetching the packument', async () => { + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + if (init?.method === 'HEAD') { + return url.includes('_amd64') + ? headResponse(false, 500, 'Internal Server Error') + : headResponse(true); + } + if (url === DIST_TAGS_URL) { + return jsonResponse({ latest: '1.2.3', beta: '2.118.0-beta.52' }); + } + if (url === PACKUMENT_URL) { + throw new Error('packument must not be fetched'); + } + throw new Error(`unexpected fetch: ${url}`); + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).rejects.toThrow( + `${cliDebUrl('2.118.0-beta.52', 'amd64')} -> 500 Internal Server Error` + ); + + expect(fetchMock.mock.calls.some(([url]) => url === PACKUMENT_URL)).toBe( + false + ); + }); + it('does not hit the network at all for an env override', async () => { process.env[STABLE_ENV] = '9.9.9'; process.env[BETA_ENV] = '9.9.9-beta.1'; From 18f83928b0ce0215959458d0292da7c051219b1e Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Tue, 22 Sep 2026 18:12:19 +0100 Subject: [PATCH 07/12] fix(sandbox): resolve only the CLI channels a run's pairs actually need Union each pair's effective channel (exact eval pins win and need none; otherwise the pair's experiment's localStack.cliChannel) before fan-out, instead of unconditionally resolving both channels. A run with no CLI-channel experiments now does zero npm/GitHub lookups. An experiment that can't be resolved to a config throws instead of silently counting as needing no channel. --- .../scripts/run-vercel-evals.test.ts | 113 +++++++++++++++++- apps/framework/scripts/run-vercel-evals.ts | 110 ++++++++++++++--- packages/core/src/index.ts | 2 + packages/sandbox/src/cli-channel.ts | 2 +- packages/sandbox/src/local-stack-runtime.ts | 10 +- 5 files changed, 212 insertions(+), 25 deletions(-) diff --git a/apps/framework/scripts/run-vercel-evals.test.ts b/apps/framework/scripts/run-vercel-evals.test.ts index 2161f956..ab4cec9a 100644 --- a/apps/framework/scripts/run-vercel-evals.test.ts +++ b/apps/framework/scripts/run-vercel-evals.test.ts @@ -1,5 +1,5 @@ import { APIError } from '@vercel/sandbox'; -import { resolveCliVersion } from '@supabase-evals/sandbox'; +import { resolveCliVersion, type CliChannel } from '@supabase-evals/sandbox'; import { execFileSync } from 'node:child_process'; import { copyFileSync, @@ -23,10 +23,12 @@ import { isTerminalSandboxCreateError, packWorkspaceScript, parsePairs, + requiredCliChannels, resolveChannelPins, runBounded, tagValue, expandJobs, + type EvalPair, } from './run-vercel-evals.js'; vi.mock('@supabase-evals/sandbox', () => ({ resolveCliVersion: vi.fn() })); @@ -97,6 +99,7 @@ describe('agentEnvironment', () => { describe('resolveChannelPins', () => { const STABLE_ENV = 'SUPABASE_CLI_STABLE_VERSION'; const BETA_ENV = 'SUPABASE_CLI_BETA_VERSION'; + const BOTH_CHANNELS = new Set(['stable', 'beta']); const originalValues = new Map(); beforeEach(() => { @@ -114,12 +117,12 @@ describe('resolveChannelPins', () => { } }); - it('resolves each channel once and returns a pin every job can share', async () => { + it('resolves only the requested channels and returns a pin every job can share', async () => { vi.mocked(resolveCliVersion).mockImplementation(async (channel) => channel === 'stable' ? '2.117.0' : '2.118.0-beta.5' ); - const pins = await resolveChannelPins(); + const pins = await resolveChannelPins(BOTH_CHANNELS); expect(resolveCliVersion).toHaveBeenCalledTimes(2); expect(resolveCliVersion).toHaveBeenCalledWith('stable'); @@ -134,13 +137,32 @@ describe('resolveChannelPins', () => { expect(jobOneEnv).toContain(`${BETA_ENV}=2.118.0-beta.5`); }); + it('resolves only stable when that is the only requested channel, never calling resolveCliVersion with beta', async () => { + vi.mocked(resolveCliVersion).mockImplementation(async (channel) => + channel === 'stable' ? '2.117.0' : '2.118.0-beta.5' + ); + + const pins = await resolveChannelPins(new Set(['stable'])); + + expect(pins).toEqual({ [STABLE_ENV]: '2.117.0' }); + expect(resolveCliVersion).toHaveBeenCalledTimes(1); + expect(resolveCliVersion).not.toHaveBeenCalledWith('beta'); + }); + + it('does no network work for an empty channel set', async () => { + const pins = await resolveChannelPins(new Set()); + + expect(pins).toEqual({}); + expect(resolveCliVersion).not.toHaveBeenCalled(); + }); + it('uses an already-set env var verbatim without calling the resolver', async () => { process.env[STABLE_ENV] = '9.9.9'; vi.mocked(resolveCliVersion).mockImplementation( async () => '2.118.0-beta.5' ); - const pins = await resolveChannelPins(); + const pins = await resolveChannelPins(BOTH_CHANNELS); expect(pins[STABLE_ENV]).toBe('9.9.9'); expect(resolveCliVersion).toHaveBeenCalledTimes(1); @@ -152,7 +174,88 @@ describe('resolveChannelPins', () => { new Error('npm unreachable') ); - await expect(resolveChannelPins()).rejects.toThrow('npm unreachable'); + await expect( + resolveChannelPins(new Set(['stable'])) + ).rejects.toThrow('npm unreachable'); + }); +}); + +describe('requiredCliChannels', () => { + const pair = (overrides: Partial = {}): EvalPair => ({ + eval_id: 'eval-1', + experiment: 'experiment-1', + experiment_suite: 'benchmark', + eval_suite: 'benchmark', + ...overrides, + }); + + it('resolves only the channel a stable-tagged experiment declares', async () => { + const loadExperimentConfig = vi.fn(async () => ({ + localStack: { cliChannel: 'stable' as const }, + })); + + const channels = await requiredCliChannels([pair()], { + loadEvalMetadata: () => ({ cliVersion: undefined }), + loadExperimentConfig, + }); + + expect(channels).toEqual(new Set(['stable'])); + }); + + it('resolves nothing when no experiment in the pair set declares a channel', async () => { + const loadExperimentConfig = vi.fn(async () => ({})); + + const channels = await requiredCliChannels( + [pair(), pair({ eval_id: 'eval-2', experiment: 'experiment-2' })], + { + loadEvalMetadata: () => ({ cliVersion: undefined }), + loadExperimentConfig, + } + ); + + expect(channels.size).toBe(0); + }); + + it("an eval's pinned cliVersion contributes no channel, even when its experiment declares one", async () => { + const loadExperimentConfig = vi.fn(async () => ({ + localStack: { cliChannel: 'beta' as const }, + })); + + const channels = await requiredCliChannels([pair()], { + loadEvalMetadata: () => ({ cliVersion: '2.109.1' }), + loadExperimentConfig, + }); + + expect(channels.size).toBe(0); + expect(loadExperimentConfig).not.toHaveBeenCalled(); + }); + + it('unions channels across pairs without resolving an experiment config twice', async () => { + const loadExperimentConfig = vi.fn(async () => ({ + localStack: { cliChannel: 'beta' as const }, + })); + + const channels = await requiredCliChannels( + [pair(), pair({ eval_id: 'eval-2' })], + { + loadEvalMetadata: () => ({ cliVersion: undefined }), + loadExperimentConfig, + } + ); + + expect(channels).toEqual(new Set(['beta'])); + expect(loadExperimentConfig).toHaveBeenCalledTimes(1); + }); + + it('throws naming an experiment that cannot be resolved to a config', async () => { + await expect( + requiredCliChannels([pair({ experiment: 'ghost' })], { + loadEvalMetadata: () => ({ cliVersion: undefined }), + loadExperimentConfig: async () => { + throw new Error('no experiment config found for "ghost"'); + }, + }) + ).rejects.toThrow('no experiment config found for "ghost"'); }); }); diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index e06666fa..8a58c553 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -1,6 +1,8 @@ #!/usr/bin/env tsx import { APIError, Sandbox } from '@vercel/sandbox'; +import type { EvalMetadata, ExperimentConfig } from '@supabase-evals/core'; +import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown'; import { rawEvalResultSchema, sandboxUsageSchema, @@ -8,8 +10,8 @@ import { } from '@supabase-evals/core/eval-metadata'; import { resolveCliVersion, type CliChannel } from '@supabase-evals/sandbox'; import { execFileSync } from 'node:child_process'; -import { readFileSync, renameSync, writeFileSync } from 'node:fs'; -import { join, resolve } from 'node:path'; +import { existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; +import { join, relative, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import pLimit from 'p-limit'; import pRetry, { AbortError } from 'p-retry'; @@ -84,27 +86,94 @@ interface PairOptions extends RunnerOptions { } /** - * Resolves each CLI channel's pin once, so every sandbox job in a fan-out - * runs against the same concrete version rather than each resolving + * Resolves each requested CLI channel's pin once, so every sandbox job in a + * fan-out runs against the same concrete version rather than each resolving * independently and risking a mid-run release landing between them. An * already-set env var is used verbatim, matching the workflow/manual - * override path in resolveCliVersion. + * override path in resolveCliVersion. An empty set does no network work. */ -export async function resolveChannelPins(): Promise> { +export async function resolveChannelPins( + channels: ReadonlySet +): Promise> { const resolved = await Promise.all( - (Object.entries(CLI_CHANNEL_ENV) as [CliChannel, string][]).map( - async ([channel, envVar]) => { - const override = process.env[envVar]; - return [ - envVar, - override ?? (await resolveCliVersion(channel)), - ] as const; - } - ) + [...channels].map(async (channel) => { + const envVar = CLI_CHANNEL_ENV[channel]; + const override = process.env[envVar]; + return [envVar, override ?? (await resolveCliVersion(channel))] as const; + }) ); return Object.fromEntries(resolved); } +export interface RequiredCliChannelsDeps { + loadEvalMetadata: (pair: EvalPair) => Pick; + loadExperimentConfig: ( + experiment: string + ) => Promise<{ localStack?: { cliChannel?: CliChannel } }>; +} + +/** + * Maps a pair set to the CLI channels at least one pair needs, so + * resolveChannelPins only resolves those. An eval's own `cliVersion` + * frontmatter is an exact pin that wins over its experiment's channel and + * needs no resolution; a pair whose experiment has no `localStack.cliChannel` + * needs none either. + */ +export async function requiredCliChannels( + pairs: readonly EvalPair[], + { loadEvalMetadata, loadExperimentConfig }: RequiredCliChannelsDeps +): Promise> { + const channels = new Set(); + const configs = new Map>(); + + for (const pair of pairs) { + if (loadEvalMetadata(pair).cliVersion !== undefined) continue; + + let config = configs.get(pair.experiment); + if (!config) { + config = loadExperimentConfig(pair.experiment); + configs.set(pair.experiment, config); + } + const channel = (await config).localStack?.cliChannel; + if (channel) channels.add(channel); + } + + return channels; +} + +/** Mirrors run-eval.ts's evals///PROMPT.md convention. */ +function loadEvalMetadata(pair: EvalPair): EvalMetadata { + const promptPath = join( + ROOT, + 'evals', + pair.eval_suite, + pair.eval_id, + 'PROMPT.md' + ); + return parseEvalMarkdown(readFileSync(promptPath, 'utf8'), promptPath) + .metadata; +} + +/** + * Mirrors run-eval.ts's loadExperiments(), which resolves an experiment name + * to experiments/.ts. Throws rather than treating a config it can't + * find as needing no channel, so a future change to this layout (e.g. a + * nested experiments//*.experiment.ts convention) fails loudly instead + * of silently resolving every channel per-sandbox again. + */ +async function loadExperimentConfig( + experiment: string +): Promise { + const path = join(ROOT, 'experiments', `${experiment}.ts`); + if (!existsSync(path)) { + throw new Error( + `no experiment config found for "${experiment}" at ${relative(ROOT, path)}` + ); + } + const mod = await import(pathToFileURL(path).href); + return mod.default as ExperimentConfig; +} + interface SandboxCommandOptions { cmd: string; args?: string[]; @@ -141,9 +210,14 @@ async function runPairs(options: RunnerOptions): Promise { `max ${options.concurrency} at a time` ); - // Resolved once so every job below writes the same pin, rather than each - // sandbox resolving its own channel version independently. - const pins = await resolveChannelPins(); + // Resolved once, for only the channels this run's pairs actually need, so + // every job below writes the same pin rather than each sandbox resolving + // its own channel version independently. + const channels = await requiredCliChannels(options.pairs, { + loadEvalMetadata, + loadExperimentConfig, + }); + const pins = await resolveChannelPins(channels); const results = await runBounded( jobs, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index f2550a1c..311fca73 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -582,6 +582,8 @@ export type LocalStackSession = { export type LocalStackRuntime = { id: string; startSession(args: LocalStackSessionArgs): Promise; + /** Channel (e.g. `beta`) this runtime resolves its CLI version against, unset when it's pinned to an exact version. */ + cliChannel?: 'stable' | 'beta'; }; export type ExperimentConfig = { diff --git a/packages/sandbox/src/cli-channel.ts b/packages/sandbox/src/cli-channel.ts index 7bc45c67..26167b84 100644 --- a/packages/sandbox/src/cli-channel.ts +++ b/packages/sandbox/src/cli-channel.ts @@ -41,7 +41,7 @@ const versionCache = new Map>(); const CLI_CHANNELS = new Set(['stable', 'beta']); -function isCliChannel(value: string): value is CliChannel { +export function isCliChannel(value: string): value is CliChannel { return CLI_CHANNELS.has(value as CliChannel); } diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index 626632ac..08ba4a71 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -15,7 +15,11 @@ import { DockerSandbox } from './docker-sandbox.js'; import { createAgentEnvironment } from './agent-environment.js'; import { ensureEdgeRuntime, teardownSupabaseProject } from './supabase.js'; import { buildSkillsPrompt } from './skills.js'; -import { resolveCliVersionOption, type CliChannel } from './cli-channel.js'; +import { + isCliChannel, + resolveCliVersionOption, + type CliChannel, +} from './cli-channel.js'; const DEFAULT_BASH_TIMEOUT_SEC = 240; const MAX_BASH_TIMEOUT_SEC = 600; @@ -80,6 +84,10 @@ export function localStackRuntime( ): LocalStackRuntime { return { id: 'local-stack', + cliChannel: + options.cliVersion !== undefined && isCliChannel(options.cliVersion) + ? options.cliVersion + : undefined, async startSession({ agent, cliVersion, From 0a794b27abe02e554f5dab9d96e078fa9dce9f1a Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Tue, 22 Sep 2026 19:55:10 +0100 Subject: [PATCH 08/12] fix(sandbox): resolve experiment configs through discoverExperimentFiles The controller's channel lookup built experiments/.ts by hand, which stopped matching when experiments moved to experiments//*.experiment.ts. Use the shared discovery helper so the path convention lives in one place. --- apps/framework/scripts/run-vercel-evals.ts | 27 ++++++++++++++-------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index 8a58c553..8de4ba29 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -10,13 +10,14 @@ import { } from '@supabase-evals/core/eval-metadata'; import { resolveCliVersion, type CliChannel } from '@supabase-evals/sandbox'; import { execFileSync } from 'node:child_process'; -import { existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; +import { readFileSync, renameSync, writeFileSync } from 'node:fs'; import { join, relative, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import pLimit from 'p-limit'; import pRetry, { AbortError } from 'p-retry'; import { z } from 'zod'; import { positiveInteger, readFlag } from '../lib/cli-args.js'; +import { discoverExperimentFiles } from '../lib/experiment-files.js'; const ROOT = fileURLToPath(new URL('../../../', import.meta.url)); /** Base for sandbox URLs printed during runs */ @@ -154,20 +155,28 @@ function loadEvalMetadata(pair: EvalPair): EvalMetadata { .metadata; } +const EXPERIMENTS_DIR = join(ROOT, 'experiments'); + +let experimentPaths: Promise> | undefined; + +function experimentPathsByName(): Promise> { + experimentPaths ??= discoverExperimentFiles(EXPERIMENTS_DIR).then( + (files) => new Map(files.map((file) => [file.name, file.path])) + ); + return experimentPaths; +} + /** - * Mirrors run-eval.ts's loadExperiments(), which resolves an experiment name - * to experiments/.ts. Throws rather than treating a config it can't - * find as needing no channel, so a future change to this layout (e.g. a - * nested experiments//*.experiment.ts convention) fails loudly instead - * of silently resolving every channel per-sandbox again. + * Throws rather than treating a config it can't find as needing no channel, + * which would silently resolve every channel per-sandbox again. */ async function loadExperimentConfig( experiment: string ): Promise { - const path = join(ROOT, 'experiments', `${experiment}.ts`); - if (!existsSync(path)) { + const path = (await experimentPathsByName()).get(experiment); + if (!path) { throw new Error( - `no experiment config found for "${experiment}" at ${relative(ROOT, path)}` + `no experiment config found for "${experiment}" under ${relative(ROOT, EXPERIMENTS_DIR)}` ); } const mod = await import(pathToFileURL(path).href); From af3c0ae5a73a8d994da373853286c8fc85a91647 Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Tue, 22 Sep 2026 11:33:05 +0100 Subject: [PATCH 09/12] feat(cli): add the cli eval suite and experiment suite plumbing Splits the final chunk out of the closed #308: the cli suite/experiment enums, CODEOWNERS scoping, the four forced-environment CLI experiment columns plus their skip predicates, and CI wiring so the eval-refresh and gh-pages-history workflows know about the new suite. --- .github/CODEOWNERS | 4 + .github/workflows/append-gh-pages-history.yml | 4 +- .github/workflows/eval-refresh.yml | 52 +++++++- CONTRIBUTING.md | 17 +++ README.md | 2 +- apps/framework/package.json | 3 +- .../ai/codex-gpt-5.6-luna.experiment.ts | 3 +- ...odex-gpt-5.6-luna-cli-absent.experiment.ts | 12 ++ .../codex-gpt-5.6-luna-cli-beta.experiment.ts | 11 ++ ...ex-gpt-5.6-luna-cli-nodaemon.experiment.ts | 12 ++ ...odex-gpt-5.6-luna-cli-stable.experiment.ts | 13 ++ experiments/cli/lib/skip.test.ts | 111 ++++++++++++++++++ experiments/cli/lib/skip.ts | 21 ++++ packages/core/src/eval-metadata.ts | 2 + 14 files changed, 258 insertions(+), 9 deletions(-) create mode 100644 experiments/cli/codex-gpt-5.6-luna-cli-absent.experiment.ts create mode 100644 experiments/cli/codex-gpt-5.6-luna-cli-beta.experiment.ts create mode 100644 experiments/cli/codex-gpt-5.6-luna-cli-nodaemon.experiment.ts create mode 100644 experiments/cli/codex-gpt-5.6-luna-cli-stable.experiment.ts create mode 100644 experiments/cli/lib/skip.test.ts create mode 100644 experiments/cli/lib/skip.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 26f4dc44..9fd3311f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,3 +2,7 @@ /evals/docs/ @supabase/docs /apps/web/src/data/docs-eval-results.json @supabase/docs + +/evals/cli/ @supabase/cli +/experiments/cli/ @supabase/cli +/apps/web/src/data/cli-eval-results.json @supabase/cli diff --git a/.github/workflows/append-gh-pages-history.yml b/.github/workflows/append-gh-pages-history.yml index 8061dab3..04e96e15 100644 --- a/.github/workflows/append-gh-pages-history.yml +++ b/.github/workflows/append-gh-pages-history.yml @@ -7,6 +7,7 @@ on: - apps/web/src/data/eval-results.json - apps/web/src/data/regression-eval-results.json - apps/web/src/data/docs-eval-results.json + - apps/web/src/data/cli-eval-results.json workflow_dispatch: permissions: @@ -58,8 +59,9 @@ jobs: append_if_changed apps/web/src/data/eval-results.json results.jsonl append_if_changed apps/web/src/data/regression-eval-results.json regression-results.jsonl append_if_changed apps/web/src/data/docs-eval-results.json docs-results.jsonl + append_if_changed apps/web/src/data/cli-eval-results.json cli-results.jsonl - for history in results.jsonl regression-results.jsonl docs-results.jsonl; do + for history in results.jsonl regression-results.jsonl docs-results.jsonl cli-results.jsonl; do [ -f "$history" ] && git add "$history" done if git diff --cached --quiet; then diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 0d072e91..904e8ff0 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -41,6 +41,14 @@ on: type: boolean required: false default: false + cli_stable_version: + description: "Pin the cli suite's stable CLI version instead of resolving npm's latest dist-tag (blank to resolve)" + required: false + default: "" + cli_beta_version: + description: "Pin the cli suite's beta CLI version instead of resolving npm's beta dist-tag (blank to resolve)" + required: false + default: "" schedule: - cron: '15 6 * * *' pull_request: @@ -80,6 +88,8 @@ jobs: filter_changed: ${{ steps.inputs.outputs.filter_changed }} do_merge: ${{ steps.inputs.outputs.do_merge }} suite: ${{ steps.inputs.outputs.suite }} + cli_stable_version: ${{ steps.cli_versions.outputs.cli_stable_version }} + cli_beta_version: ${{ steps.cli_versions.outputs.cli_beta_version }} steps: - name: Prepare inputs id: inputs @@ -98,16 +108,16 @@ jobs: elif [ "${{ github.event_name }}" = "schedule" ]; then experiments_override="" eval_id="" - suite="regression" - experiment_suite="regression" + suite="regression,cli" + experiment_suite="regression,cli" runs="3" timeout_sec="720" sandbox_concurrency="250" else experiments_override="" eval_id="" - suite="benchmark,regression,docs" - experiment_suite="benchmark,no-skills,regression,docs" + suite="benchmark,regression,docs,cli" + experiment_suite="benchmark,no-skills,regression,docs,cli" runs="3" timeout_sec="720" sandbox_concurrency="250" @@ -224,6 +234,7 @@ jobs: benchmark) experiment_suites=(benchmark no-skills) ;; regression) experiment_suites=(regression) ;; docs) experiment_suites=(docs) ;; + cli) experiment_suites=(cli) ;; *) continue ;; esac @@ -258,6 +269,33 @@ jobs: echo "pairs=$pairs" >> "$GITHUB_OUTPUT" + - name: Resolve CLI channel versions + id: cli_versions + env: + # `inputs.*` is only populated for workflow_dispatch; other event + # types see it as null, which the `||` below turns into "". + CLI_STABLE_OVERRIDE: ${{ github.event_name == 'workflow_dispatch' && inputs.cli_stable_version || '' }} + CLI_BETA_OVERRIDE: ${{ github.event_name == 'workflow_dispatch' && inputs.cli_beta_version || '' }} + shell: bash + run: | + set -euo pipefail + + if ! jq -e 'index("cli") != null' <<< '${{ steps.inputs.outputs.suite }}' > /dev/null; then + { + echo "cli_stable_version=" + echo "cli_beta_version=" + } >> "$GITHUB_OUTPUT" + exit 0 + fi + + pnpm --filter @supabase-evals/framework exec node --import tsx/esm -e " + import { resolveCliVersion } from '@supabase-evals/sandbox'; + const stable = process.env.CLI_STABLE_OVERRIDE || (await resolveCliVersion('stable')); + const beta = process.env.CLI_BETA_OVERRIDE || (await resolveCliVersion('beta')); + console.log('cli_stable_version=' + stable); + console.log('cli_beta_version=' + beta); + " >> "$GITHUB_OUTPUT" + run-evals: needs: prepare if: needs.prepare.outputs.pairs != '[]' @@ -273,6 +311,8 @@ jobs: VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} + SUPABASE_CLI_STABLE_VERSION: ${{ needs.prepare.outputs.cli_stable_version }} + SUPABASE_CLI_BETA_VERSION: ${{ needs.prepare.outputs.cli_beta_version }} steps: - name: Checkout uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 @@ -301,6 +341,8 @@ jobs: echo "OPENAI_API_KEY=${OPENAI_API_KEY}" echo "AI_GATEWAY_API_KEY=${AI_GATEWAY_API_KEY}" echo "XAI_API_KEY=${XAI_API_KEY}" + echo "SUPABASE_CLI_STABLE_VERSION=${SUPABASE_CLI_STABLE_VERSION}" + echo "SUPABASE_CLI_BETA_VERSION=${SUPABASE_CLI_BETA_VERSION}" } > .env - name: Run evals @@ -466,7 +508,7 @@ jobs: pnpm --filter @supabase-evals/framework export-results -- "${export_args[@]}" fi - for eval_suite in regression docs; do + for eval_suite in regression docs cli; do if jq -e --arg s "$eval_suite" 'any(.[]; .eval_suite == $s)' <<< "$pairs" > /dev/null; then export_args=(--suite "$eval_suite" --runs "${{ needs.prepare.outputs.runs }}" --output "apps/web/src/data/${eval_suite}-eval-results.json") if [ "${{ needs.prepare.outputs.do_merge }}" = "true" ]; then diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ddf6d41b..adaf6515 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,6 +9,7 @@ First, determine the eval suite for your scenario: - **Regression** evals are suitable for most scenarios. If we notice agents make a narrow mistake, we track it here to reproduce the issue, verify a fix, and monitor for regression. These scenarios are not included in the benchmark so they don't inflate scores. - **Benchmark** evals are scenarios we've intentionally selected for the published benchmark report. These should be representative of the user journey on Supabase to cover a breadth of dimensions. - **Docs** evals are owned by docs team for their own analysis of how agents interpret docs pages, refreshed as-needed. +- **CLI** evals are owned by the CLI team: one scenario run unchanged across forced CLI environments (Docker available / daemon unreachable / absent; pinned, stable and beta CLI) via the `cli` experiment suite. Then add a folder under `evals//` containing: @@ -68,3 +69,19 @@ Common workflows: - **Add or change a docs eval.** Add the scenario under `evals/docs//` (see [Adding an eval](#adding-an-eval)), open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview. - **Refresh every docs eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: docs` and `experiment_suite: docs`. It opens a draft PR with the updated `docs-eval-results.json` for you to review and merge. - **Analyze results over time.** Every merge that changes `docs-eval-results.json` appends a snapshot to [`docs-results.jsonl`](https://supabase.github.io/evals/docs-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl) and [regression](https://supabase.github.io/evals/regression-results.jsonl) histories. + +## CLI evals + +The CLI team owns `evals/cli/` and its results. CLI evals run on the pinned-CLI baseline (`codex-gpt-5.6-luna`) plus `codex-gpt-5.6-luna-cli-{stable,beta,nodaemon,absent}` under `experiments/cli/`, which install the latest stable or beta CLI and force Docker-less sandboxes to compare the same scenario across CLI environments. + +Which evals each arm picks up: + +- **pinned, stable, beta** run every `interface: cli` eval that isn't `hostedProject: true`. +- **nodaemon, absent** additionally only run evals that also set `needsDocker: false` and `projectRunning: false` — the harness cannot pre-start a stack, or link a hosted project, without Docker. + +Common workflows: + +- **Add or change a CLI eval.** Add the scenario under `evals/cli//` (see [Adding an eval](#adding-an-eval)); set `needsDocker: false` in its `PROMPT.md` frontmatter if it can run without Docker, open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview. +- **Refresh every CLI eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: cli` and `experiment_suite: cli`. It opens a draft PR with the updated `cli-eval-results.json` for you to review and merge. Leave `cli_stable_version`/`cli_beta_version` blank to resolve npm's latest dist-tags, or pin them to reproduce a specific run. +- **Analyze results over time.** Every merge that changes `cli-eval-results.json` appends a snapshot to [`cli-results.jsonl`](https://supabase.github.io/evals/cli-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl), [regression](https://supabase.github.io/evals/regression-results.jsonl), and [docs](https://supabase.github.io/evals/docs-results.jsonl) histories. +- **Run the unit tests.** `pnpm --filter @supabase-evals/framework test:cli-lib` (the CLI skip predicates in `experiments/cli/lib/` plus every CLI eval's scorer tests) — also part of `pnpm check`. diff --git a/README.md b/README.md index c11c781a..e1696fce 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Agent-backed runs require the relevant provider key in `.env` (e.g. `OPENAI_API_ - An **eval** is one scenario under `evals///`. It contains the prompt, scorer, and optional starting state for the two environments: `remote/` (the hosted project) and `local/` (the agent's working files). - An **experiment** is one agent/runtime/model setup under `experiments//.experiment.ts`. Its ID remains ``. -- An **eval suite** is a named set of evals to run together. An eval's suite is its parent folder under `evals/` (`benchmark`, `regression`, `docs`, or `other`). +- An **eval suite** is a named set of evals to run together. An eval's suite is its parent folder under `evals/` (`benchmark`, `regression`, `docs`, `cli`, or `other`). - An **experiment suite** is a named set of experiments with related configurations, for head to head comparisons. - An **agent** is the model driver that receives the eval prompt and calls the configured tools. - A **runtime** is the local Supabase-like environment and tool surface an experiment gives to the agent. diff --git a/apps/framework/package.json b/apps/framework/package.json index 770ad8ac..54ec6a4d 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -4,7 +4,7 @@ "version": "0.0.1", "type": "module", "scripts": { - "check": "pnpm typecheck && pnpm test && pnpm test:framework && pnpm test:vercel-runner", + "check": "pnpm typecheck && pnpm test && pnpm test:framework && pnpm test:vercel-runner && pnpm test:cli-lib", "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts", "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry", "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke", @@ -13,6 +13,7 @@ "test": "vitest run harness", "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts lib/cli-args.test.ts lib/experiment-files.test.ts lib/sample-sets.test.ts", + "test:cli-lib": "vitest run --root ../.. --passWithNoTests experiments/cli evals/cli", "export-results": "node --import tsx/esm scripts/export-results.ts", "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts", "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts" diff --git a/experiments/ai/codex-gpt-5.6-luna.experiment.ts b/experiments/ai/codex-gpt-5.6-luna.experiment.ts index b44c2574..ecdd428f 100644 --- a/experiments/ai/codex-gpt-5.6-luna.experiment.ts +++ b/experiments/ai/codex-gpt-5.6-luna.experiment.ts @@ -3,5 +3,6 @@ import { codexGpt56Luna } from '../presets.js'; export default defineExperiment({ ...codexGpt56Luna, - suite: ['benchmark', 'regression'], + // cli: the pinned-CLI baseline column for CLI-team evals. + suite: ['benchmark', 'regression', 'cli'], }); diff --git a/experiments/cli/codex-gpt-5.6-luna-cli-absent.experiment.ts b/experiments/cli/codex-gpt-5.6-luna-cli-absent.experiment.ts new file mode 100644 index 00000000..1f72c04c --- /dev/null +++ b/experiments/cli/codex-gpt-5.6-luna-cli-absent.experiment.ts @@ -0,0 +1,12 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; +import { baselineExperiment } from '../presets.js'; +import { skipUnlessDockerless } from './lib/skip.js'; + +export default defineExperiment({ + ...baselineExperiment, + suite: ['cli'], + // beta: the Docker-less path only exists in the managed stack, which ships in beta. + localStack: localStackRuntime({ cliVersion: 'beta', docker: 'absent' }), + skipEval: skipUnlessDockerless, +}); diff --git a/experiments/cli/codex-gpt-5.6-luna-cli-beta.experiment.ts b/experiments/cli/codex-gpt-5.6-luna-cli-beta.experiment.ts new file mode 100644 index 00000000..d257e532 --- /dev/null +++ b/experiments/cli/codex-gpt-5.6-luna-cli-beta.experiment.ts @@ -0,0 +1,11 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; +import { baselineExperiment } from '../presets.js'; +import { skipUnlessCli } from './lib/skip.js'; + +export default defineExperiment({ + ...baselineExperiment, + suite: ['cli'], + localStack: localStackRuntime({ cliVersion: 'beta' }), + skipEval: skipUnlessCli, +}); diff --git a/experiments/cli/codex-gpt-5.6-luna-cli-nodaemon.experiment.ts b/experiments/cli/codex-gpt-5.6-luna-cli-nodaemon.experiment.ts new file mode 100644 index 00000000..1ea6e12b --- /dev/null +++ b/experiments/cli/codex-gpt-5.6-luna-cli-nodaemon.experiment.ts @@ -0,0 +1,12 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; +import { baselineExperiment } from '../presets.js'; +import { skipUnlessDockerless } from './lib/skip.js'; + +export default defineExperiment({ + ...baselineExperiment, + suite: ['cli'], + // beta: the Docker-less path only exists in the managed stack, which ships in beta. + localStack: localStackRuntime({ cliVersion: 'beta', docker: 'no-daemon' }), + skipEval: skipUnlessDockerless, +}); diff --git a/experiments/cli/codex-gpt-5.6-luna-cli-stable.experiment.ts b/experiments/cli/codex-gpt-5.6-luna-cli-stable.experiment.ts new file mode 100644 index 00000000..0c8c5755 --- /dev/null +++ b/experiments/cli/codex-gpt-5.6-luna-cli-stable.experiment.ts @@ -0,0 +1,13 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; +import { baselineExperiment } from '../presets.js'; +import { skipUnlessCli } from './lib/skip.js'; + +export default defineExperiment({ + ...baselineExperiment, + suite: ['cli'], + // Currently equal to the pin (npm `latest` == SUPABASE_CLI_VERSION); kept + // as drift insurance between pin bumps. + localStack: localStackRuntime({ cliVersion: 'stable' }), + skipEval: skipUnlessCli, +}); diff --git a/experiments/cli/lib/skip.test.ts b/experiments/cli/lib/skip.test.ts new file mode 100644 index 00000000..b9e1dc13 --- /dev/null +++ b/experiments/cli/lib/skip.test.ts @@ -0,0 +1,111 @@ +import type { EvalMetadata } from '@supabase-evals/core'; +import { describe, expect, it } from 'vitest'; +import { skipUnlessCli, skipUnlessDockerless } from './skip.js'; + +const baseMetadata: EvalMetadata = { + stage: 'build', + product: ['database'], + topic: ['sdk'], + interface: 'cli', +}; + +describe('skipUnlessCli', () => { + it('runs a cli eval that is not hosted-linked', () => { + expect(skipUnlessCli({ id: 'e', metadata: baseMetadata })).toBe(false); + }); + + it('skips a non-cli eval', () => { + expect( + skipUnlessCli({ + id: 'e', + metadata: { ...baseMetadata, interface: 'mcp' }, + }) + ).toBe(true); + }); + + it('skips a hosted-linked eval', () => { + expect( + skipUnlessCli({ + id: 'e', + metadata: { ...baseMetadata, hostedProject: true }, + }) + ).toBe(true); + }); + + it('runs a cli eval with hostedProject explicitly false', () => { + expect( + skipUnlessCli({ + id: 'e', + metadata: { ...baseMetadata, hostedProject: false }, + }) + ).toBe(false); + }); +}); + +describe('skipUnlessDockerless', () => { + const dockerlessMetadata: EvalMetadata = { + ...baseMetadata, + needsDocker: false, + projectRunning: false, + }; + + it('runs a cli eval that needs no Docker and has no pre-started stack', () => { + expect( + skipUnlessDockerless({ id: 'e', metadata: dockerlessMetadata }) + ).toBe(false); + }); + + it('skips a non-cli eval', () => { + expect( + skipUnlessDockerless({ + id: 'e', + metadata: { ...dockerlessMetadata, interface: 'mcp' }, + }) + ).toBe(true); + }); + + it('skips a hosted-linked eval', () => { + expect( + skipUnlessDockerless({ + id: 'e', + metadata: { ...dockerlessMetadata, hostedProject: true }, + }) + ).toBe(true); + }); + + it('skips an eval that needs Docker', () => { + expect( + skipUnlessDockerless({ + id: 'e', + metadata: { ...dockerlessMetadata, needsDocker: true }, + }) + ).toBe(true); + }); + + it('skips an eval whose needsDocker is unset (defaults to true)', () => { + expect( + skipUnlessDockerless({ + id: 'e', + metadata: { ...baseMetadata, projectRunning: false }, + }) + ).toBe(true); + }); + + it('skips an eval whose stack is already running', () => { + expect( + skipUnlessDockerless({ + id: 'e', + metadata: { ...dockerlessMetadata, projectRunning: true }, + }) + ).toBe(true); + }); + + it('skips an eval whose projectRunning is unset (defaults to true)', () => { + expect( + skipUnlessDockerless({ + id: 'e', + metadata: { ...baseMetadata, needsDocker: false }, + }) + ).toBe(true); + }); +}); diff --git a/experiments/cli/lib/skip.ts b/experiments/cli/lib/skip.ts new file mode 100644 index 00000000..704f1f6a --- /dev/null +++ b/experiments/cli/lib/skip.ts @@ -0,0 +1,21 @@ +import type { EvalMetadata } from '@supabase-evals/core'; + +/** Runs only evals that exercise the real CLI (never hosted-linked ones, which seed .temp pins instead). */ +export function skipUnlessCli(ev: { + id: string; + metadata: EvalMetadata; +}): boolean { + return ev.metadata.interface !== 'cli' || ev.metadata.hostedProject === true; +} + +/** A Docker-less sandbox can only run evals that declare they need no Docker and don't expect a pre-started stack. */ +export function skipUnlessDockerless(ev: { + id: string; + metadata: EvalMetadata; +}): boolean { + return ( + skipUnlessCli(ev) || + ev.metadata.needsDocker !== false || + ev.metadata.projectRunning !== false + ); +} diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index fa699c88..8e97be86 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -44,6 +44,7 @@ export const evalSuiteSchema = z.enum([ 'benchmark', 'regression', 'docs', + 'cli', 'other', ]); export const EVAL_SUITES = evalSuiteSchema.options; @@ -54,6 +55,7 @@ export const experimentSuiteSchema = z.enum([ 'no-skills', 'regression', 'docs', + 'cli', ]); export const EXPERIMENT_SUITES = experimentSuiteSchema.options; export type ExperimentSuite = z.infer; From 9cba14ae649442a409d2b283a9472f2b8dccdff6 Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Tue, 22 Sep 2026 19:57:40 +0100 Subject: [PATCH 10/12] fix(cli): point cli experiments at the codexGpt56Luna preset presets.ts landed without the baselineExperiment alias these spread, and typechecking experiments/ now needs the test files excluded. --- apps/framework/tsconfig.json | 3 ++- experiments/cli/codex-gpt-5.6-luna-cli-absent.experiment.ts | 4 ++-- experiments/cli/codex-gpt-5.6-luna-cli-beta.experiment.ts | 4 ++-- experiments/cli/codex-gpt-5.6-luna-cli-nodaemon.experiment.ts | 4 ++-- experiments/cli/codex-gpt-5.6-luna-cli-stable.experiment.ts | 4 ++-- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/apps/framework/tsconfig.json b/apps/framework/tsconfig.json index 46ea7676..6c6f48e7 100644 --- a/apps/framework/tsconfig.json +++ b/apps/framework/tsconfig.json @@ -6,5 +6,6 @@ "shims", "scripts", "../../evals/**/EVAL.ts" - ] + ], + "exclude": ["../../experiments/**/*.test.ts"] } diff --git a/experiments/cli/codex-gpt-5.6-luna-cli-absent.experiment.ts b/experiments/cli/codex-gpt-5.6-luna-cli-absent.experiment.ts index 1f72c04c..89cab4e0 100644 --- a/experiments/cli/codex-gpt-5.6-luna-cli-absent.experiment.ts +++ b/experiments/cli/codex-gpt-5.6-luna-cli-absent.experiment.ts @@ -1,10 +1,10 @@ import { defineExperiment } from '@supabase-evals/core'; import { localStackRuntime } from '@supabase-evals/sandbox'; -import { baselineExperiment } from '../presets.js'; +import { codexGpt56Luna } from '../presets.js'; import { skipUnlessDockerless } from './lib/skip.js'; export default defineExperiment({ - ...baselineExperiment, + ...codexGpt56Luna, suite: ['cli'], // beta: the Docker-less path only exists in the managed stack, which ships in beta. localStack: localStackRuntime({ cliVersion: 'beta', docker: 'absent' }), diff --git a/experiments/cli/codex-gpt-5.6-luna-cli-beta.experiment.ts b/experiments/cli/codex-gpt-5.6-luna-cli-beta.experiment.ts index d257e532..85e1b1d1 100644 --- a/experiments/cli/codex-gpt-5.6-luna-cli-beta.experiment.ts +++ b/experiments/cli/codex-gpt-5.6-luna-cli-beta.experiment.ts @@ -1,10 +1,10 @@ import { defineExperiment } from '@supabase-evals/core'; import { localStackRuntime } from '@supabase-evals/sandbox'; -import { baselineExperiment } from '../presets.js'; +import { codexGpt56Luna } from '../presets.js'; import { skipUnlessCli } from './lib/skip.js'; export default defineExperiment({ - ...baselineExperiment, + ...codexGpt56Luna, suite: ['cli'], localStack: localStackRuntime({ cliVersion: 'beta' }), skipEval: skipUnlessCli, diff --git a/experiments/cli/codex-gpt-5.6-luna-cli-nodaemon.experiment.ts b/experiments/cli/codex-gpt-5.6-luna-cli-nodaemon.experiment.ts index 1ea6e12b..7921a9c1 100644 --- a/experiments/cli/codex-gpt-5.6-luna-cli-nodaemon.experiment.ts +++ b/experiments/cli/codex-gpt-5.6-luna-cli-nodaemon.experiment.ts @@ -1,10 +1,10 @@ import { defineExperiment } from '@supabase-evals/core'; import { localStackRuntime } from '@supabase-evals/sandbox'; -import { baselineExperiment } from '../presets.js'; +import { codexGpt56Luna } from '../presets.js'; import { skipUnlessDockerless } from './lib/skip.js'; export default defineExperiment({ - ...baselineExperiment, + ...codexGpt56Luna, suite: ['cli'], // beta: the Docker-less path only exists in the managed stack, which ships in beta. localStack: localStackRuntime({ cliVersion: 'beta', docker: 'no-daemon' }), diff --git a/experiments/cli/codex-gpt-5.6-luna-cli-stable.experiment.ts b/experiments/cli/codex-gpt-5.6-luna-cli-stable.experiment.ts index 0c8c5755..140fe328 100644 --- a/experiments/cli/codex-gpt-5.6-luna-cli-stable.experiment.ts +++ b/experiments/cli/codex-gpt-5.6-luna-cli-stable.experiment.ts @@ -1,10 +1,10 @@ import { defineExperiment } from '@supabase-evals/core'; import { localStackRuntime } from '@supabase-evals/sandbox'; -import { baselineExperiment } from '../presets.js'; +import { codexGpt56Luna } from '../presets.js'; import { skipUnlessCli } from './lib/skip.js'; export default defineExperiment({ - ...baselineExperiment, + ...codexGpt56Luna, suite: ['cli'], // Currently equal to the pin (npm `latest` == SUPABASE_CLI_VERSION); kept // as drift insurance between pin bumps. From 9ace7b165b675747e02caffc707b7ca6c227e99f Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Tue, 22 Sep 2026 20:26:03 +0100 Subject: [PATCH 11/12] fix(sandbox): root-own writeRootFile writes, correct Docker-less prompt text, and read the environment marker without trusting the agent's PATH writeRootFile only chmod'd after docker cp, leaving the marker and CLI shims owned by the host UID rather than root. buildToolSurfaceAddendum and buildLocalStackTools also described docker as installed/reachable in the no-daemon and absent states. And the environment marker was only reachable through exec(), whose SANDBOX_PATH has an agent-writable first entry that could shadow cat. Fix writeRootFile to chown+chmod in one checked root command, thread the resolved Docker state into both prompt/tool builders, and add DockerSandbox.readRootFile (docker exec in exec form, no shell) plus LocalStackScoringContext.environmentMarker to read the marker as root without going through resolveSandboxPath or the agent's PATH. --- packages/core/src/index.ts | 21 ++++ packages/sandbox/src/docker-sandbox.ts | 34 +++++- packages/sandbox/src/local-stack-runtime.ts | 104 +++++++++++++---- packages/sandbox/test/docker.test.ts | 37 +++++- .../sandbox/test/local-stack-docker.test.ts | 107 +++++++++++++++++- packages/sandbox/test/unit.test.ts | 70 ++++++++++++ 6 files changed, 344 insertions(+), 29 deletions(-) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 311fca73..9b808b91 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -303,6 +303,20 @@ export interface LocalStackStatus { anonKey: string; } +/** + * Parsed contents of the environment marker a local-stack session writes, + * recording its staged CLI version and Docker availability — for scorers to + * report, never to gate pass/fail. + */ +export interface LocalStackEnvironmentMarker { + runtime: 'local-stack'; + /** The channel (`stable`/`beta`) the session's CLI version resolved from, when the runtime named one. */ + channel?: 'stable' | 'beta'; + cliVersion: string; + docker: 'available' | 'no-daemon' | 'absent'; + sessionStartedMs: number; +} + /** * Scoring surface for local-stack evals. Everything runs inside the Docker * sandbox the agent worked in, against the local Supabase stack it (or the @@ -342,6 +356,13 @@ export interface LocalStackScoringContext { * doesn't cover, or to assert on where a key ended up. */ stackStatus: () => Promise; + /** + * The session's environment marker, read directly from the sandbox as + * root so nothing on the agent's `PATH` can forge it, and never through + * `resolveSandboxPath`/`exec`. Undefined when the session recorded none. + * For scorers to report, never to gate pass/fail. + */ + environmentMarker: () => Promise; /** * The mocked hosted project's ref, when the eval links to platform-lite * (`hostedProject: true`). Undefined for purely-local evals. diff --git a/packages/sandbox/src/docker-sandbox.ts b/packages/sandbox/src/docker-sandbox.ts index 1825a463..77a222f5 100644 --- a/packages/sandbox/src/docker-sandbox.ts +++ b/packages/sandbox/src/docker-sandbox.ts @@ -341,7 +341,8 @@ export class DockerSandbox { * with an explicit mode — e.g. installing a small CLI shim onto PATH. Content * is staged on the host and `docker cp`'d in, so it arrives as data with no * shell quoting, and the copy runs as the engine, so it can write system dirs - * regardless of the container user. The chmod then sets the mode as root. + * regardless of the container user. `docker cp` preserves host ownership, so + * one checked root command then both chowns the file to root and sets its mode. */ async writeRootFile( containerPath: string, @@ -357,14 +358,37 @@ export class DockerSandbox { } finally { rmSync(staging, { recursive: true, force: true }); } - const chmod = await this.runShellAsRoot( - `chmod ${mode} ${shellQuote(containerPath)}` + const secure = await this.runShellAsRoot( + `chown root:root ${shellQuote(containerPath)} && chmod ${mode} ${shellQuote(containerPath)}` ); - if (!chmod.ok) { - throw new Error(`failed to chmod ${containerPath}: ${chmod.stderr}`); + if (!secure.ok) { + throw new Error(`failed to secure ${containerPath}: ${secure.stderr}`); } } + /** + * Read an absolute container path as root, in exec form with an absolute + * binary rather than a shell, so nothing on the agent's `PATH` (including + * `SANDBOX_PATH`'s writable first entry) can shadow the read. + */ + async readRootFile(containerPath: string): Promise { + this.assertRunning(); + const result = await dockerCli([ + 'exec', + '--user', + 'root', + this.containerId!, + '/bin/cat', + containerPath, + ]); + if (!result.ok) { + throw new Error( + `failed to read ${containerPath} as root: ${result.stderr}` + ); + } + return result.stdout; + } + private async execCommand( command: string, options: { env?: Record; user: string; timeoutMs?: number } diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index ecfb4c67..b3564ace 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -6,11 +6,13 @@ import { type AgentHarnessId, type AgentSandbox, type HostedLink, + type LocalStackEnvironmentMarker, type LocalStackRuntime, type LocalStackScoringContext, type LocalStackStatus, type McpServerConfig, } from '@supabase-evals/core'; +import { isRecord } from '@supabase-evals/core/json'; import { DockerSandbox } from './docker-sandbox.js'; import { createAgentEnvironment } from './agent-environment.js'; import { @@ -97,18 +99,10 @@ export type DockerState = 'available' | 'no-daemon' | 'absent'; */ export const LOCAL_STACK_MARKER_PATH = '/tmp/supabase-eval-runtime.json'; -export type LocalStackEnvironmentMarker = { - runtime: 'local-stack'; - /** - * The channel `cliVersion` resolved from, when the runtime option named - * one and no per-eval `cliVersion:` pin overrode it. Undefined for any - * exact-version pin. - */ - channel?: CliChannel; - cliVersion: string; - docker: DockerState; - sessionStartedMs: number; -}; +// The shared scoring-context shape (LocalStackScoringContext.environmentMarker) +// lives in core; re-exported here so existing imports of this type from +// @supabase-evals/sandbox keep working. +export type { LocalStackEnvironmentMarker } from '@supabase-evals/core'; // Shadow the real `docker`/`supabase` binaries from the first entry on the // sandbox PATH (see SANDBOX_PATH in docker-sandbox.ts). @@ -191,11 +185,11 @@ export function localStackRuntime( const mcpServers = await resolveMcpServers(options, hosted); const session = { - tools: buildLocalStackTools(sandbox), + tools: buildLocalStackTools(sandbox, docker), sandbox: toAgentSandbox(sandbox), mcpServers, promptAddendum: [ - buildToolSurfaceAddendum(agent, { skipCliInstall }), + buildToolSurfaceAddendum(agent, { skipCliInstall, docker }), buildSkillsPrompt(agent, env.skills), ] .filter(Boolean) @@ -338,13 +332,13 @@ export function localStackRuntime( ); return { - tools: buildLocalStackTools(sandbox), + tools: buildLocalStackTools(sandbox, docker), sandbox: toAgentSandbox(sandbox), // Same wiring as the `available` path; `hosted` is always // undefined here (guarded above), so this falls back to `docs`. mcpServers: dockerlessMcpServers, promptAddendum: [ - buildToolSurfaceAddendum(agent, { skipCliInstall }), + buildToolSurfaceAddendum(agent, { skipCliInstall, docker }), buildSkillsPrompt(agent, installedSkills), ] .filter(Boolean) @@ -479,6 +473,25 @@ export function buildDockerDaemonShimScript(dockerVersion: string): string { ].join('\n'); } +/** + * Names the workspace's binaries accurately for the resolved Docker state, + * so the Docker-less paths don't claim a binary or daemon the agent doesn't + * have. + */ +function describeDockerTools(docker: DockerState): string { + switch (docker) { + case 'available': + return 'docker, psql, git, and curl are installed in the workspace'; + case 'no-daemon': + return ( + 'psql, git, and curl are installed in the workspace; docker is ' + + 'installed but there is no daemon for it to connect to' + ); + case 'absent': + return 'psql, git, and curl are installed in the workspace; docker is not installed'; + } +} + /** * Describes the session's tool surface: the binaries installed in the workspace * and the in-process `bash`/`files_*` tools from `buildLocalStackTools`. @@ -489,18 +502,24 @@ export function buildDockerDaemonShimScript(dockerVersion: string): string { */ export function buildToolSurfaceAddendum( agent: AgentHarnessId, - options: { skipCliInstall?: boolean } = {} + options: { skipCliInstall?: boolean; docker?: DockerState } = {} ): string { if (agent !== 'ai-sdk') return ''; + const docker = options.docker ?? 'available'; let addendum = - 'docker, psql, git, and curl are installed in the workspace. ' + + `${describeDockerTools(docker)}. ` + 'Use the bash tool to run commands (the working directory is always the workspace root) ' + 'and the files tools to inspect and modify files.'; if (!options.skipCliInstall) { addendum = 'The Supabase CLI (`supabase`), ' + addendum; - addendum += - ' Services started with `supabase start` are reachable on their default 127.0.0.1 ports.'; + // Without a reachable daemon, `supabase start` cannot bring services up + // at all, so the reachability claim would be false on both Docker-less + // states. + if (docker === 'available') { + addendum += + ' Services started with `supabase start` are reachable on their default 127.0.0.1 ports.'; + } } return addendum; } @@ -550,13 +569,16 @@ export function toAgentSandbox(sandbox: DockerSandbox): AgentSandbox { }; } -export function buildLocalStackTools(sandbox: DockerSandbox): ToolSet { +export function buildLocalStackTools( + sandbox: DockerSandbox, + docker: DockerState = 'available' +): ToolSet { return { bash: tool({ description: 'Run a bash command in the eval workspace (Linux). The working directory ' + 'is always the workspace root; `cd` does not persist between calls. The ' + - 'Supabase CLI (`supabase`), docker, psql, git, and curl are installed.', + `Supabase CLI (\`supabase\`), ${describeDockerTools(docker)}.`, inputSchema: jsonSchema({ type: 'object', properties: { @@ -683,6 +705,26 @@ export function buildLocalStackScoringContext( ): LocalStackScoringContext { let stackConfig: LocalStackStatus | undefined; let dbUrl: string | undefined; + let environmentMarker: LocalStackEnvironmentMarker | undefined; + let environmentMarkerRead = false; + + // Read as root, in exec form (docker-sandbox.ts's readRootFile), so + // neither `resolveSandboxPath` nor the agent's `PATH` is in the path of a + // scorer reading the environment it's being graded in. + const discoverEnvironmentMarker = async () => { + if (environmentMarkerRead) return environmentMarker; + environmentMarkerRead = true; + try { + const raw = await sandbox.readRootFile(LOCAL_STACK_MARKER_PATH); + const parsed = JSON.parse(raw); + environmentMarker = isLocalStackEnvironmentMarker(parsed) + ? parsed + : undefined; + } catch { + environmentMarker = undefined; + } + return environmentMarker; + }; // Read the DB connection string from the running stack rather than assuming // the default 127.0.0.1:54322 — same derive-from-`supabase status` approach as @@ -773,6 +815,7 @@ export function buildLocalStackScoringContext( return { rows: text ? JSON.parse(text) : [] }; }, stackStatus: () => discoverStackConfig(), + environmentMarker: () => discoverEnvironmentMarker(), getClient: async () => { const { apiUrl, publishableKey } = await discoverStackConfig(); return createClient(apiUrl, publishableKey, { @@ -786,6 +829,23 @@ export function buildLocalStackScoringContext( }; } +function isLocalStackEnvironmentMarker( + value: unknown +): value is LocalStackEnvironmentMarker { + return ( + isRecord(value) && + value.runtime === 'local-stack' && + typeof value.cliVersion === 'string' && + (value.docker === 'available' || + value.docker === 'no-daemon' || + value.docker === 'absent') && + typeof value.sessionStartedMs === 'number' && + (value.channel === undefined || + value.channel === 'stable' || + value.channel === 'beta') + ); +} + function readString( config: Record | undefined, key: string diff --git a/packages/sandbox/test/docker.test.ts b/packages/sandbox/test/docker.test.ts index bba78d49..40d78885 100644 --- a/packages/sandbox/test/docker.test.ts +++ b/packages/sandbox/test/docker.test.ts @@ -9,7 +9,10 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterAll, describe, expect, it } from 'vitest'; -import { buildLocalStackScoringContext } from '../src/local-stack-runtime.js'; +import { + buildLocalStackScoringContext, + localStackRuntime, +} from '../src/local-stack-runtime.js'; import { DockerSandbox } from '../src/docker-sandbox.js'; import { ensureSupabaseSandboxImage, @@ -237,5 +240,37 @@ describe.runIf(process.env.SANDBOX_DOCKER_TESTS)( } } ); + + it( + 'a no-daemon sandbox root-owns the marker and both shims, read-only to the agent', + { timeout: TEST_TIMEOUT_MS }, + async () => { + // Docker-less staging guards both projectRunning: false and no hosted + // link, since the harness cannot pre-start a stack or link a hosted + // project without Docker. + const session = await localStackRuntime({ + docker: 'no-daemon', + }).startSession({ agent: 'ai-sdk', projectRunning: false }); + try { + const stat = await session.scoringContext.exec( + 'stat -c "%u:%g %a" /tmp/supabase-eval-runtime.json /usr/local/sbin/supabase /usr/local/sbin/docker' + ); + expect(stat.ok, stat.stderr).toBe(true); + expect(stat.stdout.trim().split('\n')).toEqual([ + '0:0 444', + '0:0 755', + '0:0 755', + ]); + + const write = await session.scoringContext.exec( + 'echo forged > /tmp/supabase-eval-runtime.json' + ); + expect(write.ok).toBe(false); + expect(write.stderr).toMatch(/permission denied/i); + } finally { + await session.close(); + } + } + ); } ); diff --git a/packages/sandbox/test/local-stack-docker.test.ts b/packages/sandbox/test/local-stack-docker.test.ts index 720d4d87..727f34b0 100644 --- a/packages/sandbox/test/local-stack-docker.test.ts +++ b/packages/sandbox/test/local-stack-docker.test.ts @@ -1,10 +1,12 @@ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { buildDockerDaemonShimScript, + buildLocalStackScoringContext, buildSupabaseShimScript, localStackRuntime, resolveMcpServers, } from '../src/local-stack-runtime.js'; +import type { DockerSandbox } from '../src/docker-sandbox.js'; import type { SupabaseService } from '../src/types.js'; describe('localStackRuntime id', () => { @@ -139,3 +141,106 @@ describe('resolveMcpServers', () => { expect(Object.keys(servers)).toEqual(['supabase']); }); }); + +describe('buildLocalStackScoringContext environmentMarker', () => { + /** A DockerSandbox stub whose readRootFile returns/throws as configured. */ + function fakeSandbox(readRootFile: DockerSandbox['readRootFile']) { + return { + workdir: '/tmp/sandbox-x', + readRootFile, + } as unknown as DockerSandbox; + } + + for (const docker of ['available', 'no-daemon', 'absent'] as const) { + it(`parses a well-formed marker for docker: ${docker}`, async () => { + const marker = { + runtime: 'local-stack', + cliVersion: '2.109.1', + docker, + sessionStartedMs: 1_700_000_000_000, + }; + const readRootFile = vi.fn().mockResolvedValue(JSON.stringify(marker)); + const ctx = buildLocalStackScoringContext(fakeSandbox(readRootFile)); + await expect(ctx.environmentMarker()).resolves.toEqual(marker); + }); + } + + it('includes an optional channel when present', async () => { + const marker = { + runtime: 'local-stack', + channel: 'beta', + cliVersion: '2.109.1', + docker: 'available', + sessionStartedMs: 1_700_000_000_000, + }; + const readRootFile = vi.fn().mockResolvedValue(JSON.stringify(marker)); + const ctx = buildLocalStackScoringContext(fakeSandbox(readRootFile)); + await expect(ctx.environmentMarker()).resolves.toEqual(marker); + }); + + it('reads the marker as root, not through exec or resolveSandboxPath', async () => { + const readRootFile = vi.fn().mockResolvedValue( + JSON.stringify({ + runtime: 'local-stack', + cliVersion: '2.109.1', + docker: 'available', + sessionStartedMs: 0, + }) + ); + const ctx = buildLocalStackScoringContext(fakeSandbox(readRootFile)); + await ctx.environmentMarker(); + expect(readRootFile).toHaveBeenCalledWith( + '/tmp/supabase-eval-runtime.json' + ); + }); + + it('is undefined when no marker was written', async () => { + const readRootFile = vi + .fn() + .mockRejectedValue(new Error('cat: No such file or directory')); + const ctx = buildLocalStackScoringContext(fakeSandbox(readRootFile)); + await expect(ctx.environmentMarker()).resolves.toBeUndefined(); + }); + + it('is undefined for malformed JSON rather than throwing', async () => { + const readRootFile = vi.fn().mockResolvedValue('not json'); + const ctx = buildLocalStackScoringContext(fakeSandbox(readRootFile)); + await expect(ctx.environmentMarker()).resolves.toBeUndefined(); + }); + + it('is undefined for a well-formed but wrong-shaped payload', async () => { + const readRootFile = vi + .fn() + .mockResolvedValue(JSON.stringify({ runtime: 'not-local-stack' })); + const ctx = buildLocalStackScoringContext(fakeSandbox(readRootFile)); + await expect(ctx.environmentMarker()).resolves.toBeUndefined(); + }); + + it('rejects an invalid docker value instead of casting it through', async () => { + const readRootFile = vi.fn().mockResolvedValue( + JSON.stringify({ + runtime: 'local-stack', + cliVersion: '2.109.1', + docker: 'forged', + sessionStartedMs: 0, + }) + ); + const ctx = buildLocalStackScoringContext(fakeSandbox(readRootFile)); + await expect(ctx.environmentMarker()).resolves.toBeUndefined(); + }); + + it('caches the marker instead of re-reading on every call', async () => { + const readRootFile = vi.fn().mockResolvedValue( + JSON.stringify({ + runtime: 'local-stack', + cliVersion: '2.109.1', + docker: 'available', + sessionStartedMs: 0, + }) + ); + const ctx = buildLocalStackScoringContext(fakeSandbox(readRootFile)); + await ctx.environmentMarker(); + await ctx.environmentMarker(); + expect(readRootFile).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/sandbox/test/unit.test.ts b/packages/sandbox/test/unit.test.ts index 6b3e8c2a..643579ef 100644 --- a/packages/sandbox/test/unit.test.ts +++ b/packages/sandbox/test/unit.test.ts @@ -2,6 +2,7 @@ import { readFileSync } from 'node:fs'; import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { + buildLocalStackTools, buildToolSurfaceAddendum, resolveSandboxPath, truncateOutput, @@ -244,6 +245,75 @@ describe('buildToolSurfaceAddendum', () => { ); } }); + + it('defaults to the docker-available text when docker is unspecified', () => { + expect(buildToolSurfaceAddendum('ai-sdk')).toBe( + buildToolSurfaceAddendum('ai-sdk', { docker: 'available' }) + ); + }); + + it('claims docker is installed and reachable only when docker is available', () => { + const addendum = buildToolSurfaceAddendum('ai-sdk', { + docker: 'available', + }); + expect(addendum).toContain('docker, psql, git, and curl are installed'); + expect(addendum).toContain( + 'Services started with `supabase start` are reachable on their default 127.0.0.1 ports.' + ); + }); + + it('describes docker as installed but daemon-less, with no reachability claim, for no-daemon', () => { + const addendum = buildToolSurfaceAddendum('ai-sdk', { + docker: 'no-daemon', + }); + expect(addendum).toContain('psql, git, and curl are installed'); + expect(addendum).toContain( + 'docker is installed but there is no daemon for it to connect to' + ); + expect(addendum).not.toContain('docker, psql, git, and curl'); + expect(addendum).not.toContain( + 'reachable on their default 127.0.0.1 ports' + ); + }); + + it('describes docker as not installed, with no reachability claim, for absent', () => { + const addendum = buildToolSurfaceAddendum('ai-sdk', { docker: 'absent' }); + expect(addendum).toContain('psql, git, and curl are installed'); + expect(addendum).toContain('docker is not installed'); + expect(addendum).not.toContain('docker, psql, git, and curl'); + expect(addendum).not.toContain( + 'reachable on their default 127.0.0.1 ports' + ); + }); +}); + +describe('buildLocalStackTools bash description', () => { + function bashDescription(docker: 'available' | 'no-daemon' | 'absent') { + const tools = buildLocalStackTools({} as DockerSandbox, docker); + return (tools.bash as { description?: string }).description ?? ''; + } + + it('names docker as installed for the default (available) state', () => { + expect(bashDescription('available')).toContain( + 'docker, psql, git, and curl are installed' + ); + }); + + it('names docker as installed but daemon-less for no-daemon', () => { + const description = bashDescription('no-daemon'); + expect(description).toContain('psql, git, and curl are installed'); + expect(description).toContain( + 'docker is installed but there is no daemon for it to connect to' + ); + expect(description).not.toContain('docker, psql, git, and curl'); + }); + + it('names docker as not installed for absent', () => { + const description = bashDescription('absent'); + expect(description).toContain('psql, git, and curl are installed'); + expect(description).toContain('docker is not installed'); + expect(description).not.toContain('docker, psql, git, and curl'); + }); }); describe('SKILLS_CLI_VERSION', () => { From fab2e19780132eeb6b9fbb0ef6f0ff31e700ba1e Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Wed, 23 Sep 2026 12:36:08 +0100 Subject: [PATCH 12/12] =?UTF-8?q?fix(cli):=20address=20PR=20#327=20review?= =?UTF-8?q?=20=E2=80=94=20record=20the=20run's=20CLI=20version,=20defer=20?= =?UTF-8?q?channel=20resolution=20to=20the=20controller,=20and=20give=20th?= =?UTF-8?q?e=20CLI=20suite=20its=20own=20pinned=20column?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records environmentMarker().cliVersion (the CLI binary that actually ran) into raw and exported results instead of the near-always-empty frontmatter pin, widening cliVersionSchema to accept prerelease versions so beta runs validate. Strips eager npm resolution from eval-refresh.yml's prepare job so run-vercel-evals.ts's requiredCliChannels narrowing runs as intended, folding the now-pass-through step into "Prepare inputs" and treating a blank env var as unset in resolveChannelPins. Adds a CLI-owned codex-gpt-6-luna-cli-pinned experiment (skipUnlessCli, no localStack override) so pinned/stable/beta share eligibility rules, and reverts the shared benchmark/regression experiment to its original suite. --- .github/workflows/eval-refresh.yml | 39 +++-------- CONTRIBUTING.md | 2 +- apps/framework/harness/run-eval.ts | 8 +++ apps/framework/package.json | 2 +- apps/framework/scripts/export-results.test.ts | 64 +++++++++++++++++++ apps/framework/scripts/export-results.ts | 8 ++- .../scripts/run-vercel-evals.test.ts | 10 +++ apps/framework/scripts/run-vercel-evals.ts | 6 +- experiments/ai/codex-gpt-6-luna.experiment.ts | 3 +- .../codex-gpt-6-luna-cli-pinned.experiment.ts | 9 +++ packages/core/src/eval-metadata.test.ts | 41 ++++++++++++ packages/core/src/eval-metadata.ts | 4 +- 12 files changed, 157 insertions(+), 39 deletions(-) create mode 100644 apps/framework/scripts/export-results.test.ts create mode 100644 experiments/cli/codex-gpt-6-luna-cli-pinned.experiment.ts create mode 100644 packages/core/src/eval-metadata.test.ts diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 904e8ff0..e7520f4c 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -88,8 +88,8 @@ jobs: filter_changed: ${{ steps.inputs.outputs.filter_changed }} do_merge: ${{ steps.inputs.outputs.do_merge }} suite: ${{ steps.inputs.outputs.suite }} - cli_stable_version: ${{ steps.cli_versions.outputs.cli_stable_version }} - cli_beta_version: ${{ steps.cli_versions.outputs.cli_beta_version }} + cli_stable_version: ${{ steps.inputs.outputs.cli_stable_version }} + cli_beta_version: ${{ steps.inputs.outputs.cli_beta_version }} steps: - name: Prepare inputs id: inputs @@ -105,6 +105,8 @@ jobs: runs="${{ inputs.runs }}" timeout_sec="${{ inputs.timeout_sec }}" sandbox_concurrency="${{ inputs.sandbox_concurrency }}" + cli_stable_version="${{ inputs.cli_stable_version }}" + cli_beta_version="${{ inputs.cli_beta_version }}" elif [ "${{ github.event_name }}" = "schedule" ]; then experiments_override="" eval_id="" @@ -113,6 +115,8 @@ jobs: runs="3" timeout_sec="720" sandbox_concurrency="250" + cli_stable_version="" + cli_beta_version="" else experiments_override="" eval_id="" @@ -121,6 +125,8 @@ jobs: runs="3" timeout_sec="720" sandbox_concurrency="250" + cli_stable_version="" + cli_beta_version="" fi suite_json="$(jq -Rc 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))' <<< "$suite")" @@ -152,6 +158,8 @@ jobs: echo "sandbox_concurrency=$sandbox_concurrency" echo "filter_changed=$filter_changed" echo "do_merge=$do_merge" + echo "cli_stable_version=$cli_stable_version" + echo "cli_beta_version=$cli_beta_version" } >> "$GITHUB_OUTPUT" - name: Checkout @@ -269,33 +277,6 @@ jobs: echo "pairs=$pairs" >> "$GITHUB_OUTPUT" - - name: Resolve CLI channel versions - id: cli_versions - env: - # `inputs.*` is only populated for workflow_dispatch; other event - # types see it as null, which the `||` below turns into "". - CLI_STABLE_OVERRIDE: ${{ github.event_name == 'workflow_dispatch' && inputs.cli_stable_version || '' }} - CLI_BETA_OVERRIDE: ${{ github.event_name == 'workflow_dispatch' && inputs.cli_beta_version || '' }} - shell: bash - run: | - set -euo pipefail - - if ! jq -e 'index("cli") != null' <<< '${{ steps.inputs.outputs.suite }}' > /dev/null; then - { - echo "cli_stable_version=" - echo "cli_beta_version=" - } >> "$GITHUB_OUTPUT" - exit 0 - fi - - pnpm --filter @supabase-evals/framework exec node --import tsx/esm -e " - import { resolveCliVersion } from '@supabase-evals/sandbox'; - const stable = process.env.CLI_STABLE_OVERRIDE || (await resolveCliVersion('stable')); - const beta = process.env.CLI_BETA_OVERRIDE || (await resolveCliVersion('beta')); - console.log('cli_stable_version=' + stable); - console.log('cli_beta_version=' + beta); - " >> "$GITHUB_OUTPUT" - run-evals: needs: prepare if: needs.prepare.outputs.pairs != '[]' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2d893fd1..3d8739fe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -72,7 +72,7 @@ Common workflows: ## CLI evals -The CLI team owns `evals/cli/` and its results. CLI evals run on the pinned-CLI baseline (`codex-gpt-6-luna`) plus `codex-gpt-6-luna-cli-{stable,beta,nodaemon,absent}` under `experiments/cli/`, which install the latest stable or beta CLI and force Docker-less sandboxes to compare the same scenario across CLI environments. +The CLI team owns `evals/cli/` and its results. CLI evals run on `codex-gpt-6-luna-cli-{pinned,stable,beta,nodaemon,absent}` under `experiments/cli/`: `pinned` runs the repo's pinned CLI version, `stable`/`beta` install the latest stable or beta CLI, and `nodaemon`/`absent` additionally force Docker-less sandboxes — comparing the same scenario across CLI environments. Which evals each arm picks up: diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index 14cb6b9e..43e74318 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -369,6 +369,7 @@ async function runOne( stepCount?: number; toolCallCount: number; agentRunDurationMs: number; + cliVersion?: string; } > { const prompt = parseEvalMarkdown( @@ -479,6 +480,10 @@ async function runOne( // Runs after scoring so the scorer sees what the agent actually saw, not rehydrated content. await rehydrateTruncatedDocsResults(session.sandbox, run.toolCalls); + // The marker names the CLI binary the sandbox actually staged; falls back to + // the frontmatter pin only when there's no marker to read (e.g. skipCliInstall). + const marker = await session.scoringContext.environmentMarker(); + return { ...last, run: runIndex, @@ -493,6 +498,7 @@ async function runOne( stepCount: run.stepCount, toolCallCount: run.toolCalls.length, agentRunDurationMs: run.durationMs, + cliVersion: marker?.cliVersion ?? ev.metadata.cliVersion, }; } @@ -556,6 +562,8 @@ async function runOne( stepCount: run.stepCount, toolCallCount: run.toolCalls.length, agentRunDurationMs: run.durationMs, + // No sandbox in tools mode, so no marker to read; only the frontmatter pin applies. + cliVersion: ev.metadata.cliVersion, }; } diff --git a/apps/framework/package.json b/apps/framework/package.json index 54ec6a4d..57307260 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -12,7 +12,7 @@ "typecheck": "tsc --noEmit", "test": "vitest run harness", "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", - "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts lib/cli-args.test.ts lib/experiment-files.test.ts lib/sample-sets.test.ts", + "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts scripts/export-results.test.ts lib/cli-args.test.ts lib/experiment-files.test.ts lib/sample-sets.test.ts", "test:cli-lib": "vitest run --root ../.. --passWithNoTests experiments/cli evals/cli", "export-results": "node --import tsx/esm scripts/export-results.ts", "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts", diff --git a/apps/framework/scripts/export-results.test.ts b/apps/framework/scripts/export-results.test.ts new file mode 100644 index 00000000..5b285f6f --- /dev/null +++ b/apps/framework/scripts/export-results.test.ts @@ -0,0 +1,64 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import type { ExperimentExportMetadata } from './export-results.js'; +import { readResultFile } from './export-results.js'; + +// build-docs-010-edge-function-auth is the one eval whose PROMPT.md pins +// `cliVersion: 2.109.1`, needed here to exercise the frontmatter side of the +// precedence. +const PINNED_EVAL_ID = 'build-docs-010-edge-function-auth'; + +describe('readResultFile cliVersion precedence', () => { + const experimentMetadata = new Map(); + + const writeRawResult = (dir: string, raw: Record) => { + const filePath = join(dir, 'result.json'); + writeFileSync(filePath, JSON.stringify(raw)); + return filePath; + }; + + it('carries the run value that actually ran over the frontmatter pin', async () => { + const temporary = mkdtempSync(join(tmpdir(), 'export-results-test-')); + try { + const filePath = writeRawResult(temporary, { + experiment: 'test-experiment', + eval: PINNED_EVAL_ID, + interface: 'cli', + cliVersion: '2.118.0-beta.60', + }); + + const result = await readResultFile( + filePath, + 'test-experiment/run-1/result.json', + experimentMetadata + ); + + expect(result?.cliVersion).toBe('2.118.0-beta.60'); + } finally { + rmSync(temporary, { recursive: true, force: true }); + } + }); + + it('falls back to the frontmatter pin when the run recorded no version', async () => { + const temporary = mkdtempSync(join(tmpdir(), 'export-results-test-')); + try { + const filePath = writeRawResult(temporary, { + experiment: 'test-experiment', + eval: PINNED_EVAL_ID, + interface: 'cli', + }); + + const result = await readResultFile( + filePath, + 'test-experiment/run-1/result.json', + experimentMetadata + ); + + expect(result?.cliVersion).toBe('2.109.1'); + } finally { + rmSync(temporary, { recursive: true, force: true }); + } + }); +}); diff --git a/apps/framework/scripts/export-results.ts b/apps/framework/scripts/export-results.ts index 50e8912f..3a69e4c1 100644 --- a/apps/framework/scripts/export-results.ts +++ b/apps/framework/scripts/export-results.ts @@ -47,7 +47,7 @@ const OUTPUT_PATH = join( 'eval-results.json' ); -type ExperimentExportMetadata = { +export type ExperimentExportMetadata = { display: ExperimentDisplayMetadata; experimentSuite?: ExperimentSuite; }; @@ -122,7 +122,7 @@ async function readPrompt(evalId: string) { }; } -async function readResultFile( +export async function readResultFile( filePath: string, sourcePath: string, experimentMetadata: Map @@ -152,7 +152,9 @@ async function readResultFile( topic: promptData?.topic ?? parsedResult.topic, suite: promptData?.suite ?? parsedResult.suite, interface: promptData?.interface ?? parsedResult.interface, - cliVersion: promptData?.cliVersion ?? parsedResult.cliVersion, + // The run's recorded version (the binary that actually ran) wins over the + // frontmatter pin, which only names what the eval requested. + cliVersion: parsedResult.cliVersion ?? promptData?.cliVersion, passed: parsedResult.passed === true, checks: parsedResult.checks, skills: parsedResult.skills, diff --git a/apps/framework/scripts/run-vercel-evals.test.ts b/apps/framework/scripts/run-vercel-evals.test.ts index ab4cec9a..d4c9f41d 100644 --- a/apps/framework/scripts/run-vercel-evals.test.ts +++ b/apps/framework/scripts/run-vercel-evals.test.ts @@ -178,6 +178,16 @@ describe('resolveChannelPins', () => { resolveChannelPins(new Set(['stable'])) ).rejects.toThrow('npm unreachable'); }); + + it('treats a blank or whitespace-only env var as unset, resolving it instead', async () => { + process.env[STABLE_ENV] = ' '; + vi.mocked(resolveCliVersion).mockImplementation(async () => '2.117.0'); + + const pins = await resolveChannelPins(new Set(['stable'])); + + expect(pins[STABLE_ENV]).toBe('2.117.0'); + expect(resolveCliVersion).toHaveBeenCalledWith('stable'); + }); }); describe('requiredCliChannels', () => { diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index 8de4ba29..3b0f2936 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -99,8 +99,10 @@ export async function resolveChannelPins( const resolved = await Promise.all( [...channels].map(async (channel) => { const envVar = CLI_CHANNEL_ENV[channel]; - const override = process.env[envVar]; - return [envVar, override ?? (await resolveCliVersion(channel))] as const; + // A workflow that exports a blank input still sets the env var, so + // blank/whitespace must be treated as unset rather than as a pin of ''. + const override = process.env[envVar]?.trim(); + return [envVar, override || (await resolveCliVersion(channel))] as const; }) ); return Object.fromEntries(resolved); diff --git a/experiments/ai/codex-gpt-6-luna.experiment.ts b/experiments/ai/codex-gpt-6-luna.experiment.ts index d41a4e20..df537654 100644 --- a/experiments/ai/codex-gpt-6-luna.experiment.ts +++ b/experiments/ai/codex-gpt-6-luna.experiment.ts @@ -3,6 +3,5 @@ import { codexGpt6Luna } from '../presets.js'; export default defineExperiment({ ...codexGpt6Luna, - // cli: the pinned-CLI baseline column for CLI-team evals. - suite: ['benchmark', 'regression', 'cli'], + suite: ['benchmark', 'regression'], }); diff --git a/experiments/cli/codex-gpt-6-luna-cli-pinned.experiment.ts b/experiments/cli/codex-gpt-6-luna-cli-pinned.experiment.ts new file mode 100644 index 00000000..5de01107 --- /dev/null +++ b/experiments/cli/codex-gpt-6-luna-cli-pinned.experiment.ts @@ -0,0 +1,9 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { codexGpt6Luna } from '../presets.js'; +import { skipUnlessCli } from './lib/skip.js'; + +export default defineExperiment({ + ...codexGpt6Luna, + suite: ['cli'], + skipEval: skipUnlessCli, +}); diff --git a/packages/core/src/eval-metadata.test.ts b/packages/core/src/eval-metadata.test.ts new file mode 100644 index 00000000..f88163ba --- /dev/null +++ b/packages/core/src/eval-metadata.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from 'vitest'; +import { evalMetadataSchema, rawEvalResultSchema } from './eval-metadata.js'; + +describe('cliVersion schema', () => { + const BETA_VERSION = '2.118.0-beta.60'; + + it('round-trips a resolved beta version through the frontmatter metadata schema', () => { + const parsed = evalMetadataSchema.parse({ + stage: 'build', + product: ['database'], + topic: ['sql'], + interface: 'cli', + cliVersion: BETA_VERSION, + }); + + expect(parsed.cliVersion).toBe(BETA_VERSION); + }); + + it('round-trips a resolved beta version through the result schema', () => { + const parsed = rawEvalResultSchema.parse({ + experiment: 'test-experiment', + eval: 'test-eval', + interface: 'cli', + cliVersion: BETA_VERSION, + }); + + expect(parsed.cliVersion).toBe(BETA_VERSION); + }); + + it('still rejects a non-version string', () => { + expect(() => + evalMetadataSchema.parse({ + stage: 'build', + product: ['database'], + topic: ['sql'], + interface: 'cli', + cliVersion: 'stable', + }) + ).toThrow(); + }); +}); diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index 8e97be86..d3078052 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -111,7 +111,9 @@ export type ExperimentDisplayMetadata = z.infer< export const evalInterfaceSchema = z.enum(['mcp', 'cli']); export const EVAL_INTERFACES = evalInterfaceSchema.options; export type EvalInterface = z.infer; -const cliVersionSchema = z.string().regex(/^\d+\.\d+\.\d+$/); +// Matches VERSION_RE in packages/sandbox/src/cli-channel.ts, so a resolved +// beta version (e.g. "2.118.0-beta.60") round-trips through this schema too. +const cliVersionSchema = z.string().regex(/^\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?$/); export type EvalMetadata = { stage: EvalStage;