From 66d4d0b89c6aabd98334333d750b2f6c3f0108ba Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Mon, 21 Sep 2026 18:22:25 +0100 Subject: [PATCH 1/2] feat: organize experiments by owner --- .github/CODEOWNERS | 3 + CONTRIBUTING.md | 2 +- README.md | 2 +- apps/framework/harness/run-eval.ts | 11 +-- apps/framework/lib/experiment-files.test.ts | 42 ++++++++++ apps/framework/lib/experiment-files.ts | 37 ++++++++ apps/framework/package.json | 2 +- apps/framework/scripts/export-results.ts | 9 +- ...claude-code-opus-5-no-skills.experiment.ts | 8 ++ .../ai/claude-code-opus-5.experiment.ts | 7 ++ ...aude-code-sonnet-5-no-skills.experiment.ts | 8 ++ .../ai/claude-code-sonnet-5.experiment.ts | 7 ++ ...codex-gpt-5.6-luna-no-skills.experiment.ts | 10 +++ .../ai/codex-gpt-5.6-luna.experiment.ts | 7 ++ .../codex-gpt-5.6-sol-no-skills.experiment.ts | 8 ++ .../ai/codex-gpt-5.6-sol.experiment.ts | 7 ++ .../ai/grok-4.6-no-skills.experiment.ts | 8 ++ experiments/ai/grok-4.6.experiment.ts | 7 ++ .../opencode-kimi-k3-no-skills.experiment.ts | 8 ++ experiments/ai/opencode-kimi-k3.experiment.ts | 7 ++ experiments/claude-code-opus-5-no-skills.ts | 20 ----- experiments/claude-code-opus-5.ts | 20 ----- experiments/claude-code-sonnet-5-no-skills.ts | 21 ----- experiments/claude-code-sonnet-5.ts | 20 ----- experiments/codex-gpt-5.6-luna-no-skills.ts | 22 ----- experiments/codex-gpt-5.6-luna.ts | 20 ----- experiments/codex-gpt-5.6-sol-no-skills.ts | 20 ----- experiments/codex-gpt-5.6-sol.ts | 20 ----- experiments/grok-4.6-no-skills.ts | 21 ----- experiments/grok-4.6.ts | 20 ----- experiments/opencode-kimi-k3-no-skills.ts | 20 ----- experiments/opencode-kimi-k3.ts | 19 ----- experiments/presets.ts | 84 +++++++++++++++++++ 33 files changed, 271 insertions(+), 256 deletions(-) create mode 100644 apps/framework/lib/experiment-files.test.ts create mode 100644 apps/framework/lib/experiment-files.ts create mode 100644 experiments/ai/claude-code-opus-5-no-skills.experiment.ts create mode 100644 experiments/ai/claude-code-opus-5.experiment.ts create mode 100644 experiments/ai/claude-code-sonnet-5-no-skills.experiment.ts create mode 100644 experiments/ai/claude-code-sonnet-5.experiment.ts create mode 100644 experiments/ai/codex-gpt-5.6-luna-no-skills.experiment.ts create mode 100644 experiments/ai/codex-gpt-5.6-luna.experiment.ts create mode 100644 experiments/ai/codex-gpt-5.6-sol-no-skills.experiment.ts create mode 100644 experiments/ai/codex-gpt-5.6-sol.experiment.ts create mode 100644 experiments/ai/grok-4.6-no-skills.experiment.ts create mode 100644 experiments/ai/grok-4.6.experiment.ts create mode 100644 experiments/ai/opencode-kimi-k3-no-skills.experiment.ts create mode 100644 experiments/ai/opencode-kimi-k3.experiment.ts delete mode 100644 experiments/claude-code-opus-5-no-skills.ts delete mode 100644 experiments/claude-code-opus-5.ts delete mode 100644 experiments/claude-code-sonnet-5-no-skills.ts delete mode 100644 experiments/claude-code-sonnet-5.ts delete mode 100644 experiments/codex-gpt-5.6-luna-no-skills.ts delete mode 100644 experiments/codex-gpt-5.6-luna.ts delete mode 100644 experiments/codex-gpt-5.6-sol-no-skills.ts delete mode 100644 experiments/codex-gpt-5.6-sol.ts delete mode 100644 experiments/grok-4.6-no-skills.ts delete mode 100644 experiments/grok-4.6.ts delete mode 100644 experiments/opencode-kimi-k3-no-skills.ts delete mode 100644 experiments/opencode-kimi-k3.ts create mode 100644 experiments/presets.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 26f4dc44..f8a1b32e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,4 +1,7 @@ * @supabase/ai +/experiments/ai/ @supabase/ai +/experiments/presets.ts @supabase/ai + /evals/docs/ @supabase/docs /apps/web/src/data/docs-eval-results.json @supabase/docs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d0918bc3..ddf6d41b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,7 +41,7 @@ Prefer building checks declaratively and returning the list in one place instead ## Adding an experiment -Add a file under `experiments/` for the agent, model, and runtime setup you want to compare. Here you can configure which skills and MCP servers are available. +Add a `*.experiment.ts` file under `experiments//` for the agent, model, and runtime setup you want to compare. Experiment discovery only scans this owner directory depth, so supporting files can live beside experiments or in nested directories. Reuse the base configs exported from `experiments/presets.ts` where they fit. Select the experiment's `suite:` depending on your use case. If this experiment should be part of our published benchmark, assign `suite: ["benchmark"]` and include a corresponding `*-no-skills` variant to compare results with and without skills. You can also assign custom experiment suites for grouping related experiments for other head-to-head comparisons as desired. diff --git a/README.md b/README.md index 52a84349..ceb81932 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Agent-backed runs require the relevant provider key in `.env` (e.g. `OPENAI_API_ ## Concepts - An **eval** is one scenario under `evals///`. It contains the prompt, scorer, and optional starting state for the two environments: `remote/` (the hosted project) and `local/` (the agent's working files). -- An **experiment** is one agent/runtime/model setup under `experiments/.ts`. +- An **experiment** is one agent/runtime/model setup under `experiments//.experiment.ts`. Its ID remains ``. - An **eval suite** is a named set of evals to run together. An eval's suite is its parent folder under `evals/` (`benchmark`, `regression`, `docs`, or `other`). - An **experiment suite** is a named set of experiments with related configurations, for head to head comparisons. - An **agent** is the model driver that receives the eval prompt and calls the configured tools. diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index e932b827..14cb6b9e 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -27,6 +27,7 @@ import { readRepeatedFlag, readSuiteFilters, } from '../lib/cli-args.js'; +import { discoverExperimentFiles } from '../lib/experiment-files.js'; import { bootPlatformBackend } from './platform-backend.js'; import { viteBuild, vitestRun } from './project-runner.js'; import { buildSystemPrompt } from './system-prompt.js'; @@ -101,10 +102,10 @@ const DEBUG = args.has('--debug'); async function loadExperiments() { const dir = join(ROOT, 'experiments'); const out: Array<{ name: string; config: ExperimentConfig }> = []; - for (const f of readdirSync(dir).filter((f) => f.endsWith('.ts'))) { - const mod = await import(pathToFileURL(join(dir, f)).href); + for (const experiment of await discoverExperimentFiles(dir)) { + const mod = await import(pathToFileURL(experiment.path).href); out.push({ - name: f.replace(/\.ts$/, ''), + name: experiment.name, config: mod.default as ExperimentConfig, }); } @@ -396,7 +397,7 @@ async function runOne( if (!exp.localStack) { throw new Error( `eval ${ev.id} has interface: cli but experiment "${expName}" does not configure a local stack runtime. ` + - `Add \`localStack: localStackRuntime()\` (from "@supabase-evals/sandbox") to experiments/${expName}.ts.` + `Add \`localStack: localStackRuntime()\` (from "@supabase-evals/sandbox") to experiments//${expName}.experiment.ts.` ); } // Boots a platform-lite backend seeded from the eval's `remote/` dir so scorers @@ -714,7 +715,7 @@ async function main() { for (const ev of suiteFiltered) { if (ev.mode === 'local-stack' && !config.localStack) { console.log( - `SKIP ${name} x ${ev.id} (no local stack runtime — add \`localStack: localStackRuntime()\` from "@supabase-evals/sandbox" to experiments/${name}.ts)` + `SKIP ${name} x ${ev.id} (no local stack runtime — add \`localStack: localStackRuntime()\` from "@supabase-evals/sandbox" to experiments//${name}.experiment.ts)` ); continue; } diff --git a/apps/framework/lib/experiment-files.test.ts b/apps/framework/lib/experiment-files.test.ts new file mode 100644 index 00000000..aa0b5aca --- /dev/null +++ b/apps/framework/lib/experiment-files.test.ts @@ -0,0 +1,42 @@ +import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { discoverExperimentFiles } from './experiment-files.js'; + +let root: string | undefined; + +afterEach(async () => { + if (root) await rm(root, { recursive: true, force: true }); + root = undefined; +}); + +describe('discoverExperimentFiles', () => { + it('discovers one owner layer and keeps experiment IDs stable', async () => { + root = await mkdtemp(join(tmpdir(), 'eval-experiments-')); + await mkdir(join(root, 'ai', 'lib'), { recursive: true }); + await mkdir(join(root, '_private')); + await writeFile(join(root, 'presets.ts'), 'export {};'); + await writeFile( + join(root, 'ai', 'model.experiment.ts'), + 'export default {};' + ); + await writeFile(join(root, 'ai', 'support.ts'), 'export {};'); + await writeFile( + join(root, 'ai', 'lib', 'nested.experiment.ts'), + 'export default {};' + ); + await writeFile( + join(root, '_private', 'hidden.experiment.ts'), + 'export default {};' + ); + + expect(await discoverExperimentFiles(root)).toEqual([ + { + name: 'model', + path: join(root, 'ai', 'model.experiment.ts'), + }, + ]); + }); +}); diff --git a/apps/framework/lib/experiment-files.ts b/apps/framework/lib/experiment-files.ts new file mode 100644 index 00000000..e03cdaa6 --- /dev/null +++ b/apps/framework/lib/experiment-files.ts @@ -0,0 +1,37 @@ +import { readdir } from 'node:fs/promises'; +import { join } from 'node:path'; + +const EXPERIMENT_SUFFIX = '.experiment.ts'; + +export type ExperimentFile = { + name: string; + path: string; +}; + +export async function discoverExperimentFiles( + experimentsDir: string +): Promise { + const files: ExperimentFile[] = []; + + for (const owner of await readdir(experimentsDir, { withFileTypes: true })) { + if ( + !owner.isDirectory() || + owner.name.startsWith('.') || + owner.name.startsWith('_') + ) { + continue; + } + + const ownerDir = join(experimentsDir, owner.name); + for (const entry of await readdir(ownerDir, { withFileTypes: true })) { + if (!entry.isFile() || !entry.name.endsWith(EXPERIMENT_SUFFIX)) continue; + + files.push({ + name: entry.name.slice(0, -EXPERIMENT_SUFFIX.length), + path: join(ownerDir, entry.name), + }); + } + } + + return files.sort((a, b) => a.name.localeCompare(b.name)); +} diff --git a/apps/framework/package.json b/apps/framework/package.json index 7f73e481..770ad8ac 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -12,7 +12,7 @@ "typecheck": "tsc --noEmit", "test": "vitest run harness", "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", - "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts lib/cli-args.test.ts lib/sample-sets.test.ts", + "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts lib/cli-args.test.ts lib/experiment-files.test.ts lib/sample-sets.test.ts", "export-results": "node --import tsx/esm scripts/export-results.ts", "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts", "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts" diff --git a/apps/framework/scripts/export-results.ts b/apps/framework/scripts/export-results.ts index 2c7c28d5..50e8912f 100644 --- a/apps/framework/scripts/export-results.ts +++ b/apps/framework/scripts/export-results.ts @@ -27,6 +27,7 @@ import { readRepeatedFlag, readSuiteFilters, } from '../lib/cli-args.js'; +import { discoverExperimentFiles } from '../lib/experiment-files.js'; import { formatIncompleteSampleSets, splitBySampleSetCompleteness, @@ -55,12 +56,10 @@ async function loadExperimentMetadata(): Promise< Map > { const map = new Map(); - for (const f of (await readdir(EXPERIMENTS_DIR)).filter((f) => - f.endsWith('.ts') - )) { - const mod = await import(pathToFileURL(join(EXPERIMENTS_DIR, f)).href); + for (const experiment of await discoverExperimentFiles(EXPERIMENTS_DIR)) { + const mod = await import(pathToFileURL(experiment.path).href); const config = mod.default as ExperimentConfig; - map.set(f.replace(/\.ts$/, ''), { + map.set(experiment.name, { display: getExperimentDisplayMetadata(config), experimentSuite: config.suite?.[0], }); diff --git a/experiments/ai/claude-code-opus-5-no-skills.experiment.ts b/experiments/ai/claude-code-opus-5-no-skills.experiment.ts new file mode 100644 index 00000000..4971e48a --- /dev/null +++ b/experiments/ai/claude-code-opus-5-no-skills.experiment.ts @@ -0,0 +1,8 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { claudeCodeOpus5 } from '../presets.ts'; + +export default defineExperiment({ + ...claudeCodeOpus5, + suite: ['no-skills'], + skills: [], +}); diff --git a/experiments/ai/claude-code-opus-5.experiment.ts b/experiments/ai/claude-code-opus-5.experiment.ts new file mode 100644 index 00000000..0cb528f7 --- /dev/null +++ b/experiments/ai/claude-code-opus-5.experiment.ts @@ -0,0 +1,7 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { claudeCodeOpus5 } from '../presets.ts'; + +export default defineExperiment({ + ...claudeCodeOpus5, + suite: ['benchmark'], +}); diff --git a/experiments/ai/claude-code-sonnet-5-no-skills.experiment.ts b/experiments/ai/claude-code-sonnet-5-no-skills.experiment.ts new file mode 100644 index 00000000..4d689501 --- /dev/null +++ b/experiments/ai/claude-code-sonnet-5-no-skills.experiment.ts @@ -0,0 +1,8 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { claudeCodeSonnet5 } from '../presets.ts'; + +export default defineExperiment({ + ...claudeCodeSonnet5, + suite: ['no-skills'], + skills: [], +}); diff --git a/experiments/ai/claude-code-sonnet-5.experiment.ts b/experiments/ai/claude-code-sonnet-5.experiment.ts new file mode 100644 index 00000000..5ca332a7 --- /dev/null +++ b/experiments/ai/claude-code-sonnet-5.experiment.ts @@ -0,0 +1,7 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { claudeCodeSonnet5 } from '../presets.ts'; + +export default defineExperiment({ + ...claudeCodeSonnet5, + suite: ['benchmark'], +}); diff --git a/experiments/ai/codex-gpt-5.6-luna-no-skills.experiment.ts b/experiments/ai/codex-gpt-5.6-luna-no-skills.experiment.ts new file mode 100644 index 00000000..f2c6da69 --- /dev/null +++ b/experiments/ai/codex-gpt-5.6-luna-no-skills.experiment.ts @@ -0,0 +1,10 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { codexGpt56Luna } from '../presets.ts'; + +export default defineExperiment({ + ...codexGpt56Luna, + suite: ['no-skills', 'regression', 'docs'], + skills: [], + // Evals that override `skills: []` already run under the baseline experiment. + skipEval: (ev) => ev.metadata.skills?.length === 0, +}); diff --git a/experiments/ai/codex-gpt-5.6-luna.experiment.ts b/experiments/ai/codex-gpt-5.6-luna.experiment.ts new file mode 100644 index 00000000..30b8aba2 --- /dev/null +++ b/experiments/ai/codex-gpt-5.6-luna.experiment.ts @@ -0,0 +1,7 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { codexGpt56Luna } from '../presets.ts'; + +export default defineExperiment({ + ...codexGpt56Luna, + suite: ['benchmark', 'regression'], +}); diff --git a/experiments/ai/codex-gpt-5.6-sol-no-skills.experiment.ts b/experiments/ai/codex-gpt-5.6-sol-no-skills.experiment.ts new file mode 100644 index 00000000..dba259e1 --- /dev/null +++ b/experiments/ai/codex-gpt-5.6-sol-no-skills.experiment.ts @@ -0,0 +1,8 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { codexGpt56Sol } from '../presets.ts'; + +export default defineExperiment({ + ...codexGpt56Sol, + suite: ['no-skills'], + skills: [], +}); diff --git a/experiments/ai/codex-gpt-5.6-sol.experiment.ts b/experiments/ai/codex-gpt-5.6-sol.experiment.ts new file mode 100644 index 00000000..ca5618dc --- /dev/null +++ b/experiments/ai/codex-gpt-5.6-sol.experiment.ts @@ -0,0 +1,7 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { codexGpt56Sol } from '../presets.ts'; + +export default defineExperiment({ + ...codexGpt56Sol, + suite: ['benchmark'], +}); diff --git a/experiments/ai/grok-4.6-no-skills.experiment.ts b/experiments/ai/grok-4.6-no-skills.experiment.ts new file mode 100644 index 00000000..2fa297f2 --- /dev/null +++ b/experiments/ai/grok-4.6-no-skills.experiment.ts @@ -0,0 +1,8 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { grok46 } from '../presets.ts'; + +export default defineExperiment({ + ...grok46, + suite: ['no-skills'], + skills: [], +}); diff --git a/experiments/ai/grok-4.6.experiment.ts b/experiments/ai/grok-4.6.experiment.ts new file mode 100644 index 00000000..fbad7e6f --- /dev/null +++ b/experiments/ai/grok-4.6.experiment.ts @@ -0,0 +1,7 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { grok46 } from '../presets.ts'; + +export default defineExperiment({ + ...grok46, + suite: ['benchmark'], +}); diff --git a/experiments/ai/opencode-kimi-k3-no-skills.experiment.ts b/experiments/ai/opencode-kimi-k3-no-skills.experiment.ts new file mode 100644 index 00000000..da4572df --- /dev/null +++ b/experiments/ai/opencode-kimi-k3-no-skills.experiment.ts @@ -0,0 +1,8 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { opencodeKimiK3 } from '../presets.ts'; + +export default defineExperiment({ + ...opencodeKimiK3, + suite: ['no-skills'], + skills: [], +}); diff --git a/experiments/ai/opencode-kimi-k3.experiment.ts b/experiments/ai/opencode-kimi-k3.experiment.ts new file mode 100644 index 00000000..4101daa9 --- /dev/null +++ b/experiments/ai/opencode-kimi-k3.experiment.ts @@ -0,0 +1,7 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { opencodeKimiK3 } from '../presets.ts'; + +export default defineExperiment({ + ...opencodeKimiK3, + suite: ['benchmark'], +}); diff --git a/experiments/claude-code-opus-5-no-skills.ts b/experiments/claude-code-opus-5-no-skills.ts deleted file mode 100644 index a93df1d8..00000000 --- a/experiments/claude-code-opus-5-no-skills.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { - claudeCodeAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - suite: ['no-skills'], - agent: claudeCodeAgent({ - model: 'claude-opus-5', - reasoningEffort: 'high', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: [], -}); diff --git a/experiments/claude-code-opus-5.ts b/experiments/claude-code-opus-5.ts deleted file mode 100644 index 0ffe0823..00000000 --- a/experiments/claude-code-opus-5.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { - claudeCodeAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - suite: ['benchmark'], - agent: claudeCodeAgent({ - model: 'claude-opus-5', - reasoningEffort: 'high', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -}); diff --git a/experiments/claude-code-sonnet-5-no-skills.ts b/experiments/claude-code-sonnet-5-no-skills.ts deleted file mode 100644 index 5a266f92..00000000 --- a/experiments/claude-code-sonnet-5-no-skills.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { - claudeCodeAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -// Same as claude-code-sonnet-5 but with no skills, to measure skills' impact. -export default defineExperiment({ - suite: ['no-skills'], - agent: claudeCodeAgent({ - model: 'claude-sonnet-5', - reasoningEffort: 'high', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: [], -}); diff --git a/experiments/claude-code-sonnet-5.ts b/experiments/claude-code-sonnet-5.ts deleted file mode 100644 index e2b01a50..00000000 --- a/experiments/claude-code-sonnet-5.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { - claudeCodeAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - suite: ['benchmark'], - agent: claudeCodeAgent({ - model: 'claude-sonnet-5', - reasoningEffort: 'high', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -}); diff --git a/experiments/codex-gpt-5.6-luna-no-skills.ts b/experiments/codex-gpt-5.6-luna-no-skills.ts deleted file mode 100644 index ef9f97cd..00000000 --- a/experiments/codex-gpt-5.6-luna-no-skills.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { - codexAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - suite: ['no-skills', 'regression', 'docs'], - agent: codexAgent({ - model: 'gpt-5.6-luna', - reasoningEffort: 'medium', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: [], - // Evals that override `skills: []` already run under the baseline experiment. Skip them from running again here. - skipEval: (ev) => ev.metadata.skills?.length === 0, -}); diff --git a/experiments/codex-gpt-5.6-luna.ts b/experiments/codex-gpt-5.6-luna.ts deleted file mode 100644 index d4e18052..00000000 --- a/experiments/codex-gpt-5.6-luna.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { - codexAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - suite: ['benchmark', 'regression'], - agent: codexAgent({ - model: 'gpt-5.6-luna', - reasoningEffort: 'medium', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -}); diff --git a/experiments/codex-gpt-5.6-sol-no-skills.ts b/experiments/codex-gpt-5.6-sol-no-skills.ts deleted file mode 100644 index 4902cf78..00000000 --- a/experiments/codex-gpt-5.6-sol-no-skills.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { - codexAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - suite: ['no-skills'], - agent: codexAgent({ - model: 'gpt-5.6-sol', - reasoningEffort: 'medium', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: [], -}); diff --git a/experiments/codex-gpt-5.6-sol.ts b/experiments/codex-gpt-5.6-sol.ts deleted file mode 100644 index b3894270..00000000 --- a/experiments/codex-gpt-5.6-sol.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { - codexAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - suite: ['benchmark'], - agent: codexAgent({ - model: 'gpt-5.6-sol', - reasoningEffort: 'medium', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -}); diff --git a/experiments/grok-4.6-no-skills.ts b/experiments/grok-4.6-no-skills.ts deleted file mode 100644 index a53d232b..00000000 --- a/experiments/grok-4.6-no-skills.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { - defineExperiment, - grokAgent, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -// Same as grok-4.6 but with no skills, to measure skills' impact. -export default defineExperiment({ - suite: ['no-skills'], - agent: grokAgent({ - model: 'grok-4.6', - reasoningEffort: 'high', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: [], -}); diff --git a/experiments/grok-4.6.ts b/experiments/grok-4.6.ts deleted file mode 100644 index 1f6fc541..00000000 --- a/experiments/grok-4.6.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { - defineExperiment, - grokAgent, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - suite: ['benchmark'], - agent: grokAgent({ - model: 'grok-4.6', - reasoningEffort: 'high', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -}); diff --git a/experiments/opencode-kimi-k3-no-skills.ts b/experiments/opencode-kimi-k3-no-skills.ts deleted file mode 100644 index 0a6d2edd..00000000 --- a/experiments/opencode-kimi-k3-no-skills.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { - defineExperiment, - opencodeAgent, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -// Same as opencode-kimi-k3 but with no skills, to measure skills' impact. -export default defineExperiment({ - suite: ['no-skills'], - agent: opencodeAgent({ - model: 'moonshotai/kimi-k3', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: [], -}); diff --git a/experiments/opencode-kimi-k3.ts b/experiments/opencode-kimi-k3.ts deleted file mode 100644 index be08d45f..00000000 --- a/experiments/opencode-kimi-k3.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { - defineExperiment, - opencodeAgent, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - suite: ['benchmark'], - agent: opencodeAgent({ - model: 'moonshotai/kimi-k3', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -}); diff --git a/experiments/presets.ts b/experiments/presets.ts new file mode 100644 index 00000000..7b16500d --- /dev/null +++ b/experiments/presets.ts @@ -0,0 +1,84 @@ +import { + claudeCodeAgent, + codexAgent, + grokAgent, + opencodeAgent, + platformLiteRuntime, + supabaseMcpServer, + type ExperimentConfig, +} from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; + +type ExperimentPreset = Pick< + ExperimentConfig, + 'agent' | 'runtime' | 'localStack' | 'skills' +>; + +const skills = ['supabase', 'supabase-postgres-best-practices']; + +function defaultRuntime() { + return platformLiteRuntime({ + mcpServers: [supabaseMcpServer()], + }); +} + +export const claudeCodeOpus5 = { + agent: claudeCodeAgent({ + model: 'claude-opus-5', + reasoningEffort: 'high', + }), + runtime: defaultRuntime(), + localStack: localStackRuntime(), + skills, +} satisfies ExperimentPreset; + +export const claudeCodeSonnet5 = { + agent: claudeCodeAgent({ + model: 'claude-sonnet-5', + reasoningEffort: 'high', + }), + runtime: defaultRuntime(), + localStack: localStackRuntime(), + skills, +} satisfies ExperimentPreset; + +export const codexGpt56Luna = { + agent: codexAgent({ + model: 'gpt-5.6-luna', + reasoningEffort: 'medium', + }), + runtime: defaultRuntime(), + localStack: localStackRuntime(), + skills, +} satisfies ExperimentPreset; + +export const baselineExperiment = codexGpt56Luna; + +export const codexGpt56Sol = { + agent: codexAgent({ + model: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }), + runtime: defaultRuntime(), + localStack: localStackRuntime(), + skills, +} satisfies ExperimentPreset; + +export const grok46 = { + agent: grokAgent({ + model: 'grok-4.6', + reasoningEffort: 'high', + }), + runtime: defaultRuntime(), + localStack: localStackRuntime(), + skills, +} satisfies ExperimentPreset; + +export const opencodeKimiK3 = { + agent: opencodeAgent({ + model: 'moonshotai/kimi-k3', + }), + runtime: defaultRuntime(), + localStack: localStackRuntime(), + skills, +} satisfies ExperimentPreset; From 1e930b22cbdd35b64219b9bc339096669ed34754 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Tue, 22 Sep 2026 09:32:48 +0100 Subject: [PATCH 2/2] fix: address experiment ownership review Co-authored-by: OpenAI Codex --- .github/CODEOWNERS | 3 --- apps/framework/lib/experiment-files.test.ts | 18 ++++++++++++++++++ apps/framework/lib/experiment-files.ts | 17 +++++++++++++---- apps/framework/tsconfig.json | 2 +- .../claude-code-opus-5-no-skills.experiment.ts | 2 +- .../ai/claude-code-opus-5.experiment.ts | 2 +- ...laude-code-sonnet-5-no-skills.experiment.ts | 2 +- .../ai/claude-code-sonnet-5.experiment.ts | 2 +- .../codex-gpt-5.6-luna-no-skills.experiment.ts | 2 +- .../ai/codex-gpt-5.6-luna.experiment.ts | 2 +- .../codex-gpt-5.6-sol-no-skills.experiment.ts | 2 +- experiments/ai/codex-gpt-5.6-sol.experiment.ts | 2 +- .../ai/grok-4.6-no-skills.experiment.ts | 2 +- experiments/ai/grok-4.6.experiment.ts | 2 +- .../opencode-kimi-k3-no-skills.experiment.ts | 2 +- experiments/ai/opencode-kimi-k3.experiment.ts | 2 +- experiments/presets.ts | 2 -- 17 files changed, 44 insertions(+), 22 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f8a1b32e..26f4dc44 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,7 +1,4 @@ * @supabase/ai -/experiments/ai/ @supabase/ai -/experiments/presets.ts @supabase/ai - /evals/docs/ @supabase/docs /apps/web/src/data/docs-eval-results.json @supabase/docs diff --git a/apps/framework/lib/experiment-files.test.ts b/apps/framework/lib/experiment-files.test.ts index aa0b5aca..be75c307 100644 --- a/apps/framework/lib/experiment-files.test.ts +++ b/apps/framework/lib/experiment-files.test.ts @@ -39,4 +39,22 @@ describe('discoverExperimentFiles', () => { }, ]); }); + + it('rejects duplicate experiment IDs across owners', async () => { + root = await mkdtemp(join(tmpdir(), 'eval-experiments-')); + await mkdir(join(root, 'ai')); + await mkdir(join(root, 'docs')); + await writeFile( + join(root, 'ai', 'model.experiment.ts'), + 'export default {};' + ); + await writeFile( + join(root, 'docs', 'model.experiment.ts'), + 'export default {};' + ); + + await expect(discoverExperimentFiles(root)).rejects.toThrow( + 'Duplicate experiment ID "model"' + ); + }); }); diff --git a/apps/framework/lib/experiment-files.ts b/apps/framework/lib/experiment-files.ts index e03cdaa6..8d2ab49b 100644 --- a/apps/framework/lib/experiment-files.ts +++ b/apps/framework/lib/experiment-files.ts @@ -12,6 +12,7 @@ export async function discoverExperimentFiles( experimentsDir: string ): Promise { const files: ExperimentFile[] = []; + const experimentPaths = new Map(); for (const owner of await readdir(experimentsDir, { withFileTypes: true })) { if ( @@ -26,10 +27,18 @@ export async function discoverExperimentFiles( for (const entry of await readdir(ownerDir, { withFileTypes: true })) { if (!entry.isFile() || !entry.name.endsWith(EXPERIMENT_SUFFIX)) continue; - files.push({ - name: entry.name.slice(0, -EXPERIMENT_SUFFIX.length), - path: join(ownerDir, entry.name), - }); + const name = entry.name.slice(0, -EXPERIMENT_SUFFIX.length); + const path = join(ownerDir, entry.name); + const duplicatePath = experimentPaths.get(name); + + if (duplicatePath) { + throw new Error( + `Duplicate experiment ID "${name}": ${[duplicatePath, path].sort().join(', ')}` + ); + } + + experimentPaths.set(name, path); + files.push({ name, path }); } } diff --git a/apps/framework/tsconfig.json b/apps/framework/tsconfig.json index 0b21c008..46ea7676 100644 --- a/apps/framework/tsconfig.json +++ b/apps/framework/tsconfig.json @@ -1,7 +1,7 @@ { "extends": "../../tsconfig.base.json", "include": [ - "experiments", + "../../experiments", "harness", "shims", "scripts", diff --git a/experiments/ai/claude-code-opus-5-no-skills.experiment.ts b/experiments/ai/claude-code-opus-5-no-skills.experiment.ts index 4971e48a..8a5dcd2a 100644 --- a/experiments/ai/claude-code-opus-5-no-skills.experiment.ts +++ b/experiments/ai/claude-code-opus-5-no-skills.experiment.ts @@ -1,5 +1,5 @@ import { defineExperiment } from '@supabase-evals/core'; -import { claudeCodeOpus5 } from '../presets.ts'; +import { claudeCodeOpus5 } from '../presets.js'; export default defineExperiment({ ...claudeCodeOpus5, diff --git a/experiments/ai/claude-code-opus-5.experiment.ts b/experiments/ai/claude-code-opus-5.experiment.ts index 0cb528f7..1f28798b 100644 --- a/experiments/ai/claude-code-opus-5.experiment.ts +++ b/experiments/ai/claude-code-opus-5.experiment.ts @@ -1,5 +1,5 @@ import { defineExperiment } from '@supabase-evals/core'; -import { claudeCodeOpus5 } from '../presets.ts'; +import { claudeCodeOpus5 } from '../presets.js'; export default defineExperiment({ ...claudeCodeOpus5, diff --git a/experiments/ai/claude-code-sonnet-5-no-skills.experiment.ts b/experiments/ai/claude-code-sonnet-5-no-skills.experiment.ts index 4d689501..df5f5b70 100644 --- a/experiments/ai/claude-code-sonnet-5-no-skills.experiment.ts +++ b/experiments/ai/claude-code-sonnet-5-no-skills.experiment.ts @@ -1,5 +1,5 @@ import { defineExperiment } from '@supabase-evals/core'; -import { claudeCodeSonnet5 } from '../presets.ts'; +import { claudeCodeSonnet5 } from '../presets.js'; export default defineExperiment({ ...claudeCodeSonnet5, diff --git a/experiments/ai/claude-code-sonnet-5.experiment.ts b/experiments/ai/claude-code-sonnet-5.experiment.ts index 5ca332a7..da35fac1 100644 --- a/experiments/ai/claude-code-sonnet-5.experiment.ts +++ b/experiments/ai/claude-code-sonnet-5.experiment.ts @@ -1,5 +1,5 @@ import { defineExperiment } from '@supabase-evals/core'; -import { claudeCodeSonnet5 } from '../presets.ts'; +import { claudeCodeSonnet5 } from '../presets.js'; export default defineExperiment({ ...claudeCodeSonnet5, diff --git a/experiments/ai/codex-gpt-5.6-luna-no-skills.experiment.ts b/experiments/ai/codex-gpt-5.6-luna-no-skills.experiment.ts index f2c6da69..10f09e9c 100644 --- a/experiments/ai/codex-gpt-5.6-luna-no-skills.experiment.ts +++ b/experiments/ai/codex-gpt-5.6-luna-no-skills.experiment.ts @@ -1,5 +1,5 @@ import { defineExperiment } from '@supabase-evals/core'; -import { codexGpt56Luna } from '../presets.ts'; +import { codexGpt56Luna } from '../presets.js'; export default defineExperiment({ ...codexGpt56Luna, diff --git a/experiments/ai/codex-gpt-5.6-luna.experiment.ts b/experiments/ai/codex-gpt-5.6-luna.experiment.ts index 30b8aba2..b44c2574 100644 --- a/experiments/ai/codex-gpt-5.6-luna.experiment.ts +++ b/experiments/ai/codex-gpt-5.6-luna.experiment.ts @@ -1,5 +1,5 @@ import { defineExperiment } from '@supabase-evals/core'; -import { codexGpt56Luna } from '../presets.ts'; +import { codexGpt56Luna } from '../presets.js'; export default defineExperiment({ ...codexGpt56Luna, diff --git a/experiments/ai/codex-gpt-5.6-sol-no-skills.experiment.ts b/experiments/ai/codex-gpt-5.6-sol-no-skills.experiment.ts index dba259e1..220bbb29 100644 --- a/experiments/ai/codex-gpt-5.6-sol-no-skills.experiment.ts +++ b/experiments/ai/codex-gpt-5.6-sol-no-skills.experiment.ts @@ -1,5 +1,5 @@ import { defineExperiment } from '@supabase-evals/core'; -import { codexGpt56Sol } from '../presets.ts'; +import { codexGpt56Sol } from '../presets.js'; export default defineExperiment({ ...codexGpt56Sol, diff --git a/experiments/ai/codex-gpt-5.6-sol.experiment.ts b/experiments/ai/codex-gpt-5.6-sol.experiment.ts index ca5618dc..e0b74e08 100644 --- a/experiments/ai/codex-gpt-5.6-sol.experiment.ts +++ b/experiments/ai/codex-gpt-5.6-sol.experiment.ts @@ -1,5 +1,5 @@ import { defineExperiment } from '@supabase-evals/core'; -import { codexGpt56Sol } from '../presets.ts'; +import { codexGpt56Sol } from '../presets.js'; export default defineExperiment({ ...codexGpt56Sol, diff --git a/experiments/ai/grok-4.6-no-skills.experiment.ts b/experiments/ai/grok-4.6-no-skills.experiment.ts index 2fa297f2..427408b4 100644 --- a/experiments/ai/grok-4.6-no-skills.experiment.ts +++ b/experiments/ai/grok-4.6-no-skills.experiment.ts @@ -1,5 +1,5 @@ import { defineExperiment } from '@supabase-evals/core'; -import { grok46 } from '../presets.ts'; +import { grok46 } from '../presets.js'; export default defineExperiment({ ...grok46, diff --git a/experiments/ai/grok-4.6.experiment.ts b/experiments/ai/grok-4.6.experiment.ts index fbad7e6f..bd9f547f 100644 --- a/experiments/ai/grok-4.6.experiment.ts +++ b/experiments/ai/grok-4.6.experiment.ts @@ -1,5 +1,5 @@ import { defineExperiment } from '@supabase-evals/core'; -import { grok46 } from '../presets.ts'; +import { grok46 } from '../presets.js'; export default defineExperiment({ ...grok46, diff --git a/experiments/ai/opencode-kimi-k3-no-skills.experiment.ts b/experiments/ai/opencode-kimi-k3-no-skills.experiment.ts index da4572df..081efb96 100644 --- a/experiments/ai/opencode-kimi-k3-no-skills.experiment.ts +++ b/experiments/ai/opencode-kimi-k3-no-skills.experiment.ts @@ -1,5 +1,5 @@ import { defineExperiment } from '@supabase-evals/core'; -import { opencodeKimiK3 } from '../presets.ts'; +import { opencodeKimiK3 } from '../presets.js'; export default defineExperiment({ ...opencodeKimiK3, diff --git a/experiments/ai/opencode-kimi-k3.experiment.ts b/experiments/ai/opencode-kimi-k3.experiment.ts index 4101daa9..32f10260 100644 --- a/experiments/ai/opencode-kimi-k3.experiment.ts +++ b/experiments/ai/opencode-kimi-k3.experiment.ts @@ -1,5 +1,5 @@ import { defineExperiment } from '@supabase-evals/core'; -import { opencodeKimiK3 } from '../presets.ts'; +import { opencodeKimiK3 } from '../presets.js'; export default defineExperiment({ ...opencodeKimiK3, diff --git a/experiments/presets.ts b/experiments/presets.ts index 7b16500d..5ee335d6 100644 --- a/experiments/presets.ts +++ b/experiments/presets.ts @@ -52,8 +52,6 @@ export const codexGpt56Luna = { skills, } satisfies ExperimentPreset; -export const baselineExperiment = codexGpt56Luna; - export const codexGpt56Sol = { agent: codexAgent({ model: 'gpt-5.6-sol',