feat: organize experiments by owner

CONTRIBUTING.md

@@ -41,7 +41,7 @@ Prefer building checks declaratively and returning the list in one place instead
 
 ## Adding an experiment
 
-Add a file under `experiments/` for the agent, model, and runtime setup you want to compare. Here you can configure which skills and MCP servers are available.
+Add a `*.experiment.ts` file under `experiments/<owner>/` for the agent, model, and runtime setup you want to compare. Experiment discovery only scans this owner directory depth, so supporting files can live beside experiments or in nested directories. Reuse the base configs exported from `experiments/presets.ts` where they fit.
 
 Select the experiment's `suite:` depending on your use case. If this experiment should be part of our published benchmark, assign `suite: ["benchmark"]` and include a corresponding `*-no-skills` variant to compare results with and without skills. You can also assign custom experiment suites for grouping related experiments for other head-to-head comparisons as desired.
 

README.md

@@ -28,7 +28,7 @@ Agent-backed runs require the relevant provider key in `.env` (e.g. `OPENAI_API_
 ## Concepts
 
 - An **eval** is one scenario under `evals/<suite>/<id>/`. It contains the prompt, scorer, and optional starting state for the two environments: `remote/` (the hosted project) and `local/` (the agent's working files).
-- An **experiment** is one agent/runtime/model setup under `experiments/<name>.ts`.
+- An **experiment** is one agent/runtime/model setup under `experiments/<owner>/<name>.experiment.ts`. Its ID remains `<name>`.
 - An **eval suite** is a named set of evals to run together. An eval's suite is its parent folder under `evals/` (`benchmark`, `regression`, `docs`, or `other`).
 - An **experiment suite** is a named set of experiments with related configurations, for head to head comparisons.
 - An **agent** is the model driver that receives the eval prompt and calls the configured tools.

apps/framework/harness/run-eval.ts

@@ -27,6 +27,7 @@ import {
   readRepeatedFlag,
   readSuiteFilters,
 } from '../lib/cli-args.js';
+import { discoverExperimentFiles } from '../lib/experiment-files.js';
 import { bootPlatformBackend } from './platform-backend.js';
 import { viteBuild, vitestRun } from './project-runner.js';
 import { buildSystemPrompt } from './system-prompt.js';
@@ -101,10 +102,10 @@ const DEBUG = args.has('--debug');
 async function loadExperiments() {
   const dir = join(ROOT, 'experiments');
   const out: Array<{ name: string; config: ExperimentConfig }> = [];
-  for (const f of readdirSync(dir).filter((f) => f.endsWith('.ts'))) {
-    const mod = await import(pathToFileURL(join(dir, f)).href);
+  for (const experiment of await discoverExperimentFiles(dir)) {
+    const mod = await import(pathToFileURL(experiment.path).href);
     out.push({
-      name: f.replace(/\.ts$/, ''),
+      name: experiment.name,
       config: mod.default as ExperimentConfig,
     });
   }
@@ -396,7 +397,7 @@ async function runOne(
     if (!exp.localStack) {
       throw new Error(
         `eval ${ev.id} has interface: cli but experiment "${expName}" does not configure a local stack runtime. ` +
-          `Add \`localStack: localStackRuntime()\` (from "@supabase-evals/sandbox") to experiments/${expName}.ts.`
+          `Add \`localStack: localStackRuntime()\` (from "@supabase-evals/sandbox") to experiments/<team>/${expName}.experiment.ts.`
       );
     }
     // Boots a platform-lite backend seeded from the eval's `remote/` dir so scorers
@@ -714,7 +715,7 @@ async function main() {
     for (const ev of suiteFiltered) {
       if (ev.mode === 'local-stack' && !config.localStack) {
         console.log(
-          `SKIP ${name} x ${ev.id} (no local stack runtime — add \`localStack: localStackRuntime()\` from "@supabase-evals/sandbox" to experiments/${name}.ts)`
+          `SKIP ${name} x ${ev.id} (no local stack runtime — add \`localStack: localStackRuntime()\` from "@supabase-evals/sandbox" to experiments/<team>/${name}.experiment.ts)`
         );
         continue;
       }

apps/framework/lib/experiment-files.test.ts

@@ -0,0 +1,60 @@
+import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, describe, expect, it } from 'vitest';
+
+import { discoverExperimentFiles } from './experiment-files.js';
+
+let root: string | undefined;
+
+afterEach(async () => {
+  if (root) await rm(root, { recursive: true, force: true });
+  root = undefined;
+});
+
+describe('discoverExperimentFiles', () => {
+  it('discovers one owner layer and keeps experiment IDs stable', async () => {
+    root = await mkdtemp(join(tmpdir(), 'eval-experiments-'));
+    await mkdir(join(root, 'ai', 'lib'), { recursive: true });
+    await mkdir(join(root, '_private'));
+    await writeFile(join(root, 'presets.ts'), 'export {};');
+    await writeFile(
+      join(root, 'ai', 'model.experiment.ts'),
+      'export default {};'
+    );
+    await writeFile(join(root, 'ai', 'support.ts'), 'export {};');
+    await writeFile(
+      join(root, 'ai', 'lib', 'nested.experiment.ts'),
+      'export default {};'
+    );
+    await writeFile(
+      join(root, '_private', 'hidden.experiment.ts'),
+      'export default {};'
+    );
+
+    expect(await discoverExperimentFiles(root)).toEqual([
+      {
+        name: 'model',
+        path: join(root, 'ai', 'model.experiment.ts'),
+      },
+    ]);
+  });
+
+  it('rejects duplicate experiment IDs across owners', async () => {
+    root = await mkdtemp(join(tmpdir(), 'eval-experiments-'));
+    await mkdir(join(root, 'ai'));
+    await mkdir(join(root, 'docs'));
+    await writeFile(
+      join(root, 'ai', 'model.experiment.ts'),
+      'export default {};'
+    );
+    await writeFile(
+      join(root, 'docs', 'model.experiment.ts'),
+      'export default {};'
+    );
+
+    await expect(discoverExperimentFiles(root)).rejects.toThrow(
+      'Duplicate experiment ID "model"'
+    );
+  });
+});

apps/framework/lib/experiment-files.ts

@@ -0,0 +1,46 @@
+import { readdir } from 'node:fs/promises';
+import { join } from 'node:path';
+
+const EXPERIMENT_SUFFIX = '.experiment.ts';
+
+export type ExperimentFile = {
+  name: string;
+  path: string;
+};
+
+export async function discoverExperimentFiles(
+  experimentsDir: string
+): Promise<ExperimentFile[]> {
+  const files: ExperimentFile[] = [];
+  const experimentPaths = new Map<string, string>();
+
+  for (const owner of await readdir(experimentsDir, { withFileTypes: true })) {
+    if (
+      !owner.isDirectory() ||
+      owner.name.startsWith('.') ||
+      owner.name.startsWith('_')
+    ) {
+      continue;
+    }
+
+    const ownerDir = join(experimentsDir, owner.name);
+    for (const entry of await readdir(ownerDir, { withFileTypes: true })) {
+      if (!entry.isFile() || !entry.name.endsWith(EXPERIMENT_SUFFIX)) continue;
+
+      const name = entry.name.slice(0, -EXPERIMENT_SUFFIX.length);
+      const path = join(ownerDir, entry.name);
+      const duplicatePath = experimentPaths.get(name);
+
+      if (duplicatePath) {
+        throw new Error(
+          `Duplicate experiment ID "${name}": ${[duplicatePath, path].sort().join(', ')}`
+        );
+      }
+
+      experimentPaths.set(name, path);
+      files.push({ name, path });
+    }
+  }
+
+  return files.sort((a, b) => a.name.localeCompare(b.name));
+}

apps/framework/package.json

@@ -12,7 +12,7 @@
     "typecheck": "tsc --noEmit",
     "test": "vitest run harness",
     "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts",
-    "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts lib/cli-args.test.ts lib/sample-sets.test.ts",
+    "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts lib/cli-args.test.ts lib/experiment-files.test.ts lib/sample-sets.test.ts",
     "export-results": "node --import tsx/esm scripts/export-results.ts",
     "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts",
     "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts"

apps/framework/scripts/export-results.ts

@@ -27,6 +27,7 @@ import {
   readRepeatedFlag,
   readSuiteFilters,
 } from '../lib/cli-args.js';
+import { discoverExperimentFiles } from '../lib/experiment-files.js';
 import {
   formatIncompleteSampleSets,
   splitBySampleSetCompleteness,
@@ -55,12 +56,10 @@ async function loadExperimentMetadata(): Promise<
   Map<string, ExperimentExportMetadata>
 > {
   const map = new Map<string, ExperimentExportMetadata>();
-  for (const f of (await readdir(EXPERIMENTS_DIR)).filter((f) =>
-    f.endsWith('.ts')
-  )) {
-    const mod = await import(pathToFileURL(join(EXPERIMENTS_DIR, f)).href);
+  for (const experiment of await discoverExperimentFiles(EXPERIMENTS_DIR)) {
+    const mod = await import(pathToFileURL(experiment.path).href);
     const config = mod.default as ExperimentConfig;
-    map.set(f.replace(/\.ts$/, ''), {
+    map.set(experiment.name, {
       display: getExperimentDisplayMetadata(config),
       experimentSuite: config.suite?.[0],
     });

apps/framework/tsconfig.json

@@ -1,7 +1,7 @@
 {
   "extends": "../../tsconfig.base.json",
   "include": [
-    "experiments",
+    "../../experiments",
     "harness",
     "shims",
     "scripts",

experiments/ai/claude-code-opus-5-no-skills.experiment.ts

@@ -0,0 +1,8 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { claudeCodeOpus5 } from '../presets.js';
+
+export default defineExperiment({
+  ...claudeCodeOpus5,
+  suite: ['no-skills'],
+  skills: [],
+});

experiments/ai/claude-code-opus-5.experiment.ts

@@ -0,0 +1,7 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { claudeCodeOpus5 } from '../presets.js';
+
+export default defineExperiment({
+  ...claudeCodeOpus5,
+  suite: ['benchmark'],
+});

experiments/ai/claude-code-sonnet-5-no-skills.experiment.ts

@@ -0,0 +1,8 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { claudeCodeSonnet5 } from '../presets.js';
+
+export default defineExperiment({
+  ...claudeCodeSonnet5,
+  suite: ['no-skills'],
+  skills: [],
+});

experiments/ai/claude-code-sonnet-5.experiment.ts

@@ -0,0 +1,7 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { claudeCodeSonnet5 } from '../presets.js';
+
+export default defineExperiment({
+  ...claudeCodeSonnet5,
+  suite: ['benchmark'],
+});

experiments/ai/codex-gpt-5.6-luna-no-skills.experiment.ts

@@ -0,0 +1,10 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { codexGpt56Luna } from '../presets.js';
+
+export default defineExperiment({
+  ...codexGpt56Luna,
+  suite: ['no-skills', 'regression', 'docs'],
+  skills: [],
+  // Evals that override `skills: []` already run under the baseline experiment.
+  skipEval: (ev) => ev.metadata.skills?.length === 0,
+});

experiments/ai/codex-gpt-5.6-luna.experiment.ts

@@ -0,0 +1,7 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { codexGpt56Luna } from '../presets.js';
+
+export default defineExperiment({
+  ...codexGpt56Luna,
+  suite: ['benchmark', 'regression'],
+});

experiments/ai/codex-gpt-5.6-sol-no-skills.experiment.ts

@@ -0,0 +1,8 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { codexGpt56Sol } from '../presets.js';
+
+export default defineExperiment({
+  ...codexGpt56Sol,
+  suite: ['no-skills'],
+  skills: [],
+});

experiments/ai/codex-gpt-5.6-sol.experiment.ts

@@ -0,0 +1,7 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { codexGpt56Sol } from '../presets.js';
+
+export default defineExperiment({
+  ...codexGpt56Sol,
+  suite: ['benchmark'],
+});

experiments/ai/grok-4.6-no-skills.experiment.ts

@@ -0,0 +1,8 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { grok46 } from '../presets.js';
+
+export default defineExperiment({
+  ...grok46,
+  suite: ['no-skills'],
+  skills: [],
+});

experiments/ai/grok-4.6.experiment.ts

@@ -0,0 +1,7 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { grok46 } from '../presets.js';
+
+export default defineExperiment({
+  ...grok46,
+  suite: ['benchmark'],
+});

experiments/ai/opencode-kimi-k3-no-skills.experiment.ts

@@ -0,0 +1,8 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { opencodeKimiK3 } from '../presets.js';
+
+export default defineExperiment({
+  ...opencodeKimiK3,
+  suite: ['no-skills'],
+  skills: [],
+});

experiments/ai/opencode-kimi-k3.experiment.ts

@@ -0,0 +1,7 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { opencodeKimiK3 } from '../presets.js';
+
+export default defineExperiment({
+  ...opencodeKimiK3,
+  suite: ['benchmark'],
+});