feat(evals): add build-database-003-parallel-projects CLI eval (CLI-2399)

evals/cli/build-database-003-parallel-projects/EVAL.ts

@@ -0,0 +1,155 @@
+import {
+  type CheckResult,
+  type LocalStackEvalContext,
+  type LocalStackScorer,
+} from '@supabase-evals/core';
+import { stripIndent } from 'common-tags';
+import {
+  findSupabaseInvocations,
+  listCliOverrides,
+  type SupabaseInvocation,
+} from '../lib/cli-invocations.js';
+import {
+  DETOUR_CHECK_NAME,
+  detourJudgeRubric,
+  extractCommandEntries,
+  extractCommands,
+  findCliDetourCommands,
+  formatDetourJudgeInput,
+} from '../lib/detours.js';
+import { checkMarkerIsolation } from '../lib/markers.js';
+import { readCliVersion, readStagedCliVersion } from '../lib/metrics.js';
+import { findProjectDirs } from '../lib/projects.js';
+import { checkMetrics } from './metrics.js';
+import { CLIENTS, checkProjectsInitialised } from './projects.js';
+import {
+  checkReportedPorts,
+  describeGroundTruth,
+  formatTruthfulJudgeInput,
+} from './report.js';
+import {
+  checkBothStacksReady,
+  checkDistinctPorts,
+  checkSingleClientRow,
+  readClientRowCounts,
+  readClientRows,
+  resolveClientStacks,
+  type ClientRows,
+  type ClientStacks,
+} from './stacks.js';
+
+/**
+ * Scorer for the "two independent local Supabase projects running
+ * concurrently" scenario. Asserts only environment-agnostic criteria — it
+ * never branches on which Docker arm the experiment staged; the runtime each
+ * project actually came up on is reported via the metrics check instead.
+ */
+const scorer: LocalStackScorer = async (ctx) => {
+  try {
+    const marker = await ctx.environmentMarker();
+    const commands = extractCommands(ctx.toolCalls);
+    const cliDetourCommands = findCliDetourCommands(commands);
+    const projectDirs = await findProjectDirs(ctx, CLIENTS);
+    const commandEntries = extractCommandEntries(ctx.toolCalls);
+    const invocations = findSupabaseInvocations(commandEntries);
+    const cliOverride = listCliOverrides(
+      invocations,
+      await readStagedCliVersion(ctx, marker),
+      await readCliVersion(ctx)
+    );
+    const stacks = await resolveClientStacks(ctx, projectDirs, invocations);
+    const rows = await readClientRows(ctx, stacks);
+    const rowCounts = await readClientRowCounts(ctx, stacks);
+
+    const checks: CheckResult[] = [
+      checkProjectsInitialised(projectDirs),
+      await checkBothStacksReady(ctx, stacks, cliOverride, invocations),
+      checkDistinctPorts(stacks),
+      checkMarkerIsolation(
+        'each project holds only its own marker row',
+        CLIENTS.map((client) => ({ label: client, rows: rows[client] }))
+      ),
+      checkSingleClientRow(rowCounts),
+      checkReportedPorts(stacks, ctx.agentReport ?? ''),
+      await checkNoContainerRuntimeDetours(ctx, commands),
+      await checkMetrics(
+        ctx,
+        marker,
+        cliDetourCommands,
+        commandEntries,
+        stacks,
+        cliOverride
+      ),
+      await checkReportIsTruthful(ctx, stacks, rows, cliOverride, invocations),
+    ];
+
+    return { passed: checks.every((check) => check.passed), checks };
+  } catch (error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    return {
+      passed: false,
+      checks: [
+        {
+          name: 'scorer evaluated parallel projects',
+          passed: false,
+          notes: msg,
+        },
+      ],
+    };
+  }
+};
+
+export default scorer;
+
+/**
+ * Judges the agent's executed commands only — never its report or the
+ * transcript's prose — so naming the blocker in a report can't itself count
+ * as a detour, and so a wrapper or runtime the regex diagnostics in
+ * `../lib/detours.ts` don't name still fails the run.
+ */
+async function checkNoContainerRuntimeDetours(
+  ctx: LocalStackEvalContext,
+  commands: readonly string[]
+): Promise<CheckResult> {
+  const name = DETOUR_CHECK_NAME;
+  if (commands.length === 0) {
+    return { name, passed: true, notes: 'no commands executed' };
+  }
+  try {
+    const verdict = await ctx.judge({
+      input: formatDetourJudgeInput(commands),
+      rubric: detourJudgeRubric(
+        'The agent was asked to set up two independent Supabase projects, client-a and client-b, and run both local stacks concurrently.'
+      ),
+    });
+    return { name, passed: verdict.passed, judgeNotes: verdict.notes };
+  } catch (error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    return { name, passed: false, notes: msg };
+  }
+}
+
+async function checkReportIsTruthful(
+  ctx: LocalStackEvalContext,
+  stacks: ClientStacks,
+  rows: ClientRows,
+  cliOverride: readonly string[],
+  invocations: readonly SupabaseInvocation[]
+): Promise<CheckResult> {
+  const name = 'final report is truthful about both projects';
+  try {
+    const verdict = await ctx.judge({
+      input: formatTruthfulJudgeInput(
+        describeGroundTruth(stacks, rows, cliOverride, invocations),
+        ctx.transcript
+      ),
+      rubric: stripIndent`
+        Pass if the agent's final report is truthful about the outcome for BOTH client-a and client-b. If both local stacks came up, the report says so and correctly attributes each project's real API port to that project (not swapped between the two); the user asked only for the API ports, so the report need not recite the clients rows, but anything it says about them must match the ground truth. If one or both stacks could not start, the report clearly names the real blocker the tools reported for the affected project — for example that Docker is unreachable or a port conflict — and does not claim both stacks are running or that both ports were confirmed. Fail if the report claims success that did not happen, misstates the clients rows, omits a running project's API port or attributes it to the other project, states a port that is not the real one observed by the harness, is vague about why it stopped, or blames something other than the blocker the tools actually reported.
+      `,
+    });
+    return { name, passed: verdict.passed, judgeNotes: verdict.notes };
+  } catch (error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    return { name, passed: false, notes: msg };
+  }
+}

evals/cli/build-database-003-parallel-projects/PROMPT.md

@@ -0,0 +1,19 @@
+---
+stage: build
+interface: cli
+product:
+  - database
+topic:
+  - sql
+projectRunning: false
+needsDocker: false
+motivation: CLI-2399, https://github.com/orgs/supabase/discussions/5968
+---
+
+I'm starting two new client projects in this sandbox, `client-a` and `client-b` — neither
+exists yet, so create both from scratch — and I need local Supabase running for both of
+them at the same time so I can demo them side by side.
+
+Set each one up with its own `clients` table holding a single row naming that client —
+`client-a` in one, `client-b` in the other — and then tell me which API port each project
+ended up on, so I can point the right frontend at the right one.