diff --git a/README.md b/README.md index 27af033a..db6cd98a 100644 --- a/README.md +++ b/README.md @@ -161,10 +161,7 @@ Skills come from [`supabase/agent-skills`](https://github.com/supabase/agent-ski To use a skill in an experiment, reference its directory name in the experiment's `skills` array. -Both runtimes load skills lazily ([progressive disclosure](https://ai-sdk.dev/cookbook/guides/agent-skills)): only each skill's name+description is in the system prompt, and the agent pulls a skill's full instructions on demand. They differ only in how the body is fetched, because the tools-mode agent has no filesystem: - -- **Local-stack (sandbox) mode:** skills are installed into the workspace with [Vercel's `skills` CLI](https://github.com/vercel-labs/skills) (baked into the sandbox image, sourced from the local `skills/` directory — never the network), into every project scope the CLI harnesses discover natively: `.claude/skills/` for Claude Code, `.agents/skills/` for Codex and OpenCode. Each CLI discovers the scope it reads and surfaces those skills to the model itself. The framework also still injects a listing naming them, so a CLI harness hears about them twice; removing that is a separate change. -- **Tools mode:** no filesystem, so a `load_skill` tool returns a skill's full instructions when the agent calls it with the skill's name. +Skills are installed into the sandbox workspace with [Vercel's `skills` CLI](https://github.com/vercel-labs/skills) under each harness's native skills folder (`.claude/skills/` for Claude Code, `.agents/skills/` for Codex and OpenCode). The `ai-sdk` harness exposes a `load_skill` tool to support skill loading. ## Framework Checks diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index 18ac322b..257d593d 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -29,6 +29,7 @@ import { } from '../lib/cli-args.js'; import { bootPlatformBackend } from './platform-backend.js'; import { viteBuild, vitestRun } from './project-runner.js'; +import { buildSystemPrompt } from './system-prompt.js'; import { buildDocsResult, buildSkillResult, @@ -319,31 +320,6 @@ function readSessionSeedArgs(ev: EvalManifest) { }; } -function basePromptFor(mode: EvalMode): string { - if (mode === 'local-stack') { - return ( - 'You are an agent solving a Supabase eval task in a Linux workspace. ' + - 'Use the provided tools to inspect and modify the workspace and run commands. ' + - 'When you are done, end your turn with a short summary of what you did.' - ); - } - return ( - 'You are an agent solving a Supabase eval task. ' + - 'Use the provided tools to inspect and modify the project. ' + - 'When you are done, end your turn with a short summary of what you did ' + - '(or for audit tasks, your findings).' - ); -} - -function buildSystemPrompt( - mode: EvalMode, - addendum?: string, - skillContext?: string -): string { - const blocks = [basePromptFor(mode), addendum, skillContext].filter(Boolean); - return blocks.join('\n\n'); -} - /** * Adapt a `{ close() }` resource to `AsyncDisposable` so it can be bound with * `await using` — cleanup then runs on scope exit (normal fall-through, `continue`, @@ -374,6 +350,11 @@ async function runOne( transcript: TranscriptPart[]; agentReport: string; stoppedReason: string; + /** + * The system prompt the harness handed the agent, `''` for a CLI agent. + * Recorded so a run artifact shows what the agent was told. + */ + systemPrompt: string; usage?: AgentUsage; stepCount?: number; toolCallCount: number; @@ -447,8 +428,12 @@ async function runOne( }) ); + const systemPrompt = buildSystemPrompt({ + agent: exp.agent.id, + addendum: session.promptAddendum, + }); const run = await exp.agent.run({ - systemPrompt: buildSystemPrompt('local-stack', session.promptAddendum), + systemPrompt, userPrompt: prompt, tools: session.tools, sandbox: session.sandbox, @@ -492,6 +477,7 @@ async function runOne( transcript: run.transcript, agentReport: run.agentReport, stoppedReason: run.stoppedReason, + systemPrompt, usage: run.usage, stepCount: run.stepCount, toolCallCount: run.toolCalls.length, @@ -504,7 +490,6 @@ async function runOne( await using cliSandbox = agentRunsInSandbox ? disposable( await createBareSandbox({ - agent: exp.agent.id, skills: skillSources, mounts: supabaseMcpServerMounts(), }) @@ -517,16 +502,16 @@ async function runOne( }) ); - // In-process agents have no filesystem, so skills are advertised in the - // prompt and pulled on demand via the load_skill tool instead. - const skillsPrompt = agentRunsInSandbox - ? cliSandbox!.promptAddendum - : buildToolsSkillsPrompt(toolsSkills); - const systemPrompt = buildSystemPrompt( - 'tools', - session.promptAddendum, - skillsPrompt - ); + // A CLI agent discovers its installed skills itself. An in-process agent has + // no filesystem, so its skills are advertised in the prompt and pulled on + // demand via the load_skill tool instead. + const systemPrompt = buildSystemPrompt({ + agent: exp.agent.id, + addendum: session.promptAddendum, + skillContext: agentRunsInSandbox + ? undefined + : buildToolsSkillsPrompt(toolsSkills), + }); const run = await exp.agent.run({ systemPrompt, userPrompt: prompt, @@ -555,6 +540,7 @@ async function runOne( transcript: run.transcript, agentReport: run.agentReport, stoppedReason: run.stoppedReason, + systemPrompt, usage: run.usage, stepCount: run.stepCount, toolCallCount: run.toolCalls.length, @@ -805,9 +791,16 @@ async function main() { } } -main() - .then(() => process.exit(0)) - .catch((e) => { - console.error(e); - process.exit(1); - }); +// Only when this file is the entry point. Importing it (a unit test reaching +// for one of its helpers) must not dispatch a run or call process.exit. +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + main() + .then(() => process.exit(0)) + .catch((e) => { + console.error(e); + process.exit(1); + }); +} diff --git a/apps/framework/harness/system-prompt.test.ts b/apps/framework/harness/system-prompt.test.ts new file mode 100644 index 00000000..80e14d2d --- /dev/null +++ b/apps/framework/harness/system-prompt.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from 'vitest'; +import type { AgentHarnessId } from '@supabase-evals/core'; +import { + buildSkillsPrompt, + buildToolSurfaceAddendum, + type SkillEntry, +} from '@supabase-evals/sandbox'; +import { buildSystemPrompt } from './system-prompt.js'; + +const CLI_AGENTS: AgentHarnessId[] = ['claude-code', 'codex', 'opencode']; + +const skills: SkillEntry[] = [ + { + name: 'supabase', + description: 'Use for Supabase tasks.', + dir: '.claude/skills/supabase', + }, +]; + +describe('buildSystemPrompt', () => { + it('gives the ai-sdk agent task framing', () => { + // ai-sdk is the one harness with no system prompt of its own. + expect(buildSystemPrompt({ agent: 'ai-sdk' })).toContain( + 'Use the provided tools' + ); + }); + + it('gives nothing to any CLI harness', () => { + // CLI harnesses ship their own system prompt; that is what is measured. + for (const agent of CLI_AGENTS) { + expect(buildSystemPrompt({ agent })).toBe(''); + } + }); + + it('assembles to nothing for a CLI harness, even with skills installed', () => { + // The real block producers, not stand-ins: each CLI finds the skills through + // its own project-scope discovery and describes them to the model itself. + for (const agent of CLI_AGENTS) { + expect( + buildSystemPrompt({ + agent, + addendum: buildToolSurfaceAddendum(agent), + skillContext: buildSkillsPrompt(agent, skills), + }) + ).toBe(''); + } + }); + + it('keeps the runtime blocks for ai-sdk, in order, after the base prompt', () => { + const base = buildSystemPrompt({ agent: 'ai-sdk' }); + expect( + buildSystemPrompt({ + agent: 'ai-sdk', + addendum: 'Addendum.', + skillContext: 'Skills listing.', + }) + ).toBe(`${base}\n\nAddendum.\n\nSkills listing.`); + const withSkills = buildSystemPrompt({ + agent: 'ai-sdk', + addendum: buildToolSurfaceAddendum('ai-sdk'), + skillContext: buildSkillsPrompt('ai-sdk', skills), + }); + expect(withSkills).toContain('## Available skills'); + expect(withSkills).toContain('- supabase: Use for Supabase tasks.'); + }); + + it('drops empty blocks instead of leaving blank gaps', () => { + const base = buildSystemPrompt({ agent: 'ai-sdk' }); + expect( + buildSystemPrompt({ + agent: 'ai-sdk', + addendum: '', + skillContext: 'Skills listing.', + }) + ).toBe(`${base}\n\nSkills listing.`); + expect( + buildSystemPrompt({ agent: 'ai-sdk', addendum: '', skillContext: '' }) + ).toBe(base); + }); + + it('never names Supabase, a project, or how to end the turn', () => { + // Issue #164: naming Supabase hands the agent the answer to "which tool"; + // "project" presumes there is one to modify; stopping coaching shapes the + // report the judge reads. All three are part of what is measured. + const prompt = buildSystemPrompt({ agent: 'ai-sdk' }); + expect(prompt).not.toMatch(/supabase/i); + expect(prompt).not.toMatch(/project/i); + expect(prompt).not.toMatch(/eval/i); + expect(prompt).not.toMatch(/summary|end your turn/i); + }); +}); diff --git a/apps/framework/harness/system-prompt.ts b/apps/framework/harness/system-prompt.ts new file mode 100644 index 00000000..e01dbd88 --- /dev/null +++ b/apps/framework/harness/system-prompt.ts @@ -0,0 +1,41 @@ +/** + * System-prompt assembly, per agent harness. + * + * An eval measures the agent as shipped, so the harness adds as little prompt + * of its own as it can. Only the ai-sdk agent gets any: it is the one harness + * with no system prompt of its own (`aiSdkAgent` hands `systemPrompt` straight + * to the model's `system`). Every CLI agent runs with the prompt it ships with, + * and `createCliAgent` refuses a non-empty one. + */ + +import type { AgentHarnessId } from '@supabase-evals/core'; + +/** + * Base framing for the ai-sdk harness: only what it cannot infer on its own, + * that it has tools and should use them. Deliberately says nothing about + * Supabase, a project, or how to finish a turn: which tools the agent reaches + * for and when it stops are part of what is measured (see issue #164). + */ +const AI_SDK_BASE_PROMPT = + 'You are an agent. Use the provided tools to complete the task.'; + +/** + * Assemble the system prompt for the agent. Every block is ai-sdk-only, so a + * CLI harness gets `''`. The runtime blocks (tool surface, skills listing) are + * already empty for a CLI agent at their source; an MCP server's + * `promptAddendum` is not, and is left to reach `createCliAgent`, which throws. + */ +export function buildSystemPrompt({ + agent, + addendum, + skillContext, +}: { + agent: AgentHarnessId; + /** Runtime text: the session's tool surface, or the MCP servers' addenda. */ + addendum?: string; + /** The installed-skills listing. */ + skillContext?: string; +}): string { + const base = agent === 'ai-sdk' ? AI_SDK_BASE_PROMPT : ''; + return [base, addendum, skillContext].filter(Boolean).join('\n\n'); +} diff --git a/apps/framework/package.json b/apps/framework/package.json index d5554504..7f73e481 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -4,12 +4,13 @@ "version": "0.0.1", "type": "module", "scripts": { - "check": "pnpm typecheck && pnpm test:framework && pnpm test:vercel-runner", + "check": "pnpm typecheck && pnpm test && pnpm test:framework && pnpm test:vercel-runner", "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts", "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry", "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke", "eval:vercel": "node --env-file=../../.env --import tsx/esm scripts/run-vercel-evals.ts", "typecheck": "tsc --noEmit", + "test": "vitest run harness", "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts lib/cli-args.test.ts lib/sample-sets.test.ts", "export-results": "node --import tsx/esm scripts/export-results.ts", diff --git a/packages/core/src/agents/claude-code/runner.test.ts b/packages/core/src/agents/claude-code/runner.test.ts index 5ca31f5d..f19e6691 100644 --- a/packages/core/src/agents/claude-code/runner.test.ts +++ b/packages/core/src/agents/claude-code/runner.test.ts @@ -36,6 +36,35 @@ function streamJson(subtype: string, isError = false): string { ].join('\n'); } +/** The `claude` invocation from one exec, with a fake sandbox. */ +async function captureRunCommand(): Promise { + let runCommand = ''; + await claudeCodeRunner.exec({ + sandbox: { + workspace: '/w', + exec: async (cmd) => { + if (cmd.includes('/bin/claude')) runCommand = cmd; + return ok; + }, + readFile: async () => '', + }, + model: 'claude-sonnet-4-6', + apiKey: 'k', + userPromptPath: '"$HOME/.eval/user-prompt.txt"', + mcpServers: {}, + timeoutSec: 1, + }); + return runCommand; +} + +describe('claudeCodeRunner.exec', () => { + it("pipes the task in and leaves Claude Code's own system prompt intact", async () => { + const command = await captureRunCommand(); + expect(command).toContain('cat "$HOME/.eval/user-prompt.txt" |'); + expect(command).not.toContain('system-prompt'); + }); +}); + describe('claudeCodeRunner.deriveStopReason', () => { const derive = claudeCodeRunner.deriveStopReason!; diff --git a/packages/core/src/agents/claude-code/runner.ts b/packages/core/src/agents/claude-code/runner.ts index c8a70da3..fa7b425c 100644 --- a/packages/core/src/agents/claude-code/runner.ts +++ b/packages/core/src/agents/claude-code/runner.ts @@ -40,7 +40,6 @@ export const claudeCodeRunner: AgentRunner = { sandbox, model, apiKey, - systemPromptPath, userPromptPath, mcpServers, reasoningEffort, @@ -68,9 +67,6 @@ export const claudeCodeRunner: AgentRunner = { `--model ${shellQuote(model)}`, // Reasoning effort for the session; omitted leaves Claude Code's default. ...(reasoningEffort ? [`--effort ${shellQuote(reasoningEffort)}`] : []), - // Append (not replace), from a file (no ARG_MAX/shell-expansion surface), - // so Claude Code keeps its default coding-agent prompt + tool guidance. - `--append-system-prompt-file ${systemPromptPath}`, ...mcpFlags, // The sandbox is the isolation boundary, so skip permission prompts and // give the agent its full native toolset (same in both modes). diff --git a/packages/core/src/agents/codex/runner.test.ts b/packages/core/src/agents/codex/runner.test.ts index 30bce0e6..2bead0f7 100644 --- a/packages/core/src/agents/codex/runner.test.ts +++ b/packages/core/src/agents/codex/runner.test.ts @@ -1,6 +1,42 @@ import { describe, expect, it } from 'vitest'; +import type { CommandResult } from '../../index.js'; import { codexRunner, countModelResponses } from './runner.js'; +const ok: CommandResult = { ok: true, exitCode: 0, stdout: '', stderr: '' }; + +/** The `codex exec` invocation from one exec, with a fake sandbox. */ +async function captureRunCommand(): Promise { + let runCommand = ''; + await codexRunner.exec({ + sandbox: { + workspace: '/w', + exec: async (cmd) => { + if (cmd.includes(' exec ')) runCommand = cmd; + return ok; + }, + readFile: async () => '', + }, + model: 'gpt-5.4', + apiKey: 'k', + userPromptPath: '"$HOME/.eval/user-prompt.txt"', + mcpServers: {}, + timeoutSec: 1, + }); + return runCommand; +} + +describe('codexRunner.exec', () => { + it('sends the task alone on stdin', async () => { + // Codex has no system-prompt flag, so anything else here would land on the + // user prompt. Nothing is prepended. + const command = await captureRunCommand(); + expect(command.startsWith('cat "$HOME/.eval/user-prompt.txt" |')).toBe( + true + ); + expect(command).not.toContain('system-prompt'); + }); +}); + describe('codexRunner.extractUsage', () => { const extract = codexRunner.extractUsage!; diff --git a/packages/core/src/agents/codex/runner.ts b/packages/core/src/agents/codex/runner.ts index e5ef5db6..7cf8fa5d 100644 --- a/packages/core/src/agents/codex/runner.ts +++ b/packages/core/src/agents/codex/runner.ts @@ -58,7 +58,6 @@ export const codexRunner: AgentRunner = { sandbox, model, apiKey, - systemPromptPath, userPromptPath, mcpServers, reasoningEffort, @@ -94,10 +93,9 @@ export const codexRunner: AgentRunner = { '-', ].join(' '); - // Codex has no system-prompt flag; prepend the system prompt to the task, - // both staged as files, fed on stdin. + // The staged task file, fed on stdin. const command = await sandbox.exec( - `{ cat ${systemPromptPath}; printf '\\n\\n'; cat ${userPromptPath}; } | ${codex} ${flags}`, + `cat ${userPromptPath} | ${codex} ${flags}`, { timeoutMs: timeoutSec * 1000, env: { OPENAI_API_KEY: apiKey } } ); // The --json stream has no per-response boundary, but the session rollout diff --git a/packages/core/src/agents/engine.test.ts b/packages/core/src/agents/engine.test.ts new file mode 100644 index 00000000..7efcaf18 --- /dev/null +++ b/packages/core/src/agents/engine.test.ts @@ -0,0 +1,80 @@ +import { beforeEach, describe, expect, it } from 'vitest'; +import type { CommandResult } from '../index.js'; +import type { AgentTranscriptParser } from '../parsers/types.js'; +import { createCliAgent } from './engine.js'; +import { USER_PROMPT_PATH } from './shared.js'; +import type { AgentRunner } from './types.js'; + +const ok: CommandResult = { ok: true, exitCode: 0, stdout: '', stderr: '' }; + +const API_KEY_ENV_VAR = 'ENGINE_TEST_API_KEY'; + +/** A parser that reports one assistant message, so the engine stays quiet. */ +const parser: AgentTranscriptParser = { + parseTranscript: () => ({ + events: [{ type: 'message', role: 'assistant', content: 'done' }], + }), +}; + +/** + * Run a CLI agent against a fake sandbox, returning every command it ran in + * the sandbox and whether the runner's `install` was reached. + */ +async function runWithSystemPrompt( + systemPrompt: string, + effects: { commands: string[]; installed: boolean } = { + commands: [], + installed: false, + } +): Promise { + const runner: AgentRunner = { + id: 'claude-code', + displayName: 'Fake CLI', + apiKeyEnvVar: API_KEY_ENV_VAR, + cliPackage: 'fake-cli', + defaultCliVersion: '1.0.0', + defaultModel: 'fake-model', + install: async () => { + effects.installed = true; + }, + exec: async () => ({ command: ok, raw: '' }), + }; + await createCliAgent(runner, parser, { model: 'fake-model' }).run({ + systemPrompt, + userPrompt: 'the task', + timeoutSec: 1, + sandbox: { + workspace: '/w', + exec: async (command) => { + effects.commands.push(command); + return ok; + }, + readFile: async () => '', + }, + }); + return effects; +} + +describe('createCliAgent prompt staging', () => { + // The engine requires the runner's API key before it stages anything. + beforeEach(() => { + process.env[API_KEY_ENV_VAR] = 'k'; + }); + + it('stages only the user prompt', async () => { + const { commands } = await runWithSystemPrompt(''); + expect(commands.some((c) => c.includes(USER_PROMPT_PATH))).toBe(true); + expect(commands.some((c) => c.includes('system-prompt'))).toBe(false); + }); + + it('refuses a system prompt before touching the sandbox', async () => { + // A CLI agent runs with its own prompt. The refusal comes before install + // and staging, so a misconfigured experiment fails without paying for them. + const effects = { commands: [], installed: false }; + await expect( + runWithSystemPrompt('Extra framing.', effects) + ).rejects.toThrow(/runs with its own system prompt/); + expect(effects.commands).toEqual([]); + expect(effects.installed).toBe(false); + }); +}); diff --git a/packages/core/src/agents/engine.ts b/packages/core/src/agents/engine.ts index ebb1972e..7588d1d7 100644 --- a/packages/core/src/agents/engine.ts +++ b/packages/core/src/agents/engine.ts @@ -26,7 +26,6 @@ import type { AgentTranscriptParser } from '../parsers/types.js'; import type { AgentRunner } from './types.js'; import { SCRATCH, - SYSTEM_PROMPT_PATH, USER_PROMPT_PATH, processStopReason, requireEnv, @@ -86,11 +85,21 @@ export function createCliAgent( ); } + // A CLI agent runs with the system prompt it ships with; the harness has + // no way to add to it that is the same across CLIs (Codex and OpenCode + // have no flag at all), and the eval measures the agent as shipped. Refuse + // rather than drop, so a misconfigured experiment fails loudly. + if (args.systemPrompt) { + throw new Error( + `${runner.displayName} is a CLI agent and runs with its own system prompt; ` + + 'the harness must pass an empty one. Put task-specific instructions in the user prompt.' + ); + } + await runner.install(sandbox, version, apiKey); - // Stage the prompts into the sandbox scratch dir (outside the workspace). + // Stage the user prompt into the sandbox scratch dir (outside the workspace). await sandbox.exec(`mkdir -p ${SCRATCH}`); - await writeSandboxFile(sandbox, SYSTEM_PROMPT_PATH, args.systemPrompt); await writeSandboxFile(sandbox, USER_PROMPT_PATH, args.userPrompt); const start = Date.now(); @@ -98,7 +107,6 @@ export function createCliAgent( sandbox, model: options.model, apiKey, - systemPromptPath: SYSTEM_PROMPT_PATH, userPromptPath: USER_PROMPT_PATH, // Rewrite loopback hosts so in-container MCP servers can reach host-side // platform-lite; the runner writes them in its own config format. diff --git a/packages/core/src/agents/opencode/runner.test.ts b/packages/core/src/agents/opencode/runner.test.ts index 494942f1..b0528223 100644 --- a/packages/core/src/agents/opencode/runner.test.ts +++ b/packages/core/src/agents/opencode/runner.test.ts @@ -173,7 +173,6 @@ async function captureExec( }, model, apiKey: 'gw-key', - systemPromptPath: '/s', userPromptPath: '/u', mcpServers: opts.mcp ? { supabase: { command: 'srv' } } : {}, timeoutSec: 1, @@ -202,6 +201,14 @@ describe('opencode runner exec routing', () => { expect(config?.mcp).toEqual({}); expect(runCommand).toContain('OPENCODE_CONFIG='); }); + + it('sends the task alone as the message', async () => { + // opencode has no system-prompt flag, so anything else here would land on + // the user message. Nothing is prepended. + const { runCommand } = await captureExec('moonshotai/kimi-k3'); + expect(runCommand).toContain('"$(cat /u)"'); + expect(runCommand).not.toContain('system-prompt'); + }); }); describe('opencode runner extractStepCount', () => { diff --git a/packages/core/src/agents/opencode/runner.ts b/packages/core/src/agents/opencode/runner.ts index 6bd25bdd..27b41771 100644 --- a/packages/core/src/agents/opencode/runner.ts +++ b/packages/core/src/agents/opencode/runner.ts @@ -113,17 +113,15 @@ export function createOpencodeRunner( sandbox, model, apiKey, - systemPromptPath, userPromptPath, mcpServers, timeoutSec, }) { const opencode = npmGlobalBin('opencode'); - // opencode has no system-prompt flag, so prepend the system prompt to the - // task; both are staged files, joined via command substitution into the - // single message argument. - const message = `"$(cat ${systemPromptPath}; printf '\\n\\n'; cat ${userPromptPath})"`; + // The staged task file, read via command substitution into the single + // message argument. + const message = `"$(cat ${userPromptPath})"`; await sandbox.exec(`mkdir -p ${SCRATCH}`); await writeSandboxFile( diff --git a/packages/core/src/agents/shared.ts b/packages/core/src/agents/shared.ts index 2e2685a0..f9bd47dc 100644 --- a/packages/core/src/agents/shared.ts +++ b/packages/core/src/agents/shared.ts @@ -27,7 +27,6 @@ export function requireEnv(name: string, hint?: string): string { /** Scratch dir + staged files, outside the workspace so they're never scored. */ export const SCRATCH = '"$HOME/.eval"'; -export const SYSTEM_PROMPT_PATH = '"$HOME/.eval/system-prompt.txt"'; export const USER_PROMPT_PATH = '"$HOME/.eval/user-prompt.txt"'; /** npm prefix for the per-run global CLI install (outside the workspace). */ export const NPM_PREFIX = '"$HOME/.npm-global"'; diff --git a/packages/core/src/agents/types.ts b/packages/core/src/agents/types.ts index 89dc806e..1250e7d8 100644 --- a/packages/core/src/agents/types.ts +++ b/packages/core/src/agents/types.ts @@ -47,16 +47,15 @@ export interface AgentSandbox { /** * Arguments handed to a runner's `exec`. The generic layer has already staged - * the prompts into the sandbox (paths are shell expressions) and rewritten MCP - * server hosts for in-container reachability; the runner writes its own MCP - * config in whatever format/location the CLI expects. + * the user prompt into the sandbox (the path is a shell expression) and + * rewritten MCP server hosts for in-container reachability; the runner writes + * its own MCP config in whatever format/location the CLI expects. There is no + * system prompt: a CLI agent runs with the one it ships with. */ export interface RunnerExecArgs { sandbox: AgentSandbox; model: M; apiKey: string; - /** Shell path to a file holding the system prompt (skills + task framing). */ - systemPromptPath: string; /** Shell path to a file holding the user prompt (the task). */ userPromptPath: string; /** MCP servers to expose, already loopback-rewritten. Empty when none. */ diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 5f47464c..cb3a0160 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -391,6 +391,10 @@ export type LocalStackScorer = ( ) => Promise; export type AgentRunArgs = { + /** + * System prompt from the harness. Only `aiSdkAgent` takes one; a CLI agent + * runs with its own and `createCliAgent` refuses a non-empty value. + */ systemPrompt: string; userPrompt: string; tools?: ToolSet; @@ -453,8 +457,8 @@ export type SandboxMount = { export type LocalStackSessionArgs = { /** - * The agent harness this session serves. The prompt addendum builders take it - * but do not branch on it yet; a follow-up change gates their output on it. + * The agent harness this session serves. Only `ai-sdk` calls the session's + * in-process `tools`, so only it gets the prompt addendum describing them. */ agent: AgentHarnessId; /** Supabase CLI version this scenario requires, overriding the runtime default. */ @@ -548,6 +552,10 @@ export type LocalStackSession = { * the agent harness. */ mcpServers?: Record; + /** + * Text for the agent's system prompt. Must be empty for a CLI agent, which + * runs with its own prompt; `createCliAgent` throws otherwise. + */ promptAddendum?: string; scoringContext: LocalStackScoringContext; /** @@ -859,6 +867,11 @@ export type EvalSessionArgs = { export type EvalSession = { mcpServers: Record; + /** + * Text for the agent's system prompt, joined from the MCP servers' + * `promptAddendum`s. Must be empty for a CLI agent, which runs with its own + * prompt; `createCliAgent` throws otherwise. + */ promptAddendum?: string; scoringContext: ToolScoringContext; close(): Promise; diff --git a/packages/sandbox/src/bare-sandbox.ts b/packages/sandbox/src/bare-sandbox.ts index 47c4199c..fa889578 100644 --- a/packages/sandbox/src/bare-sandbox.ts +++ b/packages/sandbox/src/bare-sandbox.ts @@ -1,23 +1,17 @@ import type { - AgentHarnessId, AgentSandbox, SandboxMount, SkillSource, } from '@supabase-evals/core'; import { createAgentEnvironment } from './agent-environment.js'; import { toAgentSandbox } from './local-stack-runtime.js'; -import { buildSkillsPrompt } from './skills.js'; export interface BareSandboxHandle { sandbox: AgentSandbox; - /** Skills-discovery text to fold into the agent's system prompt. */ - promptAddendum: string; close(): Promise; } export interface BareSandboxOptions { - /** Harness driving this sandbox. */ - agent: AgentHarnessId; /** Supabase CLI version baked into the sandbox image. */ cliVersion?: string; /** Skills to install into the sandbox. */ @@ -37,7 +31,8 @@ export interface BareSandboxOptions { * stack — so the CLI agent has the same tools and the same skills in both modes. * * The eval's tools come from MCP (the in-container servers reach host-side - * platform-lite via `host.docker.internal` on the default bridge). + * platform-lite via `host.docker.internal` on the default bridge). The CLI + * agent discovers the installed skills itself; nothing is added to its prompt. */ export async function createBareSandbox( options: BareSandboxOptions @@ -49,7 +44,6 @@ export async function createBareSandbox( }); return { sandbox: toAgentSandbox(env.sandbox), - promptAddendum: buildSkillsPrompt(options.agent, env.skills), close: env.close, }; } diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index eb2b3192..bf979ee2 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -133,11 +133,16 @@ export function localStackRuntime( /** * Describes the session's tool surface: the binaries installed in the workspace * and the in-process `bash`/`files_*` tools from `buildLocalStackTools`. + * + * ai-sdk only. Those tools exist solely for `aiSdkAgent`; `createCliAgent` + * ignores `args.tools`, so a CLI agent works the same workspace through its own + * built-in tools and this text would name tools it does not have. */ export function buildToolSurfaceAddendum( agent: AgentHarnessId, options: { skipCliInstall?: boolean } = {} ): string { + if (agent !== 'ai-sdk') return ''; let addendum = 'docker, psql, git, and curl are installed in the workspace. ' + 'Use the bash tool to run commands (the working directory is always the workspace root) ' + diff --git a/packages/sandbox/src/skills.ts b/packages/sandbox/src/skills.ts index 809a891b..d11a29f5 100644 --- a/packages/sandbox/src/skills.ts +++ b/packages/sandbox/src/skills.ts @@ -15,9 +15,10 @@ * * Each CLI then discovers, advertises and loads the skills itself, in its own * words (Codex injects a `` block, OpenCode exposes a - * `skill` tool). `buildSkillsPrompt` renders the harness's own discovery listing - * on top of that, for the in-process `ai-sdk` agent — which has no such - * mechanism — and for the CLI harnesses. + * `skill` tool). The harness injects nothing on top: a second listing would + * duplicate the CLI's own and name a tool the agent does not have. The one + * exception is the in-process `ai-sdk` agent, which has no such mechanism — + * `buildSkillsPrompt` renders the listing for it alone. */ import { agentHarnessIdSchema } from '@supabase-evals/core'; @@ -82,9 +83,9 @@ export const SKILLS_INSTALL_AGENTS: readonly AgentHarnessId[] = installAgents; export const SKILLS_INSTALL_DIRS: readonly string[] = installDirs; /** - * The directory `buildSkillsPrompt`'s listing points at — it names one concrete - * path, read with the harness's own file tools. The CLI harnesses resolve their - * own paths natively. + * The directory the `ai-sdk` discovery listing points at — that agent reads + * SKILL.md with the harness's own file tools, so it needs one concrete path. The + * CLI harnesses resolve their own paths natively. */ export const SKILLS_INSTALL_DIR = CLAUDE_CODE_SKILLS_DIR; @@ -138,12 +139,16 @@ export function frontmatterDescription(markdown: string): string { * Render the skills discovery listing: only names+descriptions enter the system * prompt, keeping context lean. When a task matches, the agent reads that * skill's SKILL.md with the existing file tools (progressive disclosure). - * Empty when no skills are installed. + * + * ai-sdk only: `files_read` is an in-process tool only `aiSdkAgent` has, and + * every CLI harness discovers the installed skills itself (see the module + * comment). Empty for every CLI agent, and when no skills are installed. */ export function buildSkillsPrompt( agent: AgentHarnessId, skills: readonly SkillEntry[] ): string { + if (agent !== 'ai-sdk') return ''; if (skills.length === 0) return ''; return [ '## Available skills', diff --git a/packages/sandbox/test/unit.test.ts b/packages/sandbox/test/unit.test.ts index ba720794..bf12e96d 100644 --- a/packages/sandbox/test/unit.test.ts +++ b/packages/sandbox/test/unit.test.ts @@ -108,13 +108,12 @@ describe('buildSkillsPrompt', () => { expect(prompt).not.toContain('# Body'); }); - it('renders the same listing for every agent', () => { - // The listing is injected regardless of harness: each CLI also discovers - // the installed skills natively, so it hears about them twice. + it('is empty for every CLI agent, which discovers its own skills', () => { + // Claude Code, Codex and OpenCode each walk their own project scope and + // advertise the skills themselves; our listing would duplicate theirs and + // name `files_read`, an ai-sdk-only tool. for (const agent of ['claude-code', 'codex', 'opencode'] as const) { - expect(buildSkillsPrompt(agent, entries)).toBe( - buildSkillsPrompt('ai-sdk', entries) - ); + expect(buildSkillsPrompt(agent, entries)).toBe(''); expect(buildSkillsPrompt(agent, [])).toBe(''); } }); @@ -228,13 +227,13 @@ describe('buildToolSurfaceAddendum', () => { expect(addendum).toContain('docker, psql, git, and curl'); }); - it('describes the same tool surface for every agent', () => { + it('is empty for every CLI agent, which never sees these tools', () => { + // createCliAgent ignores `args.tools`; a CLI agent works the workspace with + // its own built-in tools, so naming ours would describe tools it lacks. for (const agent of ['claude-code', 'codex', 'opencode'] as const) { - expect(buildToolSurfaceAddendum(agent)).toBe( - buildToolSurfaceAddendum('ai-sdk') - ); + expect(buildToolSurfaceAddendum(agent)).toBe(''); expect(buildToolSurfaceAddendum(agent, { skipCliInstall: true })).toBe( - buildToolSurfaceAddendum('ai-sdk', { skipCliInstall: true }) + '' ); } });