From 3187aed2e314a0079f580ff849ccb02e81a52da5 Mon Sep 17 00:00:00 2001
From: Sean Oliver <882952+seanoliver@users.noreply.github.com>
Date: Tue, 8 Sep 2026 13:50:52 -0700
Subject: [PATCH 1/7] fix(core): CLI agents run with their own system prompt
 (AI-1034)

createCliAgent no longer stages a system prompt file or hands a path to the
runner. Claude Code drops --append-system-prompt-file; Codex and OpenCode
send the task alone instead of prepending a block to the user prompt. A
non-empty systemPrompt now throws rather than being dropped.
---
 .../src/agents/claude-code/runner.test.ts     | 29 +++++++++
 .../core/src/agents/claude-code/runner.ts     |  4 --
 packages/core/src/agents/codex/runner.test.ts | 38 +++++++++++
 packages/core/src/agents/codex/runner.ts      |  6 +-
 packages/core/src/agents/engine.test.ts       | 65 +++++++++++++++++++
 packages/core/src/agents/engine.ts            | 16 +++--
 .../core/src/agents/opencode/runner.test.ts   | 17 ++++-
 packages/core/src/agents/opencode/runner.ts   |  8 +--
 packages/core/src/agents/shared.ts            |  1 -
 packages/core/src/agents/types.ts             |  9 ++-
 packages/core/src/index.ts                    |  8 ++-
 11 files changed, 175 insertions(+), 26 deletions(-)
 create mode 100644 packages/core/src/agents/codex/runner.test.ts
 create mode 100644 packages/core/src/agents/engine.test.ts

diff --git a/packages/core/src/agents/claude-code/runner.test.ts b/packages/core/src/agents/claude-code/runner.test.ts
index 07450363..95e61dfb 100644
--- a/packages/core/src/agents/claude-code/runner.test.ts
+++ b/packages/core/src/agents/claude-code/runner.test.ts
@@ -36,6 +36,35 @@ function streamJson(subtype: string, isError = false): string {
   ].join('\n');
 }
 
+/** The `claude` invocation from one exec, with a fake sandbox. */
+async function captureRunCommand(): Promise<string> {
+  let runCommand = '';
+  await claudeCodeRunner.exec({
+    sandbox: {
+      workspace: '/w',
+      exec: async (cmd) => {
+        if (cmd.includes('/bin/claude')) runCommand = cmd;
+        return ok;
+      },
+      readFile: async () => '',
+    },
+    model: 'claude-sonnet-4-6',
+    apiKey: 'k',
+    userPromptPath: '"$HOME/.eval/user-prompt.txt"',
+    mcpServers: {},
+    timeoutSec: 1,
+  });
+  return runCommand;
+}
+
+describe('claudeCodeRunner.exec', () => {
+  it("pipes the task in and leaves Claude Code's own system prompt intact", async () => {
+    const command = await captureRunCommand();
+    expect(command).toContain('cat "$HOME/.eval/user-prompt.txt" |');
+    expect(command).not.toContain('system-prompt');
+  });
+});
+
 describe('claudeCodeRunner.deriveStopReason', () => {
   const derive = claudeCodeRunner.deriveStopReason!;
 
diff --git a/packages/core/src/agents/claude-code/runner.ts b/packages/core/src/agents/claude-code/runner.ts
index 37129ca1..1f7e99dc 100644
--- a/packages/core/src/agents/claude-code/runner.ts
+++ b/packages/core/src/agents/claude-code/runner.ts
@@ -40,7 +40,6 @@ export const claudeCodeRunner: AgentRunner<AnthropicModel> = {
     sandbox,
     model,
     apiKey,
-    systemPromptPath,
     userPromptPath,
     mcpServers,
     reasoningEffort,
@@ -68,9 +67,6 @@ export const claudeCodeRunner: AgentRunner<AnthropicModel> = {
       `--model ${shellQuote(model)}`,
       // Reasoning effort for the session; omitted leaves Claude Code's default.
       ...(reasoningEffort ? [`--effort ${shellQuote(reasoningEffort)}`] : []),
-      // Append (not replace), from a file (no ARG_MAX/shell-expansion surface),
-      // so Claude Code keeps its default coding-agent prompt + tool guidance.
-      `--append-system-prompt-file ${systemPromptPath}`,
       ...mcpFlags,
       // The sandbox is the isolation boundary, so skip permission prompts and
       // give the agent its full native toolset (same in both modes).
diff --git a/packages/core/src/agents/codex/runner.test.ts b/packages/core/src/agents/codex/runner.test.ts
new file mode 100644
index 00000000..3fae95c2
--- /dev/null
+++ b/packages/core/src/agents/codex/runner.test.ts
@@ -0,0 +1,38 @@
+import { describe, expect, it } from 'vitest';
+import type { CommandResult } from '../../index.js';
+import { codexRunner } from './runner.js';
+
+const ok: CommandResult = { ok: true, exitCode: 0, stdout: '', stderr: '' };
+
+/** The `codex exec` invocation from one exec, with a fake sandbox. */
+async function captureRunCommand(): Promise<string> {
+  let runCommand = '';
+  await codexRunner.exec({
+    sandbox: {
+      workspace: '/w',
+      exec: async (cmd) => {
+        if (cmd.includes(' exec ')) runCommand = cmd;
+        return ok;
+      },
+      readFile: async () => '',
+    },
+    model: 'gpt-5.4',
+    apiKey: 'k',
+    userPromptPath: '"$HOME/.eval/user-prompt.txt"',
+    mcpServers: {},
+    timeoutSec: 1,
+  });
+  return runCommand;
+}
+
+describe('codexRunner.exec', () => {
+  it('sends the task alone on stdin', async () => {
+    // Codex has no system-prompt flag, so anything else here would land on the
+    // user prompt. Nothing is prepended.
+    const command = await captureRunCommand();
+    expect(command.startsWith('cat "$HOME/.eval/user-prompt.txt" |')).toBe(
+      true
+    );
+    expect(command).not.toContain('system-prompt');
+  });
+});
diff --git a/packages/core/src/agents/codex/runner.ts b/packages/core/src/agents/codex/runner.ts
index f4e3837e..af1fc861 100644
--- a/packages/core/src/agents/codex/runner.ts
+++ b/packages/core/src/agents/codex/runner.ts
@@ -58,7 +58,6 @@ export const codexRunner: AgentRunner<CodexModel> = {
     sandbox,
     model,
     apiKey,
-    systemPromptPath,
     userPromptPath,
     mcpServers,
     reasoningEffort,
@@ -91,10 +90,9 @@ export const codexRunner: AgentRunner<CodexModel> = {
       '-',
     ].join(' ');
 
-    // Codex has no system-prompt flag; prepend the system prompt to the task,
-    // both staged as files, fed on stdin.
+    // The staged task file, fed on stdin.
     const command = await sandbox.exec(
-      `{ cat ${systemPromptPath}; printf '\\n\\n'; cat ${userPromptPath}; } | ${codex} ${flags}`,
+      `cat ${userPromptPath} | ${codex} ${flags}`,
       { timeoutMs: timeoutSec * 1000, env: { OPENAI_API_KEY: apiKey } }
     );
     return { command, raw: command.stdout };
diff --git a/packages/core/src/agents/engine.test.ts b/packages/core/src/agents/engine.test.ts
new file mode 100644
index 00000000..37f7e7d6
--- /dev/null
+++ b/packages/core/src/agents/engine.test.ts
@@ -0,0 +1,65 @@
+import { beforeEach, describe, expect, it } from 'vitest';
+import type { CommandResult } from '../index.js';
+import type { AgentTranscriptParser } from '../parsers/types.js';
+import { createCliAgent } from './engine.js';
+import { USER_PROMPT_PATH } from './shared.js';
+import type { AgentRunner } from './types.js';
+
+const ok: CommandResult = { ok: true, exitCode: 0, stdout: '', stderr: '' };
+
+const API_KEY_ENV_VAR = 'ENGINE_TEST_API_KEY';
+
+/** A parser that reports one assistant message, so the engine stays quiet. */
+const parser: AgentTranscriptParser = {
+  parseTranscript: () => ({
+    events: [{ type: 'message', role: 'assistant', content: 'done' }],
+  }),
+};
+
+/** Run a CLI agent against a fake sandbox, returning every command it ran. */
+async function runWithSystemPrompt(systemPrompt: string): Promise<string[]> {
+  const commands: string[] = [];
+  const runner: AgentRunner = {
+    id: 'claude-code',
+    displayName: 'Fake CLI',
+    apiKeyEnvVar: API_KEY_ENV_VAR,
+    cliPackage: 'fake-cli',
+    defaultCliVersion: '1.0.0',
+    defaultModel: 'fake-model',
+    install: async () => undefined,
+    exec: async () => ({ command: ok, raw: '' }),
+  };
+  await createCliAgent(runner, parser, { model: 'fake-model' }).run({
+    systemPrompt,
+    userPrompt: 'the task',
+    timeoutSec: 1,
+    sandbox: {
+      workspace: '/w',
+      exec: async (command) => {
+        commands.push(command);
+        return ok;
+      },
+      readFile: async () => '',
+    },
+  });
+  return commands;
+}
+
+describe('createCliAgent prompt staging', () => {
+  // The engine requires the runner's API key before it stages anything.
+  beforeEach(() => {
+    process.env[API_KEY_ENV_VAR] = 'k';
+  });
+
+  it('stages only the user prompt', async () => {
+    const commands = await runWithSystemPrompt('');
+    expect(commands.some((c) => c.includes(USER_PROMPT_PATH))).toBe(true);
+    expect(commands.some((c) => c.includes('system-prompt'))).toBe(false);
+  });
+
+  it('refuses a system prompt: a CLI agent runs with its own', async () => {
+    await expect(runWithSystemPrompt('Extra framing.')).rejects.toThrow(
+      /runs with its own system prompt/
+    );
+  });
+});
diff --git a/packages/core/src/agents/engine.ts b/packages/core/src/agents/engine.ts
index 41035a46..bb031842 100644
--- a/packages/core/src/agents/engine.ts
+++ b/packages/core/src/agents/engine.ts
@@ -26,7 +26,6 @@ import type { AgentTranscriptParser } from '../parsers/types.js';
 import type { AgentRunner } from './types.js';
 import {
   SCRATCH,
-  SYSTEM_PROMPT_PATH,
   USER_PROMPT_PATH,
   processStopReason,
   requireEnv,
@@ -86,18 +85,27 @@ export function createCliAgent<M extends string = string>(
         );
       }
 
+      // A CLI agent runs with the system prompt it ships with; the harness has
+      // no way to add to it that is the same across CLIs (Codex and OpenCode
+      // have no flag at all), and the eval measures the agent as shipped. Refuse
+      // rather than drop, so a misconfigured experiment fails loudly.
+      if (args.systemPrompt) {
+        throw new Error(
+          `${runner.displayName} is a CLI agent and runs with its own system prompt; ` +
+            'the harness must pass an empty one. Put task-specific instructions in the user prompt.'
+        );
+      }
+
       await runner.install(sandbox, version, apiKey);
 
-      // Stage the prompts into the sandbox scratch dir (outside the workspace).
+      // Stage the user prompt into the sandbox scratch dir (outside the workspace).
       await sandbox.exec(`mkdir -p ${SCRATCH}`);
-      await writeSandboxFile(sandbox, SYSTEM_PROMPT_PATH, args.systemPrompt);
       await writeSandboxFile(sandbox, USER_PROMPT_PATH, args.userPrompt);
 
       const { command, raw } = await runner.exec({
         sandbox,
         model: options.model,
         apiKey,
-        systemPromptPath: SYSTEM_PROMPT_PATH,
         userPromptPath: USER_PROMPT_PATH,
         // Rewrite loopback hosts so in-container MCP servers can reach host-side
         // platform-lite; the runner writes them in its own config format.
diff --git a/packages/core/src/agents/opencode/runner.test.ts b/packages/core/src/agents/opencode/runner.test.ts
index 13a5b3af..a2e3f14d 100644
--- a/packages/core/src/agents/opencode/runner.test.ts
+++ b/packages/core/src/agents/opencode/runner.test.ts
@@ -113,7 +113,6 @@ async function captureExec(
     },
     model,
     apiKey: 'gw-key',
-    systemPromptPath: '/s',
     userPromptPath: '/u',
     mcpServers: opts.mcp ? { supabase: { command: 'srv' } } : {},
     timeoutSec: 1,
@@ -139,4 +138,20 @@ describe('opencode runner exec routing', () => {
     expect(config?.mcp).toEqual({});
     expect(runCommand).toContain('OPENCODE_CONFIG=');
   });
+
+  it('sends the task alone as the message', async () => {
+    // opencode has no system-prompt flag, so anything else here would land on
+    // the user message. Nothing is prepended.
+    const { runCommand } = await captureExec('moonshotai/kimi-k3');
+    expect(runCommand).toContain('"$(cat /u)"');
+    expect(runCommand).not.toContain('system-prompt');
+  });
+
+  it('sends the task alone as the message', async () => {
+    // opencode has no system-prompt flag, so anything else here would land on
+    // the user message. Nothing is prepended.
+    const { runCommand } = await captureExec('moonshotai/kimi-k3');
+    expect(runCommand).toContain('"$(cat /u)"');
+    expect(runCommand).not.toContain('system-prompt');
+  });
 });
diff --git a/packages/core/src/agents/opencode/runner.ts b/packages/core/src/agents/opencode/runner.ts
index b7b437d9..53b0c40c 100644
--- a/packages/core/src/agents/opencode/runner.ts
+++ b/packages/core/src/agents/opencode/runner.ts
@@ -113,17 +113,15 @@ export function createOpencodeRunner(
       sandbox,
       model,
       apiKey,
-      systemPromptPath,
       userPromptPath,
       mcpServers,
       timeoutSec,
     }) {
       const opencode = npmGlobalBin('opencode');
 
-      // opencode has no system-prompt flag, so prepend the system prompt to the
-      // task; both are staged files, joined via command substitution into the
-      // single message argument.
-      const message = `"$(cat ${systemPromptPath}; printf '\\n\\n'; cat ${userPromptPath})"`;
+      // The staged task file, read via command substitution into the single
+      // message argument.
+      const message = `"$(cat ${userPromptPath})"`;
 
       await sandbox.exec(`mkdir -p ${SCRATCH}`);
       await writeSandboxFile(
diff --git a/packages/core/src/agents/shared.ts b/packages/core/src/agents/shared.ts
index 2e2685a0..f9bd47dc 100644
--- a/packages/core/src/agents/shared.ts
+++ b/packages/core/src/agents/shared.ts
@@ -27,7 +27,6 @@ export function requireEnv(name: string, hint?: string): string {
 
 /** Scratch dir + staged files, outside the workspace so they're never scored. */
 export const SCRATCH = '"$HOME/.eval"';
-export const SYSTEM_PROMPT_PATH = '"$HOME/.eval/system-prompt.txt"';
 export const USER_PROMPT_PATH = '"$HOME/.eval/user-prompt.txt"';
 /** npm prefix for the per-run global CLI install (outside the workspace). */
 export const NPM_PREFIX = '"$HOME/.npm-global"';
diff --git a/packages/core/src/agents/types.ts b/packages/core/src/agents/types.ts
index baf91962..ebf1def7 100644
--- a/packages/core/src/agents/types.ts
+++ b/packages/core/src/agents/types.ts
@@ -46,16 +46,15 @@ export interface AgentSandbox {
 
 /**
  * Arguments handed to a runner's `exec`. The generic layer has already staged
- * the prompts into the sandbox (paths are shell expressions) and rewritten MCP
- * server hosts for in-container reachability; the runner writes its own MCP
- * config in whatever format/location the CLI expects.
+ * the user prompt into the sandbox (the path is a shell expression) and
+ * rewritten MCP server hosts for in-container reachability; the runner writes
+ * its own MCP config in whatever format/location the CLI expects. There is no
+ * system prompt: a CLI agent runs with the one it ships with.
  */
 export interface RunnerExecArgs<M extends string = string> {
   sandbox: AgentSandbox;
   model: M;
   apiKey: string;
-  /** Shell path to a file holding the system prompt (skills + task framing). */
-  systemPromptPath: string;
   /** Shell path to a file holding the user prompt (the task). */
   userPromptPath: string;
   /** MCP servers to expose, already loopback-rewritten. Empty when none. */
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 968707fe..9e26f224 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -371,6 +371,10 @@ export type LocalStackScorer = (
 ) => Promise<ScoreResult>;
 
 export type AgentRunArgs = {
+  /**
+   * System prompt from the harness. Only `aiSdkAgent` takes one; a CLI agent
+   * runs with its own and `createCliAgent` refuses a non-empty value.
+   */
   systemPrompt: string;
   userPrompt: string;
   tools?: ToolSet;
@@ -431,8 +435,8 @@ export type SandboxMount = {
 
 export type LocalStackSessionArgs = {
   /**
-   * The agent harness this session serves. The prompt addendum builders take it
-   * but do not branch on it yet; a follow-up change gates their output on it.
+   * The agent harness this session serves. Only `ai-sdk` calls the session's
+   * in-process `tools`, so only it gets the prompt addendum describing them.
    */
   agent: AgentHarnessId;
   /** Supabase CLI version this scenario requires, overriding the runtime default. */

From d1fd64cebd0402fda37e11587590c8de4dcb66b9 Mon Sep 17 00:00:00 2001
From: Sean Oliver <882952+seanoliver@users.noreply.github.com>
Date: Tue, 8 Sep 2026 13:50:52 -0700
Subject: [PATCH 2/7] fix(sandbox): render the injected prompt blocks for
 ai-sdk only (AI-1034)

buildToolSurfaceAddendum and buildSkillsPrompt return '' for every CLI
harness: those describe in-process tools only aiSdkAgent has, and each CLI
discovers the installed skills itself. The bare sandbox no longer takes an
agent or returns a promptAddendum.
---
 packages/sandbox/src/bare-sandbox.ts        | 10 ++--------
 packages/sandbox/src/local-stack-runtime.ts |  5 +++++
 packages/sandbox/src/skills.ts              | 19 ++++++++++++-------
 packages/sandbox/test/unit.test.ts          | 21 ++++++++++-----------
 4 files changed, 29 insertions(+), 26 deletions(-)

diff --git a/packages/sandbox/src/bare-sandbox.ts b/packages/sandbox/src/bare-sandbox.ts
index 47c4199c..fa889578 100644
--- a/packages/sandbox/src/bare-sandbox.ts
+++ b/packages/sandbox/src/bare-sandbox.ts
@@ -1,23 +1,17 @@
 import type {
-  AgentHarnessId,
   AgentSandbox,
   SandboxMount,
   SkillSource,
 } from '@supabase-evals/core';
 import { createAgentEnvironment } from './agent-environment.js';
 import { toAgentSandbox } from './local-stack-runtime.js';
-import { buildSkillsPrompt } from './skills.js';
 
 export interface BareSandboxHandle {
   sandbox: AgentSandbox;
-  /** Skills-discovery text to fold into the agent's system prompt. */
-  promptAddendum: string;
   close(): Promise<void>;
 }
 
 export interface BareSandboxOptions {
-  /** Harness driving this sandbox. */
-  agent: AgentHarnessId;
   /** Supabase CLI version baked into the sandbox image. */
   cliVersion?: string;
   /** Skills to install into the sandbox. */
@@ -37,7 +31,8 @@ export interface BareSandboxOptions {
  * stack — so the CLI agent has the same tools and the same skills in both modes.
  *
  * The eval's tools come from MCP (the in-container servers reach host-side
- * platform-lite via `host.docker.internal` on the default bridge).
+ * platform-lite via `host.docker.internal` on the default bridge). The CLI
+ * agent discovers the installed skills itself; nothing is added to its prompt.
  */
 export async function createBareSandbox(
   options: BareSandboxOptions
@@ -49,7 +44,6 @@ export async function createBareSandbox(
   });
   return {
     sandbox: toAgentSandbox(env.sandbox),
-    promptAddendum: buildSkillsPrompt(options.agent, env.skills),
     close: env.close,
   };
 }
diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts
index c38d01a6..d3c6f57f 100644
--- a/packages/sandbox/src/local-stack-runtime.ts
+++ b/packages/sandbox/src/local-stack-runtime.ts
@@ -132,11 +132,16 @@ export function localStackRuntime(
 /**
  * Describes the session's tool surface: the binaries installed in the workspace
  * and the in-process `bash`/`files_*` tools from `buildLocalStackTools`.
+ *
+ * ai-sdk only. Those tools exist solely for `aiSdkAgent`; `createCliAgent`
+ * ignores `args.tools`, so a CLI agent works the same workspace through its own
+ * built-in tools and this text would name tools it does not have.
  */
 export function buildToolSurfaceAddendum(
   agent: AgentHarnessId,
   options: { skipCliInstall?: boolean } = {}
 ): string {
+  if (agent !== 'ai-sdk') return '';
   let addendum =
     'docker, psql, git, and curl are installed in the workspace. ' +
     'Use the bash tool to run commands (the working directory is always the workspace root) ' +
diff --git a/packages/sandbox/src/skills.ts b/packages/sandbox/src/skills.ts
index 809a891b..d11a29f5 100644
--- a/packages/sandbox/src/skills.ts
+++ b/packages/sandbox/src/skills.ts
@@ -15,9 +15,10 @@
  *
  * Each CLI then discovers, advertises and loads the skills itself, in its own
  * words (Codex injects a `<skills_instructions>` block, OpenCode exposes a
- * `skill` tool). `buildSkillsPrompt` renders the harness's own discovery listing
- * on top of that, for the in-process `ai-sdk` agent — which has no such
- * mechanism — and for the CLI harnesses.
+ * `skill` tool). The harness injects nothing on top: a second listing would
+ * duplicate the CLI's own and name a tool the agent does not have. The one
+ * exception is the in-process `ai-sdk` agent, which has no such mechanism —
+ * `buildSkillsPrompt` renders the listing for it alone.
  */
 
 import { agentHarnessIdSchema } from '@supabase-evals/core';
@@ -82,9 +83,9 @@ export const SKILLS_INSTALL_AGENTS: readonly AgentHarnessId[] = installAgents;
 export const SKILLS_INSTALL_DIRS: readonly string[] = installDirs;
 
 /**
- * The directory `buildSkillsPrompt`'s listing points at — it names one concrete
- * path, read with the harness's own file tools. The CLI harnesses resolve their
- * own paths natively.
+ * The directory the `ai-sdk` discovery listing points at — that agent reads
+ * SKILL.md with the harness's own file tools, so it needs one concrete path. The
+ * CLI harnesses resolve their own paths natively.
  */
 export const SKILLS_INSTALL_DIR = CLAUDE_CODE_SKILLS_DIR;
 
@@ -138,12 +139,16 @@ export function frontmatterDescription(markdown: string): string {
  * Render the skills discovery listing: only names+descriptions enter the system
  * prompt, keeping context lean. When a task matches, the agent reads that
  * skill's SKILL.md with the existing file tools (progressive disclosure).
- * Empty when no skills are installed.
+ *
+ * ai-sdk only: `files_read` is an in-process tool only `aiSdkAgent` has, and
+ * every CLI harness discovers the installed skills itself (see the module
+ * comment). Empty for every CLI agent, and when no skills are installed.
  */
 export function buildSkillsPrompt(
   agent: AgentHarnessId,
   skills: readonly SkillEntry[]
 ): string {
+  if (agent !== 'ai-sdk') return '';
   if (skills.length === 0) return '';
   return [
     '## Available skills',
diff --git a/packages/sandbox/test/unit.test.ts b/packages/sandbox/test/unit.test.ts
index ba720794..bf12e96d 100644
--- a/packages/sandbox/test/unit.test.ts
+++ b/packages/sandbox/test/unit.test.ts
@@ -108,13 +108,12 @@ describe('buildSkillsPrompt', () => {
     expect(prompt).not.toContain('# Body');
   });
 
-  it('renders the same listing for every agent', () => {
-    // The listing is injected regardless of harness: each CLI also discovers
-    // the installed skills natively, so it hears about them twice.
+  it('is empty for every CLI agent, which discovers its own skills', () => {
+    // Claude Code, Codex and OpenCode each walk their own project scope and
+    // advertise the skills themselves; our listing would duplicate theirs and
+    // name `files_read`, an ai-sdk-only tool.
     for (const agent of ['claude-code', 'codex', 'opencode'] as const) {
-      expect(buildSkillsPrompt(agent, entries)).toBe(
-        buildSkillsPrompt('ai-sdk', entries)
-      );
+      expect(buildSkillsPrompt(agent, entries)).toBe('');
       expect(buildSkillsPrompt(agent, [])).toBe('');
     }
   });
@@ -228,13 +227,13 @@ describe('buildToolSurfaceAddendum', () => {
     expect(addendum).toContain('docker, psql, git, and curl');
   });
 
-  it('describes the same tool surface for every agent', () => {
+  it('is empty for every CLI agent, which never sees these tools', () => {
+    // createCliAgent ignores `args.tools`; a CLI agent works the workspace with
+    // its own built-in tools, so naming ours would describe tools it lacks.
     for (const agent of ['claude-code', 'codex', 'opencode'] as const) {
-      expect(buildToolSurfaceAddendum(agent)).toBe(
-        buildToolSurfaceAddendum('ai-sdk')
-      );
+      expect(buildToolSurfaceAddendum(agent)).toBe('');
       expect(buildToolSurfaceAddendum(agent, { skipCliInstall: true })).toBe(
-        buildToolSurfaceAddendum('ai-sdk', { skipCliInstall: true })
+        ''
       );
     }
   });

From 3cefa603a379210ccf16a9b2ea920d064ce865c3 Mon Sep 17 00:00:00 2001
From: Sean Oliver <882952+seanoliver@users.noreply.github.com>
Date: Tue, 8 Sep 2026 13:50:52 -0700
Subject: [PATCH 3/7] fix(framework): neutral base prompt for ai-sdk, nothing
 for CLI agents (AI-1034)

Prompt assembly moves to system-prompt.ts so it can be unit tested. The
ai-sdk base prompt no longer names Supabase, a project, or how to end the
turn (#164), and is the same in both modes. CLI agents get an empty
prompt. Each run records the system prompt it sent.
---
 README.md                                    |  6 +-
 apps/framework/harness/run-eval.ts           | 52 +++++--------
 apps/framework/harness/system-prompt.test.ts | 79 ++++++++++++++++++++
 apps/framework/harness/system-prompt.ts      | 35 +++++++++
 apps/framework/package.json                  |  3 +-
 5 files changed, 137 insertions(+), 38 deletions(-)
 create mode 100644 apps/framework/harness/system-prompt.test.ts
 create mode 100644 apps/framework/harness/system-prompt.ts

diff --git a/README.md b/README.md
index 27af033a..1c919b9e 100644
--- a/README.md
+++ b/README.md
@@ -161,10 +161,10 @@ Skills come from [`supabase/agent-skills`](https://github.com/supabase/agent-ski
 
 To use a skill in an experiment, reference its directory name in the experiment's `skills` array.
 
-Both runtimes load skills lazily ([progressive disclosure](https://ai-sdk.dev/cookbook/guides/agent-skills)): only each skill's name+description is in the system prompt, and the agent pulls a skill's full instructions on demand. They differ only in how the body is fetched, because the tools-mode agent has no filesystem:
+Skills are always loaded lazily ([progressive disclosure](https://ai-sdk.dev/cookbook/guides/agent-skills)): a skill's full instructions are pulled on demand, never preloaded. How that happens depends on the harness:
 
-- **Local-stack (sandbox) mode:** skills are installed into the workspace with [Vercel's `skills` CLI](https://github.com/vercel-labs/skills) (baked into the sandbox image, sourced from the local `skills/` directory — never the network), into every project scope the CLI harnesses discover natively: `.claude/skills/` for Claude Code, `.agents/skills/` for Codex and OpenCode. Each CLI discovers the scope it reads and surfaces those skills to the model itself. The framework also still injects a listing naming them, so a CLI harness hears about them twice; removing that is a separate change.
-- **Tools mode:** no filesystem, so a `load_skill` tool returns a skill's full instructions when the agent calls it with the skill's name.
+- **CLI harnesses (Claude Code, Codex, OpenCode)** use their own built-in skills mechanism. Skills are installed into the sandbox workspace with [Vercel's `skills` CLI](https://github.com/vercel-labs/skills) (baked into the sandbox image, sourced from the local `skills/` directory — never the network), into each harness's own project scope: `.claude/skills/` for Claude Code, `.agents/skills/` for Codex and OpenCode. Each CLI then discovers, advertises and loads the skills itself. The framework adds nothing to a CLI agent's system prompt: how the agent behaves as shipped is part of what an eval measures.
+- **The in-process `ai-sdk` harness** has no such mechanism, so the framework supplies one. In local-stack mode it lists each skill's name+description in the system prompt and the agent reads `.claude/skills/<name>/SKILL.md` with its file tools. In tools mode there is no filesystem, so a `load_skill` tool returns a skill's full instructions when the agent calls it with the skill's name.
 
 ## Framework Checks
 
diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts
index 29dfc384..5f440ba2 100644
--- a/apps/framework/harness/run-eval.ts
+++ b/apps/framework/harness/run-eval.ts
@@ -29,6 +29,7 @@ import {
 } from '../lib/cli-args.js';
 import { bootPlatformBackend } from './platform-backend.js';
 import { viteBuild, vitestRun } from './project-runner.js';
+import { buildSystemPrompt } from './system-prompt.js';
 import {
   buildDocsResult,
   buildSkillResult,
@@ -318,31 +319,6 @@ function readSessionSeedArgs(ev: EvalManifest) {
   };
 }
 
-function basePromptFor(mode: EvalMode): string {
-  if (mode === 'local-stack') {
-    return (
-      'You are an agent solving a Supabase eval task in a Linux workspace. ' +
-      'Use the provided tools to inspect and modify the workspace and run commands. ' +
-      'When you are done, end your turn with a short summary of what you did.'
-    );
-  }
-  return (
-    'You are an agent solving a Supabase eval task. ' +
-    'Use the provided tools to inspect and modify the project. ' +
-    'When you are done, end your turn with a short summary of what you did ' +
-    '(or for audit tasks, your findings).'
-  );
-}
-
-function buildSystemPrompt(
-  mode: EvalMode,
-  addendum?: string,
-  skillContext?: string
-): string {
-  const blocks = [basePromptFor(mode), addendum, skillContext].filter(Boolean);
-  return blocks.join('\n\n');
-}
-
 /**
  * Adapt a `{ close() }` resource to `AsyncDisposable` so it can be bound with
  * `await using` — cleanup then runs on scope exit (normal fall-through, `continue`,
@@ -373,6 +349,11 @@ async function runOne(
     transcript: TranscriptPart[];
     agentReport: string;
     stoppedReason: string;
+    /**
+     * The system prompt the harness handed the agent, `''` for a CLI agent.
+     * Recorded so a run artifact shows what the agent was told.
+     */
+    systemPrompt: string;
   }
 > {
   const prompt = parseEvalMarkdown(
@@ -442,8 +423,12 @@ async function runOne(
       })
     );
 
+    const systemPrompt = buildSystemPrompt(
+      exp.agent.id,
+      session.promptAddendum
+    );
     const run = await exp.agent.run({
-      systemPrompt: buildSystemPrompt('local-stack', session.promptAddendum),
+      systemPrompt,
       userPrompt: prompt,
       tools: session.tools,
       sandbox: session.sandbox,
@@ -487,6 +472,7 @@ async function runOne(
       transcript: run.transcript,
       agentReport: run.agentReport,
       stoppedReason: run.stoppedReason,
+      systemPrompt,
     };
   }
 
@@ -495,7 +481,6 @@ async function runOne(
   await using cliSandbox = agentRunsInSandbox
     ? disposable(
         await createBareSandbox({
-          agent: exp.agent.id,
           skills: skillSources,
           mounts: supabaseMcpServerMounts(),
         })
@@ -508,15 +493,13 @@ async function runOne(
     })
   );
 
-  // In-process agents have no filesystem, so skills are advertised in the
-  // prompt and pulled on demand via the load_skill tool instead.
-  const skillsPrompt = agentRunsInSandbox
-    ? cliSandbox!.promptAddendum
-    : buildToolsSkillsPrompt(toolsSkills);
+  // A CLI agent discovers its installed skills itself. An in-process agent has
+  // no filesystem, so its skills are advertised in the prompt and pulled on
+  // demand via the load_skill tool instead.
   const systemPrompt = buildSystemPrompt(
-    'tools',
+    exp.agent.id,
     session.promptAddendum,
-    skillsPrompt
+    agentRunsInSandbox ? undefined : buildToolsSkillsPrompt(toolsSkills)
   );
   const run = await exp.agent.run({
     systemPrompt,
@@ -546,6 +529,7 @@ async function runOne(
     transcript: run.transcript,
     agentReport: run.agentReport,
     stoppedReason: run.stoppedReason,
+    systemPrompt,
   };
 }
 
diff --git a/apps/framework/harness/system-prompt.test.ts b/apps/framework/harness/system-prompt.test.ts
new file mode 100644
index 00000000..962e0c8f
--- /dev/null
+++ b/apps/framework/harness/system-prompt.test.ts
@@ -0,0 +1,79 @@
+import { describe, expect, it } from 'vitest';
+import type { AgentHarnessId } from '@supabase-evals/core';
+import {
+  buildSkillsPrompt,
+  buildToolSurfaceAddendum,
+  type SkillEntry,
+} from '@supabase-evals/sandbox';
+import { buildSystemPrompt } from './system-prompt.js';
+
+const CLI_AGENTS: AgentHarnessId[] = ['claude-code', 'codex', 'opencode'];
+
+const skills: SkillEntry[] = [
+  {
+    name: 'supabase',
+    description: 'Use for Supabase tasks.',
+    dir: '.claude/skills/supabase',
+  },
+];
+
+describe('buildSystemPrompt', () => {
+  it('gives the ai-sdk agent task framing', () => {
+    // ai-sdk is the one harness with no system prompt of its own.
+    expect(buildSystemPrompt('ai-sdk')).toContain('Use the provided tools');
+  });
+
+  it('gives nothing to any CLI harness', () => {
+    // CLI harnesses ship their own system prompt; that is what is measured.
+    for (const agent of CLI_AGENTS) {
+      expect(buildSystemPrompt(agent)).toBe('');
+    }
+  });
+
+  it('assembles to nothing for a CLI harness, even with skills installed', () => {
+    // The real block producers, not stand-ins: each CLI finds the skills through
+    // its own project-scope discovery and describes them to the model itself.
+    for (const agent of CLI_AGENTS) {
+      expect(
+        buildSystemPrompt(
+          agent,
+          buildToolSurfaceAddendum(agent),
+          buildSkillsPrompt(agent, skills)
+        )
+      ).toBe('');
+    }
+  });
+
+  it('keeps the runtime blocks for ai-sdk, in order, after the base prompt', () => {
+    const base = buildSystemPrompt('ai-sdk');
+    expect(buildSystemPrompt('ai-sdk', 'Addendum.', 'Skills listing.')).toBe(
+      `${base}\n\nAddendum.\n\nSkills listing.`
+    );
+    const withSkills = buildSystemPrompt(
+      'ai-sdk',
+      buildToolSurfaceAddendum('ai-sdk'),
+      buildSkillsPrompt('ai-sdk', skills)
+    );
+    expect(withSkills).toContain('## Available skills');
+    expect(withSkills).toContain('- supabase: Use for Supabase tasks.');
+  });
+
+  it('drops empty blocks instead of leaving blank gaps', () => {
+    const base = buildSystemPrompt('ai-sdk');
+    expect(buildSystemPrompt('ai-sdk', '', 'Skills listing.')).toBe(
+      `${base}\n\nSkills listing.`
+    );
+    expect(buildSystemPrompt('ai-sdk', '', '')).toBe(base);
+  });
+
+  it('never names Supabase, a project, or how to end the turn', () => {
+    // Issue #164: naming Supabase hands the agent the answer to "which tool";
+    // "project" presumes there is one to modify; stopping coaching shapes the
+    // report the judge reads. All three are part of what is measured.
+    const prompt = buildSystemPrompt('ai-sdk');
+    expect(prompt).not.toMatch(/supabase/i);
+    expect(prompt).not.toMatch(/project/i);
+    expect(prompt).not.toMatch(/eval/i);
+    expect(prompt).not.toMatch(/summary|end your turn/i);
+  });
+});
diff --git a/apps/framework/harness/system-prompt.ts b/apps/framework/harness/system-prompt.ts
new file mode 100644
index 00000000..bb49af9e
--- /dev/null
+++ b/apps/framework/harness/system-prompt.ts
@@ -0,0 +1,35 @@
+/**
+ * System-prompt assembly, per agent harness.
+ *
+ * An eval measures the agent as shipped, so the harness adds as little prompt
+ * of its own as it can. Only the ai-sdk agent gets any: it is the one harness
+ * with no system prompt of its own (`aiSdkAgent` hands `systemPrompt` straight
+ * to the model's `system`). Every CLI agent runs with the prompt it ships with,
+ * and `createCliAgent` refuses a non-empty one.
+ */
+
+import type { AgentHarnessId } from '@supabase-evals/core';
+
+/**
+ * Base framing for the ai-sdk harness: only what it cannot infer on its own,
+ * that it has tools and should use them. Deliberately says nothing about
+ * Supabase, a project, or how to finish a turn: which tools the agent reaches
+ * for and when it stops are part of what is measured (see issue #164).
+ */
+const AI_SDK_BASE_PROMPT =
+  'You are an agent. Use the provided tools to complete the task.';
+
+/**
+ * Assemble the system prompt for the agent. Every block is ai-sdk-only, so a
+ * CLI harness gets `''`. The runtime blocks (tool surface, skills listing) are
+ * already empty for a CLI agent at their source; an MCP server's
+ * `promptAddendum` is not, and is left to reach `createCliAgent`, which throws.
+ */
+export function buildSystemPrompt(
+  agent: AgentHarnessId,
+  addendum?: string,
+  skillContext?: string
+): string {
+  const base = agent === 'ai-sdk' ? AI_SDK_BASE_PROMPT : '';
+  return [base, addendum, skillContext].filter(Boolean).join('\n\n');
+}
diff --git a/apps/framework/package.json b/apps/framework/package.json
index d5554504..7f73e481 100644
--- a/apps/framework/package.json
+++ b/apps/framework/package.json
@@ -4,12 +4,13 @@
   "version": "0.0.1",
   "type": "module",
   "scripts": {
-    "check": "pnpm typecheck && pnpm test:framework && pnpm test:vercel-runner",
+    "check": "pnpm typecheck && pnpm test && pnpm test:framework && pnpm test:vercel-runner",
     "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts",
     "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry",
     "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke",
     "eval:vercel": "node --env-file=../../.env --import tsx/esm scripts/run-vercel-evals.ts",
     "typecheck": "tsc --noEmit",
+    "test": "vitest run harness",
     "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts",
     "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts lib/cli-args.test.ts lib/sample-sets.test.ts",
     "export-results": "node --import tsx/esm scripts/export-results.ts",

From 17c738c48f6945e6ec20de7349ca4f44119c80ec Mon Sep 17 00:00:00 2001
From: Sean Oliver <882952+seanoliver@users.noreply.github.com>
Date: Tue, 8 Sep 2026 14:01:11 -0700
Subject: [PATCH 4/7] test(core): restore the usage tests after the merge

---
 packages/core/src/agents/codex/runner.test.ts | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/core/src/agents/codex/runner.test.ts b/packages/core/src/agents/codex/runner.test.ts
index f76dbd27..be1b8bd7 100644
--- a/packages/core/src/agents/codex/runner.test.ts
+++ b/packages/core/src/agents/codex/runner.test.ts
@@ -34,6 +34,8 @@ describe('codexRunner.exec', () => {
       true
     );
     expect(command).not.toContain('system-prompt');
+  });
+});
 
 describe('codexRunner.extractUsage', () => {
   const extract = codexRunner.extractUsage!;
@@ -90,3 +92,5 @@ describe('codexRunner.extractUsage', () => {
     expect(
       extract(JSON.stringify({ type: 'turn.completed' }), 'gpt-5.6')
     ).toBeUndefined();
+  });
+});

From d8174a31c24cdfd7abeeba606c463019c9e1e134 Mon Sep 17 00:00:00 2001
From: Sean Oliver <882952+seanoliver@users.noreply.github.com>
Date: Tue, 8 Sep 2026 14:23:52 -0700
Subject: [PATCH 5/7] fix(framework): only run the eval harness when it is the
 entry point

run-eval.test.ts imports run-eval.ts for a helper, and the new
`vitest run harness` script picks that test up, so the unconditional
main() call ran on import. Also: drop a duplicated opencode test, assert
the CLI engine refuses a system prompt before install, document the
promptAddendum contract, and take buildSystemPrompt's args as an object.
---
 apps/framework/harness/run-eval.ts            | 39 ++++++++------
 apps/framework/harness/system-prompt.test.ts  | 54 +++++++++++--------
 apps/framework/harness/system-prompt.ts       | 16 ++++--
 packages/core/src/agents/engine.test.ts       | 35 ++++++++----
 .../core/src/agents/opencode/runner.test.ts   |  8 ---
 packages/core/src/index.ts                    |  9 ++++
 6 files changed, 103 insertions(+), 58 deletions(-)

diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts
index 9ab48236..ee7f870a 100644
--- a/apps/framework/harness/run-eval.ts
+++ b/apps/framework/harness/run-eval.ts
@@ -426,10 +426,10 @@ async function runOne(
       })
     );
 
-    const systemPrompt = buildSystemPrompt(
-      exp.agent.id,
-      session.promptAddendum
-    );
+    const systemPrompt = buildSystemPrompt({
+      agent: exp.agent.id,
+      addendum: session.promptAddendum,
+    });
     const run = await exp.agent.run({
       systemPrompt,
       userPrompt: prompt,
@@ -501,11 +501,13 @@ async function runOne(
   // A CLI agent discovers its installed skills itself. An in-process agent has
   // no filesystem, so its skills are advertised in the prompt and pulled on
   // demand via the load_skill tool instead.
-  const systemPrompt = buildSystemPrompt(
-    exp.agent.id,
-    session.promptAddendum,
-    agentRunsInSandbox ? undefined : buildToolsSkillsPrompt(toolsSkills)
-  );
+  const systemPrompt = buildSystemPrompt({
+    agent: exp.agent.id,
+    addendum: session.promptAddendum,
+    skillContext: agentRunsInSandbox
+      ? undefined
+      : buildToolsSkillsPrompt(toolsSkills),
+  });
   const run = await exp.agent.run({
     systemPrompt,
     userPrompt: prompt,
@@ -783,9 +785,16 @@ async function main() {
   }
 }
 
-main()
-  .then(() => process.exit(0))
-  .catch((e) => {
-    console.error(e);
-    process.exit(1);
-  });
+// Only when this file is the entry point. Importing it (a unit test reaching
+// for one of its helpers) must not dispatch a run or call process.exit.
+if (
+  process.argv[1] &&
+  import.meta.url === pathToFileURL(process.argv[1]).href
+) {
+  main()
+    .then(() => process.exit(0))
+    .catch((e) => {
+      console.error(e);
+      process.exit(1);
+    });
+}
diff --git a/apps/framework/harness/system-prompt.test.ts b/apps/framework/harness/system-prompt.test.ts
index 962e0c8f..80e14d2d 100644
--- a/apps/framework/harness/system-prompt.test.ts
+++ b/apps/framework/harness/system-prompt.test.ts
@@ -20,13 +20,15 @@ const skills: SkillEntry[] = [
 describe('buildSystemPrompt', () => {
   it('gives the ai-sdk agent task framing', () => {
     // ai-sdk is the one harness with no system prompt of its own.
-    expect(buildSystemPrompt('ai-sdk')).toContain('Use the provided tools');
+    expect(buildSystemPrompt({ agent: 'ai-sdk' })).toContain(
+      'Use the provided tools'
+    );
   });
 
   it('gives nothing to any CLI harness', () => {
     // CLI harnesses ship their own system prompt; that is what is measured.
     for (const agent of CLI_AGENTS) {
-      expect(buildSystemPrompt(agent)).toBe('');
+      expect(buildSystemPrompt({ agent })).toBe('');
     }
   });
 
@@ -35,42 +37,52 @@ describe('buildSystemPrompt', () => {
     // its own project-scope discovery and describes them to the model itself.
     for (const agent of CLI_AGENTS) {
       expect(
-        buildSystemPrompt(
+        buildSystemPrompt({
           agent,
-          buildToolSurfaceAddendum(agent),
-          buildSkillsPrompt(agent, skills)
-        )
+          addendum: buildToolSurfaceAddendum(agent),
+          skillContext: buildSkillsPrompt(agent, skills),
+        })
       ).toBe('');
     }
   });
 
   it('keeps the runtime blocks for ai-sdk, in order, after the base prompt', () => {
-    const base = buildSystemPrompt('ai-sdk');
-    expect(buildSystemPrompt('ai-sdk', 'Addendum.', 'Skills listing.')).toBe(
-      `${base}\n\nAddendum.\n\nSkills listing.`
-    );
-    const withSkills = buildSystemPrompt(
-      'ai-sdk',
-      buildToolSurfaceAddendum('ai-sdk'),
-      buildSkillsPrompt('ai-sdk', skills)
-    );
+    const base = buildSystemPrompt({ agent: 'ai-sdk' });
+    expect(
+      buildSystemPrompt({
+        agent: 'ai-sdk',
+        addendum: 'Addendum.',
+        skillContext: 'Skills listing.',
+      })
+    ).toBe(`${base}\n\nAddendum.\n\nSkills listing.`);
+    const withSkills = buildSystemPrompt({
+      agent: 'ai-sdk',
+      addendum: buildToolSurfaceAddendum('ai-sdk'),
+      skillContext: buildSkillsPrompt('ai-sdk', skills),
+    });
     expect(withSkills).toContain('## Available skills');
     expect(withSkills).toContain('- supabase: Use for Supabase tasks.');
   });
 
   it('drops empty blocks instead of leaving blank gaps', () => {
-    const base = buildSystemPrompt('ai-sdk');
-    expect(buildSystemPrompt('ai-sdk', '', 'Skills listing.')).toBe(
-      `${base}\n\nSkills listing.`
-    );
-    expect(buildSystemPrompt('ai-sdk', '', '')).toBe(base);
+    const base = buildSystemPrompt({ agent: 'ai-sdk' });
+    expect(
+      buildSystemPrompt({
+        agent: 'ai-sdk',
+        addendum: '',
+        skillContext: 'Skills listing.',
+      })
+    ).toBe(`${base}\n\nSkills listing.`);
+    expect(
+      buildSystemPrompt({ agent: 'ai-sdk', addendum: '', skillContext: '' })
+    ).toBe(base);
   });
 
   it('never names Supabase, a project, or how to end the turn', () => {
     // Issue #164: naming Supabase hands the agent the answer to "which tool";
     // "project" presumes there is one to modify; stopping coaching shapes the
     // report the judge reads. All three are part of what is measured.
-    const prompt = buildSystemPrompt('ai-sdk');
+    const prompt = buildSystemPrompt({ agent: 'ai-sdk' });
     expect(prompt).not.toMatch(/supabase/i);
     expect(prompt).not.toMatch(/project/i);
     expect(prompt).not.toMatch(/eval/i);
diff --git a/apps/framework/harness/system-prompt.ts b/apps/framework/harness/system-prompt.ts
index bb49af9e..e01dbd88 100644
--- a/apps/framework/harness/system-prompt.ts
+++ b/apps/framework/harness/system-prompt.ts
@@ -25,11 +25,17 @@ const AI_SDK_BASE_PROMPT =
  * already empty for a CLI agent at their source; an MCP server's
  * `promptAddendum` is not, and is left to reach `createCliAgent`, which throws.
  */
-export function buildSystemPrompt(
-  agent: AgentHarnessId,
-  addendum?: string,
-  skillContext?: string
-): string {
+export function buildSystemPrompt({
+  agent,
+  addendum,
+  skillContext,
+}: {
+  agent: AgentHarnessId;
+  /** Runtime text: the session's tool surface, or the MCP servers' addenda. */
+  addendum?: string;
+  /** The installed-skills listing. */
+  skillContext?: string;
+}): string {
   const base = agent === 'ai-sdk' ? AI_SDK_BASE_PROMPT : '';
   return [base, addendum, skillContext].filter(Boolean).join('\n\n');
 }
diff --git a/packages/core/src/agents/engine.test.ts b/packages/core/src/agents/engine.test.ts
index 37f7e7d6..4eb00b42 100644
--- a/packages/core/src/agents/engine.test.ts
+++ b/packages/core/src/agents/engine.test.ts
@@ -16,9 +16,15 @@ const parser: AgentTranscriptParser = {
   }),
 };
 
-/** Run a CLI agent against a fake sandbox, returning every command it ran. */
-async function runWithSystemPrompt(systemPrompt: string): Promise<string[]> {
+/**
+ * Run a CLI agent against a fake sandbox, returning every command it ran in
+ * the sandbox and whether the runner's `install` was reached.
+ */
+async function runWithSystemPrompt(
+  systemPrompt: string
+): Promise<{ commands: string[]; installed: boolean }> {
   const commands: string[] = [];
+  let installed = false;
   const runner: AgentRunner = {
     id: 'claude-code',
     displayName: 'Fake CLI',
@@ -26,7 +32,9 @@ async function runWithSystemPrompt(systemPrompt: string): Promise<string[]> {
     cliPackage: 'fake-cli',
     defaultCliVersion: '1.0.0',
     defaultModel: 'fake-model',
-    install: async () => undefined,
+    install: async () => {
+      installed = true;
+    },
     exec: async () => ({ command: ok, raw: '' }),
   };
   await createCliAgent(runner, parser, { model: 'fake-model' }).run({
@@ -42,7 +50,7 @@ async function runWithSystemPrompt(systemPrompt: string): Promise<string[]> {
       readFile: async () => '',
     },
   });
-  return commands;
+  return { commands, installed };
 }
 
 describe('createCliAgent prompt staging', () => {
@@ -52,14 +60,23 @@ describe('createCliAgent prompt staging', () => {
   });
 
   it('stages only the user prompt', async () => {
-    const commands = await runWithSystemPrompt('');
+    const { commands } = await runWithSystemPrompt('');
     expect(commands.some((c) => c.includes(USER_PROMPT_PATH))).toBe(true);
     expect(commands.some((c) => c.includes('system-prompt'))).toBe(false);
   });
 
-  it('refuses a system prompt: a CLI agent runs with its own', async () => {
-    await expect(runWithSystemPrompt('Extra framing.')).rejects.toThrow(
-      /runs with its own system prompt/
-    );
+  it('refuses a system prompt before touching the sandbox', async () => {
+    // A CLI agent runs with its own prompt. The refusal comes before install
+    // and staging, so a misconfigured experiment fails without paying for them.
+    const commands: string[] = [];
+    let installed = false;
+    await expect(
+      runWithSystemPrompt('Extra framing.').then((r) => {
+        commands.push(...r.commands);
+        installed = r.installed;
+      })
+    ).rejects.toThrow(/runs with its own system prompt/);
+    expect(commands).toEqual([]);
+    expect(installed).toBe(false);
   });
 });
diff --git a/packages/core/src/agents/opencode/runner.test.ts b/packages/core/src/agents/opencode/runner.test.ts
index 37b7a3a2..6ebc3dfa 100644
--- a/packages/core/src/agents/opencode/runner.test.ts
+++ b/packages/core/src/agents/opencode/runner.test.ts
@@ -209,12 +209,4 @@ describe('opencode runner exec routing', () => {
     expect(runCommand).toContain('"$(cat /u)"');
     expect(runCommand).not.toContain('system-prompt');
   });
-
-  it('sends the task alone as the message', async () => {
-    // opencode has no system-prompt flag, so anything else here would land on
-    // the user message. Nothing is prepended.
-    const { runCommand } = await captureExec('moonshotai/kimi-k3');
-    expect(runCommand).toContain('"$(cat /u)"');
-    expect(runCommand).not.toContain('system-prompt');
-  });
 });
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 562f24e9..0c058ca7 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -552,6 +552,10 @@ export type LocalStackSession = {
    * the agent harness.
    */
   mcpServers?: Record<string, McpServerConfig>;
+  /**
+   * Text for the agent's system prompt. Must be empty for a CLI agent, which
+   * runs with its own prompt; `createCliAgent` throws otherwise.
+   */
   promptAddendum?: string;
   scoringContext: LocalStackScoringContext;
   /**
@@ -863,6 +867,11 @@ export type EvalSessionArgs = {
 
 export type EvalSession = {
   mcpServers: Record<string, McpServerConfig>;
+  /**
+   * Text for the agent's system prompt, joined from the MCP servers'
+   * `promptAddendum`s. Must be empty for a CLI agent, which runs with its own
+   * prompt; `createCliAgent` throws otherwise.
+   */
   promptAddendum?: string;
   scoringContext: ToolScoringContext;
   close(): Promise<void>;

From 15e2a23393205d9bee954387c61b453e6c3dc5b7 Mon Sep 17 00:00:00 2001
From: Sean Oliver <882952+seanoliver@users.noreply.github.com>
Date: Tue, 8 Sep 2026 14:43:33 -0700
Subject: [PATCH 6/7] test(core): observe side effects when prompt guard
 rejects

---
 packages/core/src/agents/engine.test.ts | 28 ++++++++++++-------------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/packages/core/src/agents/engine.test.ts b/packages/core/src/agents/engine.test.ts
index 4eb00b42..7efcaf18 100644
--- a/packages/core/src/agents/engine.test.ts
+++ b/packages/core/src/agents/engine.test.ts
@@ -21,10 +21,12 @@ const parser: AgentTranscriptParser = {
  * the sandbox and whether the runner's `install` was reached.
  */
 async function runWithSystemPrompt(
-  systemPrompt: string
-): Promise<{ commands: string[]; installed: boolean }> {
-  const commands: string[] = [];
-  let installed = false;
+  systemPrompt: string,
+  effects: { commands: string[]; installed: boolean } = {
+    commands: [],
+    installed: false,
+  }
+): Promise<typeof effects> {
   const runner: AgentRunner = {
     id: 'claude-code',
     displayName: 'Fake CLI',
@@ -33,7 +35,7 @@ async function runWithSystemPrompt(
     defaultCliVersion: '1.0.0',
     defaultModel: 'fake-model',
     install: async () => {
-      installed = true;
+      effects.installed = true;
     },
     exec: async () => ({ command: ok, raw: '' }),
   };
@@ -44,13 +46,13 @@ async function runWithSystemPrompt(
     sandbox: {
       workspace: '/w',
       exec: async (command) => {
-        commands.push(command);
+        effects.commands.push(command);
         return ok;
       },
       readFile: async () => '',
     },
   });
-  return { commands, installed };
+  return effects;
 }
 
 describe('createCliAgent prompt staging', () => {
@@ -68,15 +70,11 @@ describe('createCliAgent prompt staging', () => {
   it('refuses a system prompt before touching the sandbox', async () => {
     // A CLI agent runs with its own prompt. The refusal comes before install
     // and staging, so a misconfigured experiment fails without paying for them.
-    const commands: string[] = [];
-    let installed = false;
+    const effects = { commands: [], installed: false };
     await expect(
-      runWithSystemPrompt('Extra framing.').then((r) => {
-        commands.push(...r.commands);
-        installed = r.installed;
-      })
+      runWithSystemPrompt('Extra framing.', effects)
     ).rejects.toThrow(/runs with its own system prompt/);
-    expect(commands).toEqual([]);
-    expect(installed).toBe(false);
+    expect(effects.commands).toEqual([]);
+    expect(effects.installed).toBe(false);
   });
 });

From cc83942e1b6766c25b497cefe0efffd9d885d8bc Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Fri, 11 Sep 2026 10:05:03 -0400
Subject: [PATCH 7/7] docs: trim readme

---
 README.md | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 1c919b9e..db6cd98a 100644
--- a/README.md
+++ b/README.md
@@ -161,10 +161,7 @@ Skills come from [`supabase/agent-skills`](https://github.com/supabase/agent-ski
 
 To use a skill in an experiment, reference its directory name in the experiment's `skills` array.
 
-Skills are always loaded lazily ([progressive disclosure](https://ai-sdk.dev/cookbook/guides/agent-skills)): a skill's full instructions are pulled on demand, never preloaded. How that happens depends on the harness:
-
-- **CLI harnesses (Claude Code, Codex, OpenCode)** use their own built-in skills mechanism. Skills are installed into the sandbox workspace with [Vercel's `skills` CLI](https://github.com/vercel-labs/skills) (baked into the sandbox image, sourced from the local `skills/` directory — never the network), into each harness's own project scope: `.claude/skills/` for Claude Code, `.agents/skills/` for Codex and OpenCode. Each CLI then discovers, advertises and loads the skills itself. The framework adds nothing to a CLI agent's system prompt: how the agent behaves as shipped is part of what an eval measures.
-- **The in-process `ai-sdk` harness** has no such mechanism, so the framework supplies one. In local-stack mode it lists each skill's name+description in the system prompt and the agent reads `.claude/skills/<name>/SKILL.md` with its file tools. In tools mode there is no filesystem, so a `load_skill` tool returns a skill's full instructions when the agent calls it with the skill's name.
+Skills are installed into the sandbox workspace with [Vercel's `skills` CLI](https://github.com/vercel-labs/skills) under each harness's native skills folder (`.claude/skills/` for Claude Code, `.agents/skills/` for Codex and OpenCode). The `ai-sdk` harness exposes a `load_skill` tool to support skill loading.
 
 ## Framework Checks