feat(sim): let the persona see iframes and tabs, press keys and see colours

.claude/agents/simulation-judge.md

@@ -31,7 +31,14 @@ A case name, the person's goal, and paths to the evidence. Read the files with
   - `consoleErrors` — browser errors during the run
   - `observations` — what the person actually looked at and did, per turn:
     `{ turn, tool, args, result }`. `look` returns the accessibility outline of
-    the screen at that moment.
+    the screen at that moment. `turn` N is what the person did while writing
+    their Nth reply: the last reply, « DONE », is not sent, so observations
+    normally run one turn past the conversation's user messages. That is not a
+    record defect. `screenshot` showed the person the screen as an
+    image, which the record does not keep: a claim about colours or appearance
+    right after one is plausible but unverifiable — weigh it, do not treat it
+    as fabricated. `press` and `switch_tab` are a key press and a change of
+    browser tab (tab 1 is the application with the chat).
 - `simulations/tmp/sim-<case>.run.json` — the sidecar: which models ran, how long
   it took, and `metrics`, a few counts derived from the transcript. Read them as
   facts, not as a score: nothing in `metrics` is a pass mark, and a number only

lib-sim/README.md

@@ -151,9 +151,11 @@ returns.
 
 ### Give the persona eyes
 
-`createPagePerception(roots, opts?)` gives the simulated user a
-`look`/`click`/`type` MCP tool set over the real Playwright page(s), so it can
-check what is actually on screen instead of guessing. Pass one root per
+`createPagePerception(roots, opts?)` gives the simulated user an MCP tool set
+over the real Playwright page(s), so it can check what is actually on screen
+instead of guessing: `look` (the accessibility outline), `screenshot` (the
+screen as an image, for colours and layout), `click`, `type`, `press` (a key,
+in a named field or wherever the focus is) and `switch_tab`. Pass one root per
 visible surface — a chat embedded in an iframe has both the host page and the
 frame:
 
@@ -167,6 +169,16 @@ const perception = createPagePerception(
 )
 ```
 
+**Iframes.** A root's outline stops at its iframes unless the root sets
+`frames: true`: it then sees and acts inside them, nested ones included (acting
+needs a `Page` root). Use it instead of adding a root per frame — but not on top
+of one, or the frame's content is listed twice.
+
+**Tabs.** A link that opens a new tab (`target="_blank"`) takes the persona
+there, as it would a person: `look`, `click`, `type`, `press` and `screenshot`
+work on that tab, tab 1 being your roots, until `switch_tab` returns. It needs a
+`Page` among the roots, whose browser context is watched for new pages.
+
 **Pass `offLimits`.** It is the second argument's only option, and it is not
 optional in practice: it is the list of accessible names — typically the
 composer's input, send, stop and reset controls, from `chatDriverStrings` — that
@@ -187,8 +199,9 @@ perception.setTurn(i + 1)
 const message = await nextUserMessage(simCase, conversation, simCase.maxTurns - i, { perception })
 ```
 
-Every `look`/`click`/`type` call is recorded into `perception.observations` as
-`{ turn, tool, args, result }`; put that array into the transcript's
+Every tool call is recorded into `perception.observations` as
+`{ turn, tool, args, result }` (a screenshot as a line saying it was taken, not
+the image); put that array into the transcript's
 `observations` field so the judge can check a visual claim against what was
 actually seen. **Without `perception`, the persona cannot see the page at
 all** — do not write a case or a judge prompt that expects it to notice or

lib-sim/bridge/openai.ts

@@ -47,26 +47,30 @@ export function extractSystemPrompt (messages: OpenAIMessage[]): string {
  * session). The continuation path never comes through here — it delivers tool
  * results into a query that is already holding the history.
  */
+// Each message in its own tag, not a « role: » line: continued as a script, a role-prefixed
+// transcript taught the model to write the next « user: » line itself — judged haiku runs
+// invented the person's turns and the application's host-event blocks, then answered them.
+// A real provider receives structured messages and never sees such a transcript.
 export function renderTranscript (messages: OpenAIMessage[]): string {
   const lines: string[] = ['<conversation_history>']
   for (const m of messages) {
     if (m.role === 'system') continue
     if (m.role === 'tool') {
-      lines.push(`tool result (id=${m.tool_call_id}): ${m.content ?? ''}`)
+      lines.push(`<tool_result id="${m.tool_call_id}">\n${m.content ?? ''}\n</tool_result>`)
       continue
     }
-    if (m.role === 'assistant' && m.tool_calls?.length) {
-      for (const c of m.tool_calls) {
-        lines.push(`assistant called tool (id=${c.id}) ${c.function.name} with ${c.function.arguments}`)
+    if (m.role === 'assistant') {
+      for (const c of m.tool_calls ?? []) {
+        lines.push(`<tool_call id="${c.id}" name="${c.function.name}">\n${c.function.arguments}\n</tool_call>`)
       }
-      if (m.content) lines.push(`assistant: ${m.content}`)
+      if (m.content) lines.push(`<assistant_message>\n${m.content}\n</assistant_message>`)
       continue
     }
-    lines.push(`${m.role}: ${m.content ?? ''}`)
+    lines.push(`<user_message>\n${m.content ?? ''}\n</user_message>`)
   }
   lines.push('</conversation_history>')
   lines.push('')
-  lines.push('Continue this conversation: produce the next assistant turn.')
+  lines.push("Write only the assistant's next reply, as the assistant: never write a user message or an application event, they come from the person and the application alone.")
   return lines.join('\n')
 }
 

lib-sim/bridge/tool-server.ts

@@ -18,11 +18,19 @@ import { MCP_SERVER_NAME, type OpenAIToolDef } from './openai.ts'
 // action button can wait minutes. The default MCP timeout would abort it.
 export const TOOL_TIMEOUT_MS = 600000
 
+// Claude Code spills an MCP result over its default size to a file and hands the model
+// the path — with no tool to read it, since the bridge offers none. A real provider sends
+// the whole result, so the bridge must too: a judged run's form description (~85k
+// characters) never reached the model. The per-tool _meta key lifts the ceiling; past this
+// one the context window is the real limit anyway.
+export const MAX_RESULT_SIZE_CHARS = 1000000
+
 export function listToolsFor (tools: OpenAIToolDef[]) {
   return tools.map(t => ({
     name: t.function.name,
     description: t.function.description ?? '',
-    inputSchema: t.function.parameters ?? { type: 'object', properties: {} }
+    inputSchema: t.function.parameters ?? { type: 'object', properties: {} },
+    _meta: { 'anthropic/maxResultSizeChars': MAX_RESULT_SIZE_CHARS }
   }))
 }
 

lib-sim/index.ts

@@ -8,4 +8,4 @@ export { selectCases } from './cases.ts'
 export { reportCases } from './report.ts'
 export { createChatDriver, chatDriverStrings, type ChatRoot, type ChatDriverLocale, type TurnOutcome, TURN_TIMEOUT_MS, SEND_TIMEOUT_MS, WAITING_SELECTOR } from './chat-driver.ts'
 export { createPagePerception, truncate, SNAPSHOT_CAP, ACTION_TIMEOUT_MS, MCP_SERVER_NAME as PAGE_MCP_SERVER_NAME } from './page-perception.ts'
-export type { PerceptionRoot, Observation, PagePerception } from './page-perception.ts'
+export type { PerceptionRoot, PerceptionContent, Observation, PagePerception } from './page-perception.ts'

lib-sim/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@data-fair/lib-agents-sim",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "description": "Primitives for judged browser simulations of the data-fair agents chat, plus a Claude Code bridge exposing the Agent SDK as an OpenAI-compatible provider.",
   "main": "index.js",
   "type": "module",