diff --git a/evals/cli/lib/detours.test.ts b/evals/cli/lib/detours.test.ts index 10d43c43..68bc7b1e 100644 --- a/evals/cli/lib/detours.test.ts +++ b/evals/cli/lib/detours.test.ts @@ -16,13 +16,12 @@ import { skipEnvOptions, } from './detours.js'; -// `cwd`/`endedAt` mirror the fields newer core versions add to ToolCallRecord. -type RecordFields = Partial> & { - cwd?: unknown; - endedAt?: unknown; -}; -const record = (fields: RecordFields): ToolCallRecord => - ({ tool: {}, body: {}, ts: 0, ...fields }) as ToolCallRecord; +const record = (fields: Partial): ToolCallRecord => ({ + tool: { kind: 'other', toolName: 'shell' }, + body: {}, + ts: 0, + ...fields, +}); // Exact `.source` text of the DETOUR_PATTERNS entries in detours.ts, so a // reorder of that array doesn't silently weaken these assertions. Order @@ -412,7 +411,7 @@ describe('extractCommandEntries', () => { body: { command: ['supabase', 'stop'] }, cwd: '/tmp/sandbox-x/legacy-import', }), - record({ command: 'supabase start', cwd: 42 }), + record({ command: 'supabase start' }), record({ command: 'supabase status' }), ]; @@ -422,7 +421,7 @@ describe('extractCommandEntries', () => { ); }); - it('carries a string cwd and omits a missing or non-string one', () => { + it('carries a cwd when the record has one', () => { expect(extractCommandEntries(calls)).toEqual([ { command: 'ls', cwd: '/tmp/sandbox-x' }, { command: 'supabase stop', cwd: '/tmp/sandbox-x/legacy-import' }, @@ -431,27 +430,15 @@ describe('extractCommandEntries', () => { ]); }); - it('takes the call time from endedAt', () => { + it('takes the call time from resultTs, whatever ts holds', () => { expect( extractCommandEntries([ - record({ command: 'supabase stop', ts: 1000, endedAt: 2000 }), - ]) - ).toEqual([{ command: 'supabase stop', at: 2000 }]); - }); - - it('omits the time when endedAt is missing or not a positive number, whatever ts holds', () => { - expect( - extractCommandEntries([ - record({ command: 'supabase stop', ts: 1000 }), - record({ command: 'supabase start', ts: 1500, endedAt: 'soon' }), - record({ command: 'supabase status', ts: 1700, endedAt: 0 }), - record({ command: 'supabase db reset', ts: 0 }), + record({ command: 'supabase stop', ts: 1000, resultTs: 2000 }), + record({ command: 'supabase start', ts: 1500 }), ]) ).toEqual([ - { command: 'supabase stop' }, + { command: 'supabase stop', at: 2000 }, { command: 'supabase start' }, - { command: 'supabase status' }, - { command: 'supabase db reset' }, ]); }); }); diff --git a/evals/cli/lib/detours.ts b/evals/cli/lib/detours.ts index aab27a68..88ba56ef 100644 --- a/evals/cli/lib/detours.ts +++ b/evals/cli/lib/detours.ts @@ -513,7 +513,8 @@ function recordCommand(record: ToolCallRecord): string { * with it. `cwd` comes from agent parsers that record a per-call directory * (Codex, OpenCode) and is absent otherwise; a `cd` persisting across separate * tool calls in a persistent shell (e.g. Claude Code) is not tracked. `at` is - * the call's completion time (epoch ms), only present when the agent parser records it. + * the call's `resultTs` (epoch ms) when the agent records it, never `ts`, which + * parsers disagree on whether it marks issue or completion. */ export function extractCommandEntries( toolCalls: readonly ToolCallRecord[] @@ -521,28 +522,16 @@ export function extractCommandEntries( return toolCalls.flatMap((record) => { const command = recordCommand(record); if (command.length === 0) return []; - // `cwd` isn't on every core version's ToolCallRecord yet. - const cwd = (record as { cwd?: unknown }).cwd; - const at = recordTime(record); return [ { command, - ...(typeof cwd === 'string' ? { cwd } : {}), - ...(at === undefined ? {} : { at }), + ...(record.cwd === undefined ? {} : { cwd: record.cwd }), + ...(record.resultTs === undefined ? {} : { at: record.resultTs }), }, ]; }); } -// `endedAt` isn't on every core version's ToolCallRecord. `ts` is never used: -// parsers disagree on whether it marks when a call was issued or finished. -function recordTime(record: ToolCallRecord): number | undefined { - const { endedAt } = record as { endedAt?: unknown }; - return typeof endedAt === 'number' && Number.isFinite(endedAt) && endedAt > 0 - ? endedAt - : undefined; -} - /** Numbers `commands` in order for the detour judge's input, each passed through in full. */ export function formatDetourJudgeInput(commands: readonly string[]): string { return commands.map((command, i) => `${i + 1}. ${command}`).join('\n'); diff --git a/packages/core/src/agents/claude-code/parser.test.ts b/packages/core/src/agents/claude-code/parser.test.ts index 9db0115c..c1710ca3 100644 --- a/packages/core/src/agents/claude-code/parser.test.ts +++ b/packages/core/src/agents/claude-code/parser.test.ts @@ -202,6 +202,7 @@ describe('adaptTranscript', () => { result: 'file1\nfile2', error: undefined, ts: Date.parse('2026-06-18T10:00:00.000Z'), + resultTs: Date.parse('2026-06-18T10:00:01.000Z'), }, { tool: { kind: 'mcp', server: 'supabase', toolName: 'search_docs' }, diff --git a/packages/core/src/agents/codex/parser.test.ts b/packages/core/src/agents/codex/parser.test.ts index 416098b0..f126c8a8 100644 --- a/packages/core/src/agents/codex/parser.test.ts +++ b/packages/core/src/agents/codex/parser.test.ts @@ -233,6 +233,7 @@ describe('codexParser', () => { expect(adapted.transcript[0]).toEqual({ type: 'tool_call', name: 'execute_sql', + id: 'item_7', input: args, output: toolResult, error: undefined, @@ -422,7 +423,47 @@ describe('enrichFromRollout', () => { expect(events.map((e) => [e.timestamp, e.requestId, e.usage])).toEqual([ ['t1', 'r1', usage], ['t2', 'r1', usage], - ['t5', undefined, undefined], + ['t4', undefined, undefined], + ]); + }); + + it("sets a paired tool call's cwd from the rollout item, resolving file:// URLs", () => { + const item = (id: string, command: string, cwd?: string) => + JSON.stringify({ + timestamp: 't', + type: 'event_msg', + payload: { + type: 'item_completed', + item: { + type: 'CommandExecution', + id, + command: ['/bin/bash', '-lc', command], + ...(cwd === undefined ? {} : { cwd }), + }, + }, + }); + const call = (id: string, command: string): TranscriptEvent => ({ + type: 'tool_call', + tool: { + name: 'shell', + originalName: 'command_execution', + id, + command: `/bin/bash -lc ${command}`, + }, + }); + const events = [call('i1', 'a'), call('i2', 'b'), call('i3', 'c')]; + enrichFromRollout( + events, + [ + item('c1', 'a', '/work/client-a'), + item('c2', 'b', 'file:///work/my%20app'), + item('c3', 'c'), + ].join('\n') + ); + expect(events.map((e) => e.tool?.cwd)).toEqual([ + '/work/client-a', + '/work/my app', + undefined, ]); }); @@ -530,12 +571,46 @@ describe('enrichFromRollout tool pairing', () => { ).toEqual(['t1', 't2']); }); - it('skips all pairing when one command has extra arguments', () => { + it('takes a yielded command completion time from its item_completed', () => { + const completedAt = (id: string, at: string) => + line(at, 'event_msg', { + type: 'item_completed', + item: { + type: 'CommandExecution', + id, + command: ['/bin/bash', '-lc', 'sleep 90'], + }, + }); + expect( + times(call('item_1', '/bin/bash -lc "sleep 90"'), [ + issued('c1', 't1'), + finished('c1', 't2'), + issued('c2', 't3'), + finished('c2', 't4'), + completedAt('c1', 't5'), + ]) + ).toEqual(['t1', 't5']); + }); + + it('falls back to the output time without an item_completed timestamp', () => { + expect( + times(call('item_1', '/bin/bash -lc "sleep 90"'), [ + issued('c1', 't1'), + finished('c1', 't2'), + issued('c2', 't3'), + finished('c2', 't4'), + completed('c1', 'sleep 90'), + ]) + ).toEqual(['t1', 't2']); + }); + + it('leaves a command with extra arguments unpaired and pairs the rest', () => { expect( times( [ ...call('item_1', '/bin/bash -lc pwd'), ...call('item_2', '/bin/bash -lc ls -la'), + ...call('item_3', '/bin/bash -lc date'), ], [ issued('c1', 't1'), @@ -544,9 +619,12 @@ describe('enrichFromRollout tool pairing', () => { issued('c2', 't3'), completed('c2', 'ls'), finished('c2', 't4'), + issued('c3', 't5'), + completed('c3', 'date'), + finished('c3', 't6'), ] ) - ).toEqual([undefined, undefined, undefined, undefined]); + ).toEqual(['t1', 't2', undefined, undefined, 't5', 't6']); }); it('matches a redacted secret', () => { @@ -575,7 +653,7 @@ describe('enrichFromRollout tool pairing', () => { ).toEqual([undefined, undefined]); }); - it('skips all pairing when the rollout is missing a command', () => { + it('pairs what it can when the rollout has fewer commands', () => { expect( times( [ @@ -584,10 +662,66 @@ describe('enrichFromRollout tool pairing', () => { ], [issued('c1', 't1'), completed('c1', 'pwd'), finished('c1', 't2')] ) - ).toEqual([undefined, undefined, undefined, undefined]); + ).toEqual(['t1', 't2', undefined, undefined]); }); - it('adds only itemless empty messages when tools do not pair', () => { + it('skips a rollout command the stream never listed', () => { + const events = [ + ...call('item_1', '/bin/bash -lc pwd'), + ...call('item_2', '/bin/bash -lc ls'), + ]; + const withCwd = (id: string, script: string, cwd: string) => + line('', 'event_msg', { + type: 'item_completed', + item: { + type: 'CommandExecution', + id, + cwd, + command: ['/bin/bash', '-lc', script], + }, + }); + enrichFromRollout( + events, + [ + issued('c1', 't1'), + withCwd('c1', 'pwd', '/a'), + finished('c1', 't2'), + issued('c2', 't3'), + withCwd('c2', 'supabase functions serve', '/killed'), + finished('c2', 't4'), + issued('c3', 't5'), + withCwd('c3', 'ls', '/b'), + finished('c3', 't6'), + ].join('\n') + ); + expect(events.map((e) => [e.timestamp, e.tool?.cwd])).toEqual([ + ['t1', '/a'], + ['t2', undefined], + ['t5', '/b'], + ['t6', undefined], + ]); + }); + + it('never pairs out of order, even when a later item matches an earlier call', () => { + expect( + times( + [ + ...call('item_1', '/bin/bash -lc ls'), + ...call('item_2', '/bin/bash -lc pwd'), + ], + [ + issued('c1', 't1'), + completed('c1', 'pwd'), + finished('c1', 't2'), + issued('c2', 't3'), + completed('c2', 'ls'), + finished('c2', 't4'), + ] + ) + ).toEqual(['t3', 't4', undefined, undefined]); + }); + + it('adds only itemless empty messages when a tool call does not pair', () => { const events = [ ...call('item_1', '/bin/bash -lc pwd'), ...call('item_2', '/bin/bash -lc ls'), @@ -605,7 +739,7 @@ describe('enrichFromRollout tool pairing', () => { ].join('\n') ); expect(events.map((e) => [e.type, e.requestId])).toEqual([ - ['tool_call', undefined], + ['tool_call', 'r1'], ['tool_result', undefined], ['tool_call', undefined], ['tool_result', undefined], diff --git a/packages/core/src/agents/codex/parser.ts b/packages/core/src/agents/codex/parser.ts index d9e8842b..9d15bf16 100644 --- a/packages/core/src/agents/codex/parser.ts +++ b/packages/core/src/agents/codex/parser.ts @@ -21,6 +21,7 @@ * format (event_msg/response_item) that older parsers targeted. */ +import { fileURLToPath } from 'node:url'; import { isRecord, parseJsonlRecords } from '../../json.js'; import type { ParsedTranscript, @@ -264,17 +265,21 @@ const ROLLOUT_TOOL_ITEMS = new Set([ * Fills in event times, model requests, and per-request usage from the session * rollout, which the `--json` stream lacks. The two streams list tool items and * assistant messages in the same order but under different ids, so they're - * paired by position. Tool pairs must also agree on the command or MCP tool, - * and any count or content mismatch leaves those messages or tool calls - * untouched. A request no event came from, like a compaction call, gets an - * empty message. If the messages or tool calls didn't pair, only a request with - * no `response_item` records gets one, since the others may just be missing - * their requestId. + * paired by order. Messages pair only when their counts match. Each tool call + * pairs with the next rollout item that agrees on its command or MCP tool, so + * a call with no such item stays untouched without affecting the others. A + * request no event came from, like a compaction call, gets an empty message. + * If the messages or any tool call didn't pair, only a request with no + * `response_item` records gets one, since the others may just be missing their + * requestId. * * rollout: reasoning → message → function_call(c1) → token_usage_record(r1) * → function_call_output(c1) * events: message → tool_call → tool_result, each tagged requestId r1 * + * A command finishes when its item completes, not when its output arrives: + * a yielded long-running command outputs early, then `write_stdin` polls it. + * * Returns when the rollout recorded the first user message, in epoch ms. */ export function enrichFromRollout( @@ -291,6 +296,7 @@ export function enrichFromRollout( const callEnds = new Map(); const messages: { at?: string; request: Request }[] = []; const toolItems: Record[] = []; + const itemEnds = new Map(); const requests: { at?: string; request: Request }[] = []; let open: Request | undefined; let promptAt: number | undefined; @@ -328,6 +334,7 @@ export function enrichFromRollout( const { type, id } = payload.item; if (typeof id === 'string' && ROLLOUT_TOOL_ITEMS.has(String(type))) { toolItems.push(payload.item); + if (at && type === 'CommandExecution') itemEnds.set(id, at); } } } @@ -347,25 +354,38 @@ export function enrichFromRollout( }); } const calls = events.filter((e) => e.type === 'tool_call'); - const toolsPaired = - calls.length === toolItems.length && - calls.every((event, i) => sameCall(event, toolItems[i])); - if (toolsPaired) { - const callIdByItem = new Map(); - calls.forEach((event, i) => { - const itemId = String(toolItems[i].id); - const start = callStarts.get(itemId); - if (event.tool?.id) callIdByItem.set(event.tool.id, itemId); - if (!start) return; - event.timestamp = start.at ?? event.timestamp; - tag(event, start.request); - }); - for (const event of events) { - if (event.type !== 'tool_result' || !event.tool?.id) continue; - const callId = callIdByItem.get(event.tool.id); - const end = callId ? callEnds.get(callId) : undefined; - if (end) event.timestamp = end; - } + // The rollout keeps items the stream omits, like a killed long-running + // command, so each call takes the next matching item and skips the rest. + let cursor = 0; + const items = calls.map((event) => { + const found = toolItems.findIndex( + (item, i) => i >= cursor && sameCall(event, item) + ); + if (found === -1) return undefined; + cursor = found + 1; + return toolItems[found]; + }); + const toolsPaired = items.every((item) => item); + const callIdByItem = new Map(); + calls.forEach((event, i) => { + const item = items[i]; + if (!item) return; + const itemId = String(item.id); + const start = callStarts.get(itemId); + if (event.tool?.id) callIdByItem.set(event.tool.id, itemId); + const cwd = directoryPath(item.cwd); + if (cwd && event.tool) event.tool.cwd = cwd; + if (!start) return; + event.timestamp = start.at ?? event.timestamp; + tag(event, start.request); + }); + for (const event of events) { + if (event.type !== 'tool_result' || !event.tool?.id) continue; + const callId = callIdByItem.get(event.tool.id); + const end = + (callId ? itemEnds.get(callId) : undefined) ?? + (callId ? callEnds.get(callId) : undefined); + if (end) event.timestamp = end; } const tagged = new Set(events.map((e) => e.requestId)); for (const { at, request } of requests) { @@ -409,7 +429,21 @@ function sameCall(event: TranscriptEvent, item: Record) { if (item.type === 'McpToolCall') { return event.tool?.call?.toolName === item.tool; } - return true; + if (item.type === 'FileChange') { + return event.tool?.call?.toolName === 'file_change'; + } + return event.tool?.call?.toolName === 'web_search'; +} + +/** A rollout `cwd` is a plain path or, for some Codex versions, a `file://` URL. */ +function directoryPath(value: unknown): string | undefined { + if (typeof value !== 'string' || !value) return undefined; + if (!value.startsWith('file:')) return value; + try { + return fileURLToPath(value); + } catch { + return undefined; + } } const REDACTED = '[REDACTED_SECRET]'; diff --git a/packages/core/src/agents/opencode/parser.test.ts b/packages/core/src/agents/opencode/parser.test.ts index 218c97a9..cec507d4 100644 --- a/packages/core/src/agents/opencode/parser.test.ts +++ b/packages/core/src/agents/opencode/parser.test.ts @@ -111,6 +111,32 @@ describe('opencodeParser', () => { expect(results.every((e) => e.tool?.success === true)).toBe(true); }); + it("normalizes bash's workdir to the call's cwd", () => { + const stream = JSON.stringify({ + type: 'tool_use', + part: { + type: 'tool', + tool: 'bash', + callID: 'tool_1', + state: { + status: 'completed', + input: { command: 'supabase start', workdir: '/work/client-a' }, + output: '', + metadata: { exit: 0 }, + }, + }, + }); + + const { toolCalls } = adaptTranscript( + opencodeParser.parseTranscript(stream).events + ); + expect(toolCalls[0].cwd).toBe('/work/client-a'); + expect( + adaptTranscript(opencodeParser.parseTranscript(SESSION).events) + .toolCalls[0].cwd + ).toBeUndefined(); + }); + it('surfaces reasoning + the assistant report via the adapter', () => { const events = opencodeParser.parseTranscript(SESSION).events; expect( @@ -131,6 +157,7 @@ describe('opencodeParser', () => { result: 'file1\nfile2', error: undefined, ts: 1782295624290, // epoch ms preserved through toISO -> parseTs + resultTs: 1782295624290, }, { tool: { kind: 'other', toolName: 'write' }, @@ -140,6 +167,7 @@ describe('opencodeParser', () => { result: 'written', error: undefined, ts: 1782295624300, + resultTs: 1782295624300, }, ]); }); diff --git a/packages/core/src/agents/opencode/parser.ts b/packages/core/src/agents/opencode/parser.ts index 4fd3e809..29e71eb9 100644 --- a/packages/core/src/agents/opencode/parser.ts +++ b/packages/core/src/agents/opencode/parser.ts @@ -105,13 +105,15 @@ const OPENCODE_TOOLS: AgentToolMap = { /** * opencode tool args → normalized fields. `bash` carries the command in - * `command`; file tools the path in `filePath` (or `path`); `webfetch` the URL - * in `url`. The shared extractor reads whichever keys this map names. + * `command` (and an optional `workdir`); file tools the path in `filePath` (or + * `path`); `webfetch` the URL in `url`. The shared extractor reads whichever + * keys this map names. */ const OPENCODE_ARG_FIELDS: ArgFieldMap = { path: ['filePath', 'file_path', 'path'], command: ['command'], url: ['url'], + cwd: ['workdir'], }; /** @@ -221,6 +223,7 @@ function partToEvents( if (normalized.path) tool.path = normalized.path; if (normalized.command) tool.command = normalized.command; if (normalized.url) tool.url = normalized.url; + if (normalized.cwd) tool.cwd = normalized.cwd; const loadedSkills = loadedSkillsFromOpencodeCall(tool); if (loadedSkills.length > 0) tool.loadedSkills = loadedSkills; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 5061645a..d8e4a5eb 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -221,6 +221,8 @@ export interface ToolCallRecord { path?: string; command?: string; url?: string; + /** Absolute directory the call ran in, when the agent records one per call. */ + cwd?: string; /** Canonical tool category, set by CLI agent parsers; unset for ai-sdk tools which have no normalization layer. */ name?: ToolName; /** Skill names loaded by this call, when the harness can identify any. */ @@ -228,6 +230,8 @@ export interface ToolCallRecord { result?: unknown; error?: string; ts: number; + /** Epoch ms the call's result arrived, when the agent records it per call. */ + resultTs?: number; } export interface CommandResult { diff --git a/packages/core/src/parsers/adapt.test.ts b/packages/core/src/parsers/adapt.test.ts new file mode 100644 index 00000000..1a339134 --- /dev/null +++ b/packages/core/src/parsers/adapt.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from 'vitest'; +import type { TranscriptEvent } from '../transcript/types.js'; +import { adaptTranscript } from './adapt.js'; + +describe('adaptTranscript', () => { + const events: TranscriptEvent[] = [ + { + type: 'tool_call', + timestamp: '2026-06-18T10:00:00.000Z', + tool: { + name: 'shell', + originalName: 'Bash', + id: 't1', + command: 'supabase start', + cwd: '/work/client-a', + }, + }, + { + type: 'tool_result', + timestamp: '2026-06-18T10:00:05.000Z', + tool: { name: 'shell', originalName: 'Bash', id: 't1', result: 'ok' }, + }, + { + type: 'tool_call', + tool: { name: 'shell', originalName: 'Bash', id: 't2', command: 'ls' }, + }, + ]; + + it("copies a call's cwd and result time onto its ToolCallRecord", () => { + const [first, second] = adaptTranscript(events).toolCalls; + expect(first.cwd).toBe('/work/client-a'); + expect(first.resultTs).toBe(Date.parse('2026-06-18T10:00:05.000Z')); + expect(second.cwd).toBeUndefined(); + expect(second.resultTs).toBeUndefined(); + }); +}); diff --git a/packages/core/src/parsers/adapt.ts b/packages/core/src/parsers/adapt.ts index 525963b3..605fb358 100644 --- a/packages/core/src/parsers/adapt.ts +++ b/packages/core/src/parsers/adapt.ts @@ -90,10 +90,12 @@ export function adaptTranscript(events: TranscriptEvent[]): AdaptedTranscript { path: event.tool.path, command: event.tool.command, url: event.tool.url, + cwd: event.tool.cwd, loadedSkills: event.tool.loadedSkills, result: resolved?.error === undefined ? resolved?.result : undefined, error: resolved?.error, ts: parseTs(event.timestamp), + ...(resolved?.ts ? { resultTs: resolved.ts } : {}), }); } } diff --git a/packages/core/src/parsers/shared/extract.ts b/packages/core/src/parsers/shared/extract.ts index ce727e8d..bddb175a 100644 --- a/packages/core/src/parsers/shared/extract.ts +++ b/packages/core/src/parsers/shared/extract.ts @@ -17,6 +17,7 @@ export interface ArgFieldMap { path?: readonly string[]; command?: readonly string[]; url?: readonly string[]; + cwd?: readonly string[]; } /** Normalized values extracted from a tool call's args. */ @@ -24,6 +25,7 @@ export interface ExtractedArgs { path?: string; command?: string; url?: string; + cwd?: string; } // Skill reads can appear as bare paths or quoted shell args in tool-call logs. @@ -63,6 +65,7 @@ export function extractArgs( path: firstField(args, map.path), command: firstField(args, map.command), url: firstField(args, map.url), + cwd: firstField(args, map.cwd), }; } diff --git a/packages/core/src/transcript/types.ts b/packages/core/src/transcript/types.ts index c0582674..88096b8c 100644 --- a/packages/core/src/transcript/types.ts +++ b/packages/core/src/transcript/types.ts @@ -92,6 +92,8 @@ export interface TranscriptEvent { path?: string; command?: string; url?: string; + /** Absolute directory the call ran in, when the agent records one per call. */ + cwd?: string; /** Skill names loaded by this call, when the parser can identify any. */ loadedSkills?: string[]; /** Tool result payload (for `tool_result`). */