Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions apps/framework/scripts/upload-braintrust.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,54 @@ describe('logTranscript', () => {
});
});

it('starts LLM spans during a backgrounded command at the previous edge', () => {
const root: RecordedSpan = { children: [] };
const message = (requestId: string, s: number) => ({
type: 'message' as const,
role: 'assistant' as const,
content: requestId,
ts: (100 + s) * 1000,
requestId,
});
logTranscript(recorder(root), {
prompt: 'go',
agentReport: '',
checks: [],
judgeCalls: [],
passed: true,
modelId: 'm',
startTime: 100,
toolLabels: [],
transcript: [
{
type: 'tool_call',
name: 'Bash',
input: {},
id: 't1',
ts: 104_000,
resultTs: 120_000,
requestId: 'r1',
},
message('r2', 8),
message('r3', 12),
message('r4', 25),
],
});
expect(
root.children[0]?.children.map(({ name, start, end }) => [
name,
start,
end,
])
).toEqual([
['m', 100, 104],
['Bash', 104, 120],
['m', 104, 108],
['m', 108, 112],
['m', 120, 125],
]);
});

it('ends setup at a leading non-assistant message and starts task there', () => {
const root: RecordedSpan = { children: [] };
logTranscript(recorder(root), {
Expand Down
22 changes: 15 additions & 7 deletions apps/framework/scripts/upload-braintrust.ts
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,7 @@ export interface SpanSink {
* https://github.com/braintrustdata/braintrust-coding-agent-plugins/blob/c0346dcdb16ae9f136b3abf83efb6458191f9300/bt-daemon/src/translate/claude.rs#L1173-L1300
* https://github.com/braintrustdata/braintrust-spec/blob/b068e39112e081e45b6070e035877f1e2e83f9b7/skills/instrumentation-spec/references/features/eval-spans.md#trace-shape
*
* An LLM span starts where the previous span ended, since the agent only
* An LLM span starts where the latest earlier span ended, since the agent only
* writes a request's timestamps once its blocks finish generating. Braintrust's
* Claude Code translator starts it at the first block instead, which leaves the
* generation time as a gap. Their Codex translator does what we do.
Expand Down Expand Up @@ -498,6 +498,15 @@ export function logTranscript(
});
task.log({ input: row.prompt, output: row.agentReport });
let latestTime = taskStart;
// A backgrounded command can end after later requests, so each request
// starts at the latest edge before it.
const edges = taskStart === undefined ? [] : [taskStart];
const mark = (time: number | undefined) => {
if (time !== undefined) {
edges.push(time);
}
latestTime = latest(latestTime, time);
};
let toolIndex = 0;
// Seeded with the prompt, which the transcript omits.
const history: Record<string, unknown>[] = row.prompt
Expand All @@ -522,10 +531,8 @@ export function logTranscript(
open.usage ??= part.usage;
return open;
}
const startTime =
latestTime === undefined || at === undefined
? (latestTime ?? at)
: Math.min(latestTime, at);
const before = at === undefined ? [] : edges.filter((edge) => edge <= at);
const startTime = before.length ? Math.max(...before) : (at ?? latestTime);
const request: Request = {
id,
span: task.startSpan({
Expand Down Expand Up @@ -565,7 +572,7 @@ export function logTranscript(
? `${message.content}\n${part.content}`
: part.content;
}
latestTime = latest(latestTime, at);
mark(at);
continue;
}
if (part.requestId) {
Expand All @@ -586,7 +593,8 @@ export function logTranscript(
}
const label = row.toolLabels[toolIndex++];
const endTime = sec(part.resultTs) ?? at;
latestTime = latest(latestTime, endTime);
mark(at);
mark(endTime);
const span = task.startSpan({
name: label ? `${part.name}: ${label}` : part.name,
type: 'tool',
Expand Down