From 62fbed9aa57a3f0cd4241c39c6d240f737282c33 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 7 Oct 2026 11:40:43 +0100 Subject: [PATCH 01/11] fix(session): validate structured output and bound corrective retries --- bun.lock | 1 + packages/cli/package.json | 1 + packages/cli/src/cli/cmd/generate.ts | 18 + packages/cli/src/session/index.ts | 14 + packages/cli/src/session/llm.ts | 13 +- packages/cli/src/session/output-schema.ts | 64 ++++ packages/cli/src/session/processor.ts | 9 +- packages/cli/src/session/prompt.ts | 135 +++++-- packages/cli/test/fixture/output-schema.ts | 29 ++ .../cli/test/session/output-schema.test.ts | 349 ++++++++++++++++++ .../test/session/structured-output.test.ts | 48 +-- packages/sdk/script/build.ts | 42 ++- packages/sdk/src/gen/types.gen.ts | 48 +-- packages/sdk/src/v2/gen/types.gen.ts | 141 +++++++ 14 files changed, 804 insertions(+), 108 deletions(-) create mode 100644 packages/cli/src/session/output-schema.ts create mode 100644 packages/cli/test/fixture/output-schema.ts create mode 100644 packages/cli/test/session/output-schema.test.ts diff --git a/bun.lock b/bun.lock index 1a07967f..24a7ba27 100644 --- a/bun.lock +++ b/bun.lock @@ -50,6 +50,7 @@ "@standard-schema/spec": "1.0.0", "@zip.js/zip.js": "2.7.62", "ai": "catalog:", + "ajv": "^8", "decimal.js": "10.5.0", "diff": "catalog:", "drizzle-orm": "1.0.0-beta.12-a5629fb", diff --git a/packages/cli/package.json b/packages/cli/package.json index ccf8d2c7..430e56bf 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -102,6 +102,7 @@ "@standard-schema/spec": "1.0.0", "@zip.js/zip.js": "2.7.62", "ai": "catalog:", + "ajv": "^8", "decimal.js": "10.5.0", "diff": "catalog:", "drizzle-orm": "1.0.0-beta.12-a5629fb", diff --git a/packages/cli/src/cli/cmd/generate.ts b/packages/cli/src/cli/cmd/generate.ts index 3e4760ab..cf9b83bb 100644 --- a/packages/cli/src/cli/cmd/generate.ts +++ b/packages/cli/src/cli/cmd/generate.ts @@ -1,6 +1,7 @@ import type { CommandModule } from "yargs" import z from "zod" import { MessageV2 } from "@/session/message-v2" +import { Session } from "@/session" import { ProviderTermination } from "@/provider/termination" export const GenerateCommand = { @@ -10,9 +11,26 @@ export const GenerateCommand = { const registry = z.registry<{ id: string }>() registry.add(MessageV2.StepFinishPart, { id: "StepFinishPart" }) registry.add(ProviderTermination.Info, { id: "ProviderTermination" }) + registry.add(MessageV2.Format, { id: "OutputFormat" }) + registry.add(MessageV2.Assistant, { id: "AssistantMessage" }) + registry.add( + z.object({ + type: z.literal(Session.Event.StructuredOutput.type), + properties: Session.Event.StructuredOutput.properties, + }), + { id: "EventSessionStructuredOutput" }, + ) + registry.add( + z.object({ + type: z.literal(Session.Event.StructuredOutputRejected.type), + properties: Session.Event.StructuredOutputRejected.properties, + }), + { id: "EventSessionStructuredOutputRejected" }, + ) const components = z.toJSONSchema(registry, { metadata: z.registry(), target: "draft-2020-12", + io: "input", uri: (id) => `#/components/schemas/${id}`, }) // Component JSON pointers are references, not standalone schema resource IDs. diff --git a/packages/cli/src/session/index.ts b/packages/cli/src/session/index.ts index 49560c0e..e19ee2ec 100644 --- a/packages/cli/src/session/index.ts +++ b/packages/cli/src/session/index.ts @@ -1,3 +1,4 @@ +import { OutputSchema } from "./output-schema" import { Slug } from "@aictrl/util/slug" import path from "path" import { BusEvent } from "@/bus/bus-event" @@ -178,6 +179,19 @@ export namespace Session { export type GlobalInfo = z.output export const Event = { + StructuredOutputRejected: BusEvent.define( + "session.structured_output_rejected", + z.object({ + sessionID: z.string(), + attempt: z.number(), + maxAttempts: z.number(), + errors: OutputSchema.Diagnostic.array(), + }), + ), + StructuredOutput: BusEvent.define( + "session.structured_output", + z.object({ sessionID: z.string(), outcome: OutputSchema.Outcome }), + ), Created: BusEvent.define( "session.created", z.object({ diff --git a/packages/cli/src/session/llm.ts b/packages/cli/src/session/llm.ts index 2ff4ac25..e4b49e3d 100644 --- a/packages/cli/src/session/llm.ts +++ b/packages/cli/src/session/llm.ts @@ -40,6 +40,7 @@ export namespace LLM { tools: Record retries?: number toolChoice?: "auto" | "required" | "none" + structured?: { reject: (input: string) => Promise } } export type StreamOutput = StreamTextResult @@ -173,10 +174,20 @@ export namespace LLM { return streamText({ onError(error) { l.error("stream error", { - error, + error: input.structured ? "Structured output stream failed" : error, }) }, async experimental_repairToolCall(failed) { + if (failed.toolCall.toolName === "StructuredOutput" && input.structured) { + return { + ...failed.toolCall, + toolName: "invalid", + input: JSON.stringify({ + tool: "StructuredOutput", + error: await input.structured.reject(failed.toolCall.input), + }), + } + } const lower = failed.toolCall.toolName.toLowerCase() if (lower !== failed.toolCall.toolName && tools[lower]) { l.info("repairing tool call", { diff --git a/packages/cli/src/session/output-schema.ts b/packages/cli/src/session/output-schema.ts new file mode 100644 index 00000000..279a48fe --- /dev/null +++ b/packages/cli/src/session/output-schema.ts @@ -0,0 +1,64 @@ +import Ajv, { type ErrorObject } from "ajv" +import Ajv2020 from "ajv/dist/2020" +import z from "zod" + +export namespace OutputSchema { + export const Diagnostic = z.object({ path: z.string(), keyword: z.string(), message: z.string() }) + export const Outcome = z.discriminatedUnion("status", [ + z.object({ status: z.literal("accepted"), attempts: z.number(), value: z.unknown() }), + z.object({ + status: z.literal("failed"), + reason: z.enum(["exhausted", "missing", "step_limit", "aborted", "error"]), + attempts: z.number(), + }), + ]) + export type Outcome = z.infer + export type Diagnostic = z.infer + + // Share the canonical validator across CLI preflight, tool execution and final + // acceptance, including schemas cloned by Zod or loaded from session storage. + const cache = new Map>() + + export function compile(schema: Record) { + if (schema.type !== "object") throw new Error('schema root must have type "object"') + const key = JSON.stringify(schema) + const cached = cache.get(key) + if (cached) return cached + const ajv = + typeof schema.$schema === "string" && + /^https:\/\/json-schema\.org\/draft\/2020-12\/schema#?$/.test(schema.$schema) + ? Ajv2020 + : Ajv + const validate = new ajv({ allErrors: true }).compile(schema) + if (cache.size >= 32) cache.delete(cache.keys().next().value!) + cache.set(key, validate) + return validate + } + + export function diagnostics(errors?: ErrorObject[] | null): Diagnostic[] { + return (errors ?? []).slice(0, 10).reduce((result, error) => { + const item = { + path: error.instancePath.slice(0, 96), + keyword: error.keyword.slice(0, 32), + message: (error.message ?? "invalid value").slice(0, 96), + } + // Bound the encoded bytes too: Unicode and JSON escaping can expand text. + return Buffer.byteLength(JSON.stringify([...result, item])) < 1600 ? [...result, item] : result + }, []) + } + + export function parse(input: string, validate: ReturnType) { + try { + const value: unknown = JSON.parse(input) + if (validate(value)) return [] + return diagnostics(validate.errors) + } catch { + // JSON parser errors can echo the entire submitted input. + return [{ path: "", keyword: "parse", message: "arguments must be complete, valid JSON" }] + } + } + + export function error(errors: Diagnostic[]) { + return new Error(`StructuredOutput rejected: ${JSON.stringify(errors)}`) + } +} diff --git a/packages/cli/src/session/processor.ts b/packages/cli/src/session/processor.ts index d2da3a7e..20b51311 100644 --- a/packages/cli/src/session/processor.ts +++ b/packages/cli/src/session/processor.ts @@ -232,6 +232,11 @@ export namespace SessionProcessor { const lastThree = history.slice(-DOOM_LOOP_THRESHOLD) if ( + !( + streamInput.structured && + (value.toolName === "StructuredOutput" || + (value.toolName === "invalid" && value.input?.tool === "StructuredOutput")) + ) && lastThree.length === DOOM_LOOP_THRESHOLD && lastThree.every( (p) => @@ -460,8 +465,8 @@ export namespace SessionProcessor { } } catch (e: any) { log.error("process", { - error: e, - stack: JSON.stringify(e.stack), + error: streamInput.structured ? "Structured output stream failed" : e, + stack: streamInput.structured ? undefined : JSON.stringify(e.stack), }) const error = MessageV2.APIError.isInstance(e) ? e diff --git a/packages/cli/src/session/prompt.ts b/packages/cli/src/session/prompt.ts index 1885473f..2548d604 100644 --- a/packages/cli/src/session/prompt.ts +++ b/packages/cli/src/session/prompt.ts @@ -1,3 +1,4 @@ +import { OutputSchema } from "./output-schema" import path from "path" import os from "os" import fs from "fs/promises" @@ -52,7 +53,7 @@ globalThis.AI_SDK_LOG_WARNINGS = false const STRUCTURED_OUTPUT_DESCRIPTION = `Use this tool to return your final response in the requested structured format. IMPORTANT: -- You MUST call this tool exactly once at the end of your response +- Call this tool at the end of your response; repair rejected arguments within the allowed budget - The input must be valid JSON matching the required schema - Complete all necessary research and tool calls BEFORE calling this tool - This tool provides your final answer - no further actions are taken after calling it` @@ -182,6 +183,7 @@ export namespace SessionPrompt { export type PromptInput = z.infer export const prompt = fn(PromptInput, async (input) => { + if (input.format?.type === "json_schema") OutputSchema.compile(input.format.schema) const session = await Session.get(input.sessionID) await SessionRevert.cleanup(session) @@ -326,6 +328,16 @@ export namespace SessionPrompt { // Note: On session resumption, state is reset but outputFormat is preserved // on the user message and will be retrieved from lastUser below let structuredOutput: unknown | undefined + let attempts = 0 + let rejected = 0 + let contract: + | { + id: string + format: MessageV2.OutputFormat & { type: "json_schema" } + validate: ReturnType + } + | undefined + let outcome: OutputSchema.Outcome | undefined let step = 0 const session = await Session.get(sessionID) @@ -353,6 +365,12 @@ export namespace SessionPrompt { } if (!lastUser) throw new Error("No user message found in stream. This should never happen.") + if (lastUser.format?.type === "json_schema" && contract?.id !== lastUser.id) { + contract = { id: lastUser.id, format: lastUser.format, validate: OutputSchema.compile(lastUser.format.schema) } + attempts = 0 + rejected = 0 + structuredOutput = undefined + } const lastAssistantMsg = msgs.findLast((msg) => msg.info.id === lastAssistant?.id) if ( lastAssistant && @@ -368,10 +386,24 @@ export namespace SessionPrompt { const agent = await Agent.get(lastUser.agent) const maxSteps = agent.steps ?? Infinity if (step > maxSteps) { - const error = new MessageV2.APIError({ - message: `Agent step limit (${maxSteps}) reached before a final response.`, - isRetryable: false, - }).toObject() + const error = contract + ? new MessageV2.StructuredOutputError({ + message: "Agent step limit reached without structured output", + retries: Math.max(0, attempts - 1), + }).toObject() + : new MessageV2.APIError({ + message: `Agent step limit (${maxSteps}) reached before a final response.`, + isRetryable: false, + }).toObject() + if (contract) { + outcome = { status: "failed", reason: "step_limit", attempts } + if (lastAssistant) { + delete lastAssistant.structured + lastAssistant.error = error + lastAssistant.time.completed = Date.now() + await Session.updateMessage(lastAssistant) + } + } if ( lastAssistant && (lastAssistant.finish === "tool-calls" || lastAssistant.finish === "unknown") && @@ -671,11 +703,32 @@ export namespace SessionPrompt { messages: msgs, }) - // Inject StructuredOutput tool if JSON schema mode enabled - if (lastUser.format?.type === "json_schema") { + const reject = async (input: string) => { + if (!contract) throw new Error("Missing output schema") + const errors = OutputSchema.parse(input, contract.validate) + if (attempts < contract.format.retryCount + 1) { + attempts++ + rejected++ + await Bus.publish(Session.Event.StructuredOutputRejected, { + sessionID, + attempt: attempts, + maxAttempts: contract.format.retryCount + 1, + errors, + }) + } + return OutputSchema.error(errors).message + } + + if (contract) { tools["StructuredOutput"] = createStructuredOutputTool({ - schema: lastUser.format.schema, - onSuccess(output) { + schema: contract.format.schema, + validate: contract.validate, + model, + async onSuccess(output) { + if (structuredOutput !== undefined) return + if (attempts >= contract!.format.retryCount + 1) + throw new Error("StructuredOutput attempt budget exhausted") + attempts++ structuredOutput = output }, }) @@ -738,15 +791,32 @@ export namespace SessionPrompt { ], tools, model, - toolChoice: format.type === "json_schema" ? "required" : undefined, + structured: contract ? { reject } : undefined, }) // If structured output was captured, save it and exit immediately // This takes priority because the StructuredOutput tool was called successfully - if (structuredOutput !== undefined) { + if ( + structuredOutput !== undefined && + !processor.message.error && + !abort.aborted && + contract?.validate(structuredOutput) + ) { processor.message.structured = structuredOutput processor.message.finish = processor.message.finish ?? "stop" await Session.updateMessage(processor.message) + outcome = { status: "accepted", attempts, value: structuredOutput } + break + } + + if (contract && !processor.message.error && rejected > contract.format.retryCount) { + processor.message.error = new MessageV2.StructuredOutputError({ + message: "Structured output corrective attempts exhausted", + retries: Math.max(0, attempts - 1), + }).toObject() + await Session.updateMessage(processor.message) + await Bus.publish(Session.Event.Error, { sessionID, error: processor.message.error }) + outcome = { status: "failed", reason: "exhausted", attempts } break } @@ -756,9 +826,11 @@ export namespace SessionPrompt { ) { processor.message.error = new MessageV2.StructuredOutputError({ message: "Model did not produce structured output", - retries: 0, + retries: Math.max(0, attempts - 1), }).toObject() await Session.updateMessage(processor.message) + await Bus.publish(Session.Event.Error, { sessionID, error: processor.message.error }) + outcome = { status: "failed", reason: "missing", attempts } break } @@ -774,6 +846,12 @@ export namespace SessionPrompt { continue } await SessionCompaction.prune({ sessionID }) + if (contract) { + await Bus.publish(Session.Event.StructuredOutput, { + sessionID, + outcome: outcome ?? { status: "failed", reason: abort.aborted ? "aborted" : "error", attempts }, + }) + } for await (const item of MessageV2.stream(sessionID)) { if (item.info.role === "user") continue const queued = state()[sessionID]?.callbacks ?? [] @@ -991,19 +1069,25 @@ export namespace SessionPrompt { /** @internal Exported for testing */ export function createStructuredOutputTool(input: { - schema: Record - onSuccess: (output: unknown) => void + schema: Record + validate?: ReturnType + model?: Provider.Model + onSuccess: (output: unknown) => void | Promise }): AITool { - // Remove $schema property if present (not needed for tool input) - const { $schema, ...toolSchema } = input.schema - + const validate = input.validate ?? OutputSchema.compile(input.schema) + const schema = Object.fromEntries(Object.entries(input.schema).filter(([key]) => key !== "$schema")) return tool({ - id: "StructuredOutput" as any, description: STRUCTURED_OUTPUT_DESCRIPTION, - inputSchema: jsonSchema(toolSchema as any), + inputSchema: jsonSchema(input.model ? ProviderTransform.schema(input.model, schema) : schema, { + validate(value) { + return validate(value) + ? { success: true, value } + : { success: false, error: OutputSchema.error(OutputSchema.diagnostics(validate.errors)) } + }, + }), async execute(args) { - // AI SDK validates args against inputSchema before calling execute() - input.onSuccess(args) + if (!validate(args)) throw OutputSchema.error(OutputSchema.diagnostics(validate.errors)) + await input.onSuccess(args) return { output: "Structured output captured successfully.", title: "Structured Output", @@ -1011,11 +1095,10 @@ export namespace SessionPrompt { } }, toModelOutput(result) { - return { - type: "text", - value: result.output, - } + return { type: "text", value: result.output } }, + // Retain the internal identifier used by tool consumers. + ...{ id: "StructuredOutput" }, }) } @@ -1786,6 +1869,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the model: z.string().optional(), arguments: z.string(), command: z.string(), + format: MessageV2.Format.optional(), variant: z.string().optional(), parts: z .array( @@ -1943,6 +2027,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the model: userModel, agent: userAgent, parts, + format: input.format, variant: input.variant, })) as MessageV2.WithParts diff --git a/packages/cli/test/fixture/output-schema.ts b/packages/cli/test/fixture/output-schema.ts new file mode 100644 index 00000000..b78610c0 --- /dev/null +++ b/packages/cli/test/fixture/output-schema.ts @@ -0,0 +1,29 @@ +export const schema = { + type: "object", + properties: { + result: { type: "string" }, + nested: { + type: "object", + properties: { count: { type: "integer" } }, + required: ["count"], + additionalProperties: false, + }, + }, + required: ["result"], + additionalProperties: false, +} + +export const corpus = [ + {}, + { result: 42 }, + { result: "valid", nested: { count: "42" } }, + { result: "valid", extra: "must remain" }, + { result: "valid", nested: { count: 3, extra: true } }, + { result: "valid", nested: null }, + { result: "valid", nested: { count: 1.5 } }, + { result: "valid" }, + { result: "valid", nested: { count: 42 } }, + null, + [], + "value", +] diff --git a/packages/cli/test/session/output-schema.test.ts b/packages/cli/test/session/output-schema.test.ts new file mode 100644 index 00000000..7d64ba05 --- /dev/null +++ b/packages/cli/test/session/output-schema.test.ts @@ -0,0 +1,349 @@ +import { expect, test } from "bun:test" +import Ajv from "ajv" +import Ajv2020 from "ajv/dist/2020" +import { Instance } from "../../src/project/instance" +import { Session } from "../../src/session" +import { SessionPrompt } from "../../src/session/prompt" +import { Bus } from "../../src/bus" +import { OutputSchema } from "../../src/session/output-schema" +import { tmpdir } from "../fixture/fixture" +import { schema, corpus } from "../fixture/output-schema" + +async function run( + inputs: string[], + retryCount = 2, + options: { + schema?: Record + steps?: number + finish?: string + abort?: boolean + timeout?: boolean + failure?: boolean + other?: boolean + plain?: boolean + gemini?: boolean + } = {}, +) { + const requests: Record[] = [] + const server = Bun.serve({ + port: 0, + async fetch(request) { + requests.push((await request.json()) as Record) + if ( + requests.length > (inputs.length === 1 && inputs[0] === "{}" ? retryCount + 1 : retryCount + inputs.length + 1) + ) + return Response.json({ error: { message: "Fixture request budget exceeded" } }, { status: 400 }) + if (options.failure) + return new Response(JSON.stringify({ error: { message: "Fixture provider failed" } }), { + status: 400, + headers: { "Content-Type": "application/json" }, + }) + if (options.abort || options.timeout) + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode( + 'data: {"choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}\n\n', + ), + ) + }, + }), + { headers: { "Content-Type": "text/event-stream" } }, + ) + const args = inputs[Math.min(requests.length - 1, inputs.length - 1)] + const calls = [ + { + index: 0, + id: `call_${requests.length}`, + type: "function", + function: { name: "StructuredOutput", arguments: args }, + }, + ] + const chunks = [ + { choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }] }, + { + choices: [ + { + index: 0, + delta: + args === "prose" + ? { content: "No final tool call" } + : { + tool_calls: options.other + ? [ + ...calls, + { + index: 1, + id: `read_${requests.length}`, + type: "function", + function: { name: "read", arguments: JSON.stringify({ filePath: "aictrl.json" }) }, + }, + ] + : calls, + }, + finish_reason: null, + }, + ], + }, + { + choices: [{ index: 0, delta: {}, finish_reason: options.finish ?? "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }, + ] + return new Response( + chunks + .map((chunk) => `data: ${JSON.stringify(chunk)}`) + .concat("data: [DONE]") + .join("\n\n") + "\n\n", + { headers: { "Content-Type": "text/event-stream" } }, + ) + }, + }) + try { + await using tmp = await tmpdir({ + config: { + ...(options.steps ? { agent: { build: { steps: options.steps } } } : {}), + enabled_providers: options.gemini ? ["fixture"] : ["alibaba"], + provider: options.gemini + ? { + fixture: { + npm: "@ai-sdk/openai-compatible", + options: { apiKey: "test-key", baseURL: `${server.url.origin}/v1` }, + models: { "gemini-fixture": { name: "fixture", limit: { context: 100000, output: 1000 } } }, + }, + } + : { alibaba: { options: { apiKey: "test-key", baseURL: `${server.url.origin}/v1` } } }, + }, + }) + return await Instance.provide({ + directory: tmp.path, + fn: async () => { + const session = await Session.create({ title: "Schema boundary fixture" }) + const rejected: { + sessionID: string + attempt: number + maxAttempts: number + errors: OutputSchema.Diagnostic[] + }[] = [] + const outcomes: OutputSchema.Outcome[] = [] + const unsubscribe = [ + Bus.subscribe(Session.Event.StructuredOutputRejected, (event) => { + rejected.push(event.properties) + }), + Bus.subscribe(Session.Event.StructuredOutput, (event) => { + outcomes.push(event.properties.outcome) + }), + ] + const idle = process.env.AICTRL_MODEL_STREAM_IDLE_TIMEOUT_MS + if (options.timeout) process.env.AICTRL_MODEL_STREAM_IDLE_TIMEOUT_MS = "50" + const timer = options.abort ? setTimeout(() => SessionPrompt.cancel(session.id), 100) : undefined + try { + const result = await SessionPrompt.prompt({ + sessionID: session.id, + model: options.gemini + ? { providerID: "fixture", modelID: "gemini-fixture" } + : { providerID: "alibaba", modelID: "qwen-plus" }, + parts: [{ type: "text", text: "Return a result." }], + ...(options.plain + ? {} + : { format: { type: "json_schema" as const, schema: options.schema ?? schema, retryCount } }), + }) + if (result.info.role !== "assistant") throw new Error("Expected assistant") + return { + info: result.info, + requests, + rejected, + outcomes, + messages: await Session.messages({ sessionID: session.id }), + } + } finally { + unsubscribe.forEach((fn) => fn()) + if (timer) clearTimeout(timer) + if (idle === undefined) delete process.env.AICTRL_MODEL_STREAM_IDLE_TIMEOUT_MS + else process.env.AICTRL_MODEL_STREAM_IDLE_TIMEOUT_MS = idle + } + }, + }) + } finally { + server.stop(true) + } +} + +test("missing required field crosses real tool boundary and must be repaired before capture", async () => { + const result = await run(["{}", JSON.stringify({ result: "repaired" })]) + expect(result.info.structured).toEqual({ result: "repaired" }) + expect(result.requests).toHaveLength(2) + expect(result.info.error).toBeUndefined() +}, 15_000) + +for (const [name, value] of [ + ["wrong scalar type", { result: 42 }], + ["wrong nested type", { result: "valid", nested: { count: "42" } }], + ["extra property is not stripped", { result: "valid", extra: "private" }], +] as const) { + test( + name, + async () => { + const result = await run([JSON.stringify(value), '{"result":"repaired"}']) + expect(result.info.structured).toEqual({ result: "repaired" }) + expect(result.rejected).toHaveLength(1) + expect(result.rejected[0]).toMatchObject({ attempt: 1, maxAttempts: 3 }) + expect(result.outcomes).toEqual([{ status: "accepted", attempts: 2, value: { result: "repaired" } }]) + expect(result.requests).toHaveLength(2) + }, + 15_000, + ) +} + +test("truncated arguments are a counted rejected attempt", async () => { + const result = await run(['{"result":', '{"result":"repaired"}']) + expect(result.info.structured).toEqual({ result: "repaired" }) + expect(result.rejected).toEqual([ + { + sessionID: result.info.sessionID, + attempt: 1, + maxAttempts: 3, + errors: [{ path: "", keyword: "parse", message: "arguments must be complete, valid JSON" }], + }, + ]) +}, 15_000) + +for (const retries of [0, 2, 4]) { + test(`retryCount ${retries} bounds requests and reports corrective attempts`, async () => { + const result = await run(["{}"], retries) + expect(result.requests).toHaveLength(retries + 1) + expect(result.rejected).toHaveLength(retries + 1) + expect(result.info.structured).toBeUndefined() + expect(result.info.error).toMatchObject({ name: "StructuredOutputError", data: { retries } }) + expect(result.outcomes).toEqual([{ status: "failed", reason: "exhausted", attempts: retries + 1 }]) + }, 15_000) +} + +test("repair diagnostics sent to model are bounded and never echo submitted values", async () => { + const secret = "submitted-secret-" + "x".repeat(5000) + const result = await run([ + JSON.stringify({ + result: secret, + ...Object.fromEntries(Array.from({ length: 40 }, (_, i) => [`extra${i}`, secret])), + }), + '{"result":"repaired"}', + ]) + expect(result.rejected[0].errors).toHaveLength(10) + const request = result.requests[1] as { messages: { role: string; content: unknown }[] } + const feedback = request.messages + .filter((message) => message.role === "tool") + .map((message) => JSON.stringify(message.content)) + .join("") + expect(Buffer.byteLength(feedback)).toBeLessThan(2048) + const calls = (request.messages as { tool_calls?: { function: { name: string; arguments: string } }[] }[]) + .flatMap((message) => message.tool_calls ?? []) + .filter((call) => call.function.name === "invalid") + expect(calls.length).toBeGreaterThan(0) + calls.forEach((call) => { + expect(Buffer.byteLength(call.function.arguments)).toBeLessThan(2048) + }) + expect(JSON.stringify(request)).not.toContain("submitted-secret-") + expect(feedback).toContain("additionalProperties") + expect(result.info.structured).toEqual({ result: "repaired" }) +}, 15_000) + +test("valid StructuredOutput wins when another tool is called in the same step", async () => { + const result = await run(['{"result":"accepted"}'], 0, { other: true }) + expect(result.info.structured).toEqual({ result: "accepted" }) + expect(result.requests).toHaveLength(1) +}, 15_000) + +test("model finish without StructuredOutput has an explicit missing outcome", async () => { + const result = await run(["prose"]) + expect(result.info.structured).toBeUndefined() + expect(result.info.error).toMatchObject({ name: "StructuredOutputError", data: { retries: 0 } }) + expect(result.outcomes).toEqual([{ status: "failed", reason: "missing", attempts: 0 }]) +}, 15_000) + +test("step cap without a valid result has an explicit failure", async () => { + const result = await run(["{}"], 4, { steps: 1 }) + expect(result.info.structured).toBeUndefined() + expect(result.info.error).toMatchObject({ name: "StructuredOutputError", data: { retries: 0 } }) + expect(result.outcomes).toEqual([{ status: "failed", reason: "step_limit", attempts: 1 }]) + expect(result.requests).toHaveLength(1) +}, 15_000) + +for (const [name, options, reason] of [ + ["cancellation", { abort: true }, "aborted"], + ["stream timeout", { timeout: true }, "error"], + ["provider error", { failure: true }, "error"], + ["provider failed finish after a valid tool call", { finish: "content_filter" }, "error"], +] as const) { + test(`${name} never accepts a result`, async () => { + const result = await run(['{"result":"must not accept"}'], 2, options) + expect(result.info.structured).toBeUndefined() + expect(result.info.error).toBeDefined() + expect(result.outcomes).toEqual([{ status: "failed", reason, attempts: options.finish ? 1 : 0 }]) + }, 15_000) +} + +test("plain session keeps prose and emits no structured events", async () => { + const result = await run(["prose"], 2, { plain: true }) + expect(result.info.error).toBeUndefined() + expect(result.info.structured).toBeUndefined() + expect(result.outcomes).toEqual([]) + expect(result.rejected).toEqual([]) + expect(result.requests).toHaveLength(1) +}, 15_000) + +test("Ajv verdict parity across hostile corpus without mutating values", () => { + const reference = new Ajv({ allErrors: true }).compile(schema) + const validate = OutputSchema.compile(schema) + for (const value of corpus) { + const original = JSON.stringify(value) + expect(validate(value)).toBe(reference(value)) + expect(JSON.stringify(value)).toBe(original) + } + expect(OutputSchema.compile(JSON.parse(JSON.stringify(schema)))).toBe(validate) +}) + +test("draft 2020-12 uses Ajv2020 with default strict validation", () => { + const canonical = { ...schema, $schema: "https://json-schema.org/draft/2020-12/schema", unevaluatedProperties: false } + const reference = new Ajv2020({ allErrors: true }).compile(canonical) + const validate = OutputSchema.compile(canonical) + for (const value of corpus) expect(validate(value)).toBe(reference(value)) +}) + +test("execute defence in depth rejects invalid input before capture", async () => { + const captured: unknown[] = [] + const tool = SessionPrompt.createStructuredOutputTool({ + schema, + onSuccess: (value) => { + captured.push(value) + }, + }) + await expect(tool.execute!({}, { toolCallId: "fixture", messages: [] })).rejects.toThrow("StructuredOutput rejected") + expect(captured).toEqual([]) +}) + +test("draft 2020-12 declaration with a fragment uses the same validator dialect", () => { + const canonical = { ...schema, $schema: "https://json-schema.org/draft/2020-12/schema#" } + const reference = new Ajv2020({ allErrors: true }).compile(canonical) + const validate = OutputSchema.compile(canonical) + for (const value of corpus) expect(validate(value)).toBe(reference(value)) +}) + +test("missing result after rejections reports corrective attempts truthfully", async () => { + const result = await run(["{}", "{}", "prose"]) + expect(result.info.structured).toBeUndefined() + expect(result.info.error).toMatchObject({ name: "StructuredOutputError", data: { retries: 1 } }) + expect(result.outcomes).toEqual([{ status: "failed", reason: "missing", attempts: 2 }]) +}, 15_000) + +test("Gemini tool schema is transformed but canonical numeric enum remains authoritative", async () => { + const canonical = { type: "object", properties: { result: { type: "integer", enum: [1, 2] } }, required: ["result"] } + const result = await run(['{"result":"1"}', '{"result":1}'], 2, { schema: canonical, gemini: true }) + expect(result.info.structured).toEqual({ result: 1 }) + expect(result.rejected).toHaveLength(1) + const tools = result.requests[0].tools as { function: { name: string; parameters: unknown } }[] + expect(tools.find((tool) => tool.function.name === "StructuredOutput")?.function.parameters).toMatchObject({ + properties: { result: { type: "string", enum: ["1", "2"] } }, + }) + expect(canonical.properties.result).toEqual({ type: "integer", enum: [1, 2] }) +}, 15_000) diff --git a/packages/cli/test/session/structured-output.test.ts b/packages/cli/test/session/structured-output.test.ts index 2be4257d..7d1ca1f5 100644 --- a/packages/cli/test/session/structured-output.test.ts +++ b/packages/cli/test/session/structured-output.test.ts @@ -1,4 +1,5 @@ import { describe, expect, test } from "bun:test" +import { asSchema } from "ai" import { MessageV2 } from "../../src/session/message-v2" import { SessionPrompt } from "../../src/session/prompt" @@ -237,48 +238,28 @@ describe("structured-output.createStructuredOutputTool", () => { expect(result.metadata.valid).toBe(true) }) - test("AI SDK validates schema before execute - missing required field", async () => { - // Note: The AI SDK validates the input against the schema BEFORE calling execute() - // So invalid inputs never reach the tool's execute function - // This test documents the expected schema behavior + test("AI SDK boundary rejects missing required fields", async () => { const tool = SessionPrompt.createStructuredOutputTool({ schema: { type: "object", - properties: { - name: { type: "string" }, - age: { type: "number" }, - }, + properties: { name: { type: "string" }, age: { type: "number" } }, required: ["name", "age"], }, - onSuccess: () => {}, + onSuccess: () => { + throw new Error("Invalid input captured") + }, }) - - // The schema requires both 'name' and 'age' - expect(tool.inputSchema).toBeDefined() - const inputSchema = tool.inputSchema as any - expect(inputSchema.jsonSchema?.required).toContain("name") - expect(inputSchema.jsonSchema?.required).toContain("age") + expect(await asSchema(tool.inputSchema).validate!({})).toMatchObject({ success: false }) }) - test("AI SDK validates schema types before execute - wrong type", async () => { - // Note: The AI SDK validates the input against the schema BEFORE calling execute() - // So invalid inputs never reach the tool's execute function - // This test documents the expected schema behavior + test("AI SDK boundary rejects wrong scalar types", async () => { const tool = SessionPrompt.createStructuredOutputTool({ - schema: { - type: "object", - properties: { - count: { type: "number" }, - }, - required: ["count"], + schema: { type: "object", properties: { count: { type: "number" } }, required: ["count"] }, + onSuccess: () => { + throw new Error("Invalid input captured") }, - onSuccess: () => {}, }) - - // The schema defines 'count' as a number - expect(tool.inputSchema).toBeDefined() - const inputSchema = tool.inputSchema as any - expect(inputSchema.jsonSchema?.properties?.count?.type).toBe("number") + expect(await asSchema(tool.inputSchema).validate!({ count: "wrong" })).toMatchObject({ success: false }) }) test("execute handles nested objects", async () => { @@ -379,7 +360,6 @@ describe("structured-output.createStructuredOutputTool", () => { expect(modelOutput.value).toBe("Test output") }) - // Note: Retry behavior is handled by the AI SDK and the prompt loop, not the tool itself - // The tool simply calls onSuccess when execute() is called with valid args - // See prompt.ts loop() for actual retry logic + // The prompt loop owns the corrective attempt budget; this tool enforces the + // canonical validator before capture, including direct execute() calls. }) diff --git a/packages/sdk/script/build.ts b/packages/sdk/script/build.ts index 36537533..a562626d 100755 --- a/packages/sdk/script/build.ts +++ b/packages/sdk/script/build.ts @@ -38,17 +38,39 @@ await createClient({ ], }) -// The headless generator writes v2 types; the legacy SDK client still refers to -// src/gen/types.gen.ts. Bridge this shared part to its canonical generated type -// so event unions in both SDK versions retain the same termination contract. +// The headless generator writes canonical v2 schemas. Keep the legacy HTTP +// client's shared types aligned without regenerating its existing endpoints. const legacy = Bun.file("./src/gen/types.gen.ts") -const source = await legacy.text() -const pattern = - /^export type StepFinishPart = (?:\{[\s\S]*?^\}|import\("\.\.\/v2\/gen\/types\.gen\.js"\)\.StepFinishPart)$/m -if (!pattern.test(source)) throw new Error("Legacy StepFinishPart declaration not found; update the SDK schema bridge") -await legacy.write( - source.replace(pattern, 'export type StepFinishPart = import("../v2/gen/types.gen.js").StepFinishPart'), -) +let source = await legacy.text() +for (const name of [ + "StepFinishPart", + "AssistantMessage", + "OutputFormat", + "EventSessionStructuredOutput", + "EventSessionStructuredOutputRejected", +]) { + const pattern = new RegExp( + `^export type ${name} = (?:\\{[\\s\\S]*?^\\}|import\\("\\.\\.\\/v2\\/gen\\/types\\.gen\\.js"\\)\\.${name})$`, + "m", + ) + const declaration = `export type ${name} = import("../v2/gen/types.gen.js").${name}` + source = pattern.test(source) ? source.replace(pattern, declaration) : source + "\n" + declaration + "\n" +} +for (const name of ["SessionPromptData", "SessionPromptAsyncData", "SessionCommandData"]) { + const start = source.indexOf(`export type ${name} = {`) + const end = source.indexOf("\n path:", start) + if (start < 0 || end < 0) throw new Error(`Legacy ${name} declaration not found`) + const body = source.slice(start, end) + if (!body.includes("format?: OutputFormat")) + source = + source.slice(0, start) + body.replace(" body?: {", " body?: {\n format?: OutputFormat") + source.slice(end) +} +if (!source.includes(" | EventSessionStructuredOutput\n")) + source = source.replace( + "export type Event =\n", + "export type Event =\n | EventSessionStructuredOutput\n | EventSessionStructuredOutputRejected\n", + ) +await legacy.write(source) await $`bun prettier --write src/gen` await $`bun prettier --write src/v2` diff --git a/packages/sdk/src/gen/types.gen.ts b/packages/sdk/src/gen/types.gen.ts index 7432044d..b44897b7 100644 --- a/packages/sdk/src/gen/types.gen.ts +++ b/packages/sdk/src/gen/types.gen.ts @@ -117,42 +117,7 @@ export type ApiError = { } } -export type AssistantMessage = { - id: string - sessionID: string - role: "assistant" - time: { - created: number - completed?: number - } - error?: - | ProviderAuthError - | UnknownError - | MessageOutputLengthError - | MessageAbortedError - | StreamIdleTimeoutError - | ApiError - parentID: string - modelID: string - providerID: string - mode: string - path: { - cwd: string - root: string - } - summary?: boolean - cost: number - tokens: { - input: number - output: number - reasoning: number - cache: { - read: number - write: number - } - } - finish?: string -} +export type AssistantMessage = import("../v2/gen/types.gen.js").AssistantMessage export type Message = UserMessage | AssistantMessage @@ -706,6 +671,8 @@ export type EventServerConnected = { } export type Event = + | EventSessionStructuredOutput + | EventSessionStructuredOutputRejected | EventServerInstanceDisposed | EventInstallationUpdated | EventInstallationUpdateAvailable @@ -2588,6 +2555,7 @@ export type SessionMessagesResponse = SessionMessagesResponses[keyof SessionMess export type SessionPromptData = { body?: { + format?: OutputFormat messageID?: string model?: { providerID: string @@ -2683,6 +2651,7 @@ export type SessionMessageResponse = SessionMessageResponses[keyof SessionMessag export type SessionPromptAsyncData = { body?: { + format?: OutputFormat messageID?: string model?: { providerID: string @@ -2732,6 +2701,7 @@ export type SessionPromptAsyncResponse = SessionPromptAsyncResponses[keyof Sessi export type SessionCommandData = { body?: { + format?: OutputFormat messageID?: string agent?: string model?: string @@ -3906,3 +3876,9 @@ export type EventSubscribeResponse = EventSubscribeResponses[keyof EventSubscrib export type ClientOptions = { baseUrl: `${string}://${string}` | (string & {}) } + +export type OutputFormat = import("../v2/gen/types.gen.js").OutputFormat + +export type EventSessionStructuredOutput = import("../v2/gen/types.gen.js").EventSessionStructuredOutput + +export type EventSessionStructuredOutputRejected = import("../v2/gen/types.gen.js").EventSessionStructuredOutputRejected diff --git a/packages/sdk/src/v2/gen/types.gen.ts b/packages/sdk/src/v2/gen/types.gen.ts index 8a91f5c6..f2c1d236 100644 --- a/packages/sdk/src/v2/gen/types.gen.ts +++ b/packages/sdk/src/v2/gen/types.gen.ts @@ -45,3 +45,144 @@ export type ProviderTermination = { truncated: boolean } } + +export type OutputFormat = + | { + type: "text" + } + | { + type: "json_schema" + schema: { + [key: string]: unknown + } + retryCount?: number + } + +export type AssistantMessage = { + id: string + sessionID: string + role: "assistant" + time: { + created: number + completed?: number + } + error?: + | { + name: "ProviderAuthError" + data: { + providerID: string + message: string + } + } + | { + name: "UnknownError" + data: { + message: string + } + } + | { + name: "MessageOutputLengthError" + data: { + [key: string]: unknown + } + } + | { + name: "MessageAbortedError" + data: { + message: string + } + } + | { + name: "StreamIdleTimeoutError" + data: { + message: string + timeout: number + } + } + | { + name: "StructuredOutputError" + data: { + message: string + retries: number + } + } + | { + name: "ContextOverflowError" + data: { + message: string + responseBody?: string + } + } + | { + name: "APIError" + data: { + message: string + statusCode?: number + isRetryable: boolean + responseHeaders?: { + [key: string]: string + } + responseBody?: string + metadata?: { + [key: string]: string + } + } + } + parentID: string + modelID: string + providerID: string + mode: string + agent: string + path: { + cwd: string + root: string + } + summary?: boolean + cost: number + tokens: { + total?: number + input: number + output: number + reasoning: number + cache: { + read: number + write: number + } + } + usageStatus?: "reported" | "missing" | "estimated" + structured?: unknown + variant?: string + finish?: string +} + +export type EventSessionStructuredOutput = { + type: "session.structured_output" + properties: { + sessionID: string + outcome: + | { + status: "accepted" + attempts: number + value: unknown + } + | { + status: "failed" + reason: "exhausted" | "missing" | "step_limit" | "aborted" | "error" + attempts: number + } + } +} + +export type EventSessionStructuredOutputRejected = { + type: "session.structured_output_rejected" + properties: { + sessionID: string + attempt: number + maxAttempts: number + errors: Array<{ + path: string + keyword: string + message: string + }> + } +} From 51b19bce2f0d179a0d30fe86ac4142f6ce65b4cf Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 7 Oct 2026 11:40:49 +0100 Subject: [PATCH 02/11] feat(cli): expose validated output schema and result channels --- EVENTS.md | 65 +++ README.md | 31 ++ packages/cli/README.md | 43 +- packages/cli/src/cli/cmd/run.output.ts | 50 +++ packages/cli/src/cli/cmd/run.ts | 141 ++++-- .../cli/test/cli/run-output-schema.test.ts | 420 ++++++++++++++++++ 6 files changed, 720 insertions(+), 30 deletions(-) create mode 100644 packages/cli/src/cli/cmd/run.output.ts create mode 100644 packages/cli/test/cli/run-output-schema.test.ts diff --git a/EVENTS.md b/EVENTS.md index e3b72014..4bc0de55 100644 --- a/EVENTS.md +++ b/EVENTS.md @@ -530,3 +530,68 @@ Emitted when a permission request resolves to `allow` (either by matching an `al "input": { "command": "ls" } } ``` + +## Structured final results + +These events are additive in schema version `"1"` and emitted only when +`--output-schema ` is supplied. Other runs retain their event types and order. +The schema is validated before any model request, with Ajv defaults and +`allErrors: true` (draft-07, or Ajv2020 for a declared draft 2020-12). Validation +does not coerce types, insert defaults or remove extra properties. + +`--output-schema-retries ` permits N additional corrective attempts (default 2). +Zero permits exactly one attempt. `--output-result ` writes only accepted +JSON using two-space indentation and a trailing newline, with a temporary file +in the destination directory followed by an atomic rename. Failures preserve the +file. Both flags require `--output-schema`; the destination directory must exist. +Without `--output-result`, formatted mode prints final JSON; NDJSON includes the +value in the terminal event below. + +### `structured_output_rejected` + +Emitted once per rejected StructuredOutput attempt, including unparseable JSON: + +```json +{ + "type": "structured_output_rejected", + "attempt": 1, + "maxAttempts": 3, + "errors": [{ "path": "/count", "keyword": "type", "message": "must be integer" }] +} +``` + +The usual event envelope fields also apply. `attempt` is one-based; +`maxAttempts` equals `retryCount + 1`. Diagnostics include at most ten errors, +with bounded JSON-pointer paths, keywords and messages. Their encoded payload +is under 2 KB and never includes submitted values or raw JSON arguments. +Feedback goes to the model in the same session through the `invalid` tool. +Other tools remain usable until a valid final result; their outputs are not final +results. Provider-compatible tool schemas may be transformed, while local +validation always uses the canonical schema. + +### `structured_output` + +Exactly one terminal event per schema-enabled session, before `session_complete`: + +```json +{ "type": "structured_output", "status": "accepted", "attempts": 2, "value": { "result": "ok" } } +``` + +or: + +```json +{ "type": "structured_output", "status": "failed", "reason": "exhausted", "attempts": 3 } +``` + +`attempts` counts StructuredOutput calls, including rejected/unparseable ones; +prose and other tool calls do not count. Failure reasons are `exhausted`, `missing`, +`step_limit`, `aborted`, or `error` (including provider failures and stream idle +timeouts). Failed events have no `value`. No prose fallback is accepted. +Configuration failures happen before a session starts and use the existing +`invocation_error`/`invocation_complete` envelope, with code `OUTPUT_SCHEMA_CONFIG`. + +Exit codes are **0** on accepted results, **2** for schema configuration errors, +**3** for exhausted attempts, missing output and step limits. Provider errors and +timeouts keep **1**; SIGINT and SIGTERM keep **130** and **143**. +`StructuredOutputError.data.retries` records the number of corrective +StructuredOutput attempts used (initial attempt excluded). diff --git a/README.md b/README.md index 10114526..356bd3f2 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,37 @@ For programmatic consumption, use `--format json` to get raw events. aictrl run --format json "review this PR" | jq '.type' ``` +### Validated Final Results + +`aictrl run --output-schema ` enforces a JSON Schema on the final result +while the agent can continue using tools. The schema root must be `type: "object"`. +Schemas use Ajv defaults (strict validation, draft-07); a declared draft 2020-12 +schema uses Ajv2020. Invalid configuration exits before any model request. + +```bash +aictrl run --output-schema result.schema.json --output-schema-retries 2 \ + --output-result result.json "review this change" +``` + +- `--output-schema `: JSON Schema file, forwarded in local and `--attach` runs. +- `--output-schema-retries `: additional corrective attempts, integer >= 0, + default 2. Zero permits one attempt; N permits at most N+1 attempts. +- `--output-result `: atomically write only the validated JSON value, + indented with two spaces and a trailing newline. Failures preserve an existing + file and leave an absent file absent. The destination directory must exist. + +The retries and result flags require `--output-schema`. Without a result file, +formatted mode prints validated JSON at the end; `--format json` emits it in a +terminal `structured_output` event. Rejections emit `structured_output_rejected` +with bounded JSON-pointer diagnostics. See [EVENTS.md](EVENTS.md) for the contract. +No valid result means failure, including missing output and the agent step limit; +there is no prose fallback. Earlier tools are not replayed for corrective attempts. + +Exit codes: **0** for accepted results, **2** for schema configuration errors, +**3** for exhausted attempts, missing output or a step limit without a result. +Provider failures and stream timeouts retain **1**; SIGINT/SIGTERM retain +**130**/**143**. Invocations without `--output-schema` keep their existing output. + ### Non-Interactive Execution In headless mode, Aictrl automatically rejects all interactive permission requests (like `question` or `plan_enter`), ensuring your pipelines never hang. diff --git a/packages/cli/README.md b/packages/cli/README.md index a43bcd80..3d8dcac3 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -28,6 +28,37 @@ For programmatic pipes: aictrl run --format json "scan for secrets" | jq '.properties.part.text' ``` +### Validated Final Results + +`aictrl run --output-schema ` enforces a JSON Schema on the final result +while the agent can continue using tools. The schema root must be `type: "object"`. +Schemas use Ajv defaults (strict validation, draft-07); a declared draft 2020-12 +schema uses Ajv2020. Invalid configuration exits before any model request. + +```bash +aictrl run --output-schema result.schema.json --output-schema-retries 2 \ + --output-result result.json "review this change" +``` + +- `--output-schema `: JSON Schema file, forwarded in local and `--attach` runs. +- `--output-schema-retries `: additional corrective attempts, integer >= 0, + default 2. Zero permits one attempt; N permits at most N+1 attempts. +- `--output-result `: atomically write only the validated JSON value, + indented with two spaces and a trailing newline. Failures preserve an existing + file and leave an absent file absent. The destination directory must exist. + +The retries and result flags require `--output-schema`. Without a result file, +formatted mode prints validated JSON at the end; `--format json` emits it in a +terminal `structured_output` event. Rejections emit `structured_output_rejected` +with bounded JSON-pointer diagnostics. See [EVENTS.md](../../EVENTS.md) for the contract. +No valid result means failure, including missing output and the agent step limit; +there is no prose fallback. Earlier tools are not replayed for corrective attempts. + +Exit codes: **0** for accepted results, **2** for schema configuration errors, +**3** for exhausted attempts, missing output or a step limit without a result. +Provider failures and stream timeouts retain **1**; SIGINT/SIGTERM retain +**130**/**143**. Invocations without `--output-schema` keep their existing output. + ### Auto-Reject Permissions In headless mode, Aictrl automatically rejects permissions that would otherwise prompt a user. @@ -41,6 +72,7 @@ aictrl github install ``` This adds a GitHub Action that can: + 1. Commmit and push code changes. 2. Create and update PRs. 3. Respond to issue comments. @@ -53,6 +85,7 @@ This adds a GitHub Action that can: ```bash aictrl pr ``` + Automatically fetches the PR branch and imports the agent session used to create it. ## Configuration @@ -61,11 +94,11 @@ Aictrl reads config from `.aictrl/` (project) or `~/.config/aictrl/` (global). ### Models -| Provider | Env Var | -|----------|---------| -| Anthropic | `ANTHROPIC_API_KEY` | -| OpenAI | `OPENAI_API_KEY` | -| Google | `GOOGLE_API_KEY` | +| Provider | Env Var | +| ---------- | -------------------- | +| Anthropic | `ANTHROPIC_API_KEY` | +| OpenAI | `OPENAI_API_KEY` | +| Google | `GOOGLE_API_KEY` | | OpenRouter | `OPENROUTER_API_KEY` | Use `--variant` to select provider-specific reasoning effort. For OpenAI-compatible diff --git a/packages/cli/src/cli/cmd/run.output.ts b/packages/cli/src/cli/cmd/run.output.ts new file mode 100644 index 00000000..0e386cba --- /dev/null +++ b/packages/cli/src/cli/cmd/run.output.ts @@ -0,0 +1,50 @@ +import path from "path" +import fs from "fs/promises" +import { OutputSchema } from "../../session/output-schema" + +export const OUTPUT_CONFIG_EXIT = 2 +export const OUTPUT_FAILED_EXIT = 3 + +export async function outputSchema(input: { schema?: string; retries?: number; result?: string }) { + if (!input.schema) { + if (input.retries !== undefined || input.result !== undefined) + throw new Error("--output-schema-retries and --output-result require --output-schema") + return + } + if (input.retries !== undefined && (!Number.isSafeInteger(input.retries) || input.retries < 0)) + throw new Error(`${input.schema}: --output-schema-retries must be an integer >= 0`) + const text = await Bun.file(input.schema) + .text() + .catch(() => { + throw new Error(`${input.schema}: cannot read schema file`) + }) + const schema: unknown = (() => { + try { + return JSON.parse(text) + } catch { + throw new Error(`${input.schema}: invalid JSON`) + } + })() + if (!schema || typeof schema !== "object" || Array.isArray(schema)) + throw new Error(`${input.schema}: schema root must have type "object"`) + const canonical = schema as Record + const validate = (() => { + try { + return OutputSchema.compile(canonical) + } catch (error) { + throw new Error(`${input.schema}: ${error instanceof Error ? error.message : "cannot compile schema"}`) + } + })() + return { format: { type: "json_schema" as const, schema: canonical, retryCount: input.retries ?? 2 }, validate } +} + +export async function outputResult(file: string, value: unknown, aborted = () => false) { + const temp = path.join(path.dirname(file), `.${path.basename(file)}.${crypto.randomUUID()}.tmp`) + try { + await Bun.write(temp, JSON.stringify(value, null, 2) + "\n", { createPath: false }) + if (aborted()) throw new Error("Structured output cancelled") + await fs.rename(temp, file) + } finally { + await fs.rm(temp, { force: true }) + } +} diff --git a/packages/cli/src/cli/cmd/run.ts b/packages/cli/src/cli/cmd/run.ts index 90eac2c2..e4b5f825 100644 --- a/packages/cli/src/cli/cmd/run.ts +++ b/packages/cli/src/cli/cmd/run.ts @@ -1,3 +1,5 @@ +import { outputSchema, outputResult, OUTPUT_CONFIG_EXIT, OUTPUT_FAILED_EXIT } from "./run.output" +import { OutputSchema } from "../../session/output-schema" import type { Argv } from "yargs" import path from "path" import { pathToFileURL } from "bun" @@ -347,6 +349,12 @@ export const RunCommand = cmd({ default: "default", describe: "format: default (formatted) or json (raw JSON events)", }) + .option("output-schema", { type: "string", describe: "JSON Schema file for the final result (object root)" }) + .option("output-schema-retries", { type: "number", describe: "additional corrective attempts (default: 2)" }) + .option("output-result", { + type: "string", + describe: "atomically write only the validated JSON result to this file", + }) .option("file", { alias: ["f"], type: "string", @@ -382,10 +390,10 @@ export const RunCommand = cmd({ handler: async (args) => { const invocation = createRunInvocation(args.format === "json") - async function fail(message: string, code: string) { + async function fail(message: string, code: string, exit = 1) { UI.error(message) await invocation.abort(message, code) - process.exit(1) + process.exit(exit) } let message = [...args.message, ...(args["--"] || [])] @@ -403,6 +411,19 @@ export const RunCommand = cmd({ } })() + const schema = await outputSchema({ + schema: args["output-schema"], + retries: args["output-schema-retries"], + result: args["output-result"], + }).catch(async (error: unknown) => { + await fail( + error instanceof Error ? error.message : "Invalid output schema", + "OUTPUT_SCHEMA_CONFIG", + OUTPUT_CONFIG_EXIT, + ) + return undefined + }) + const files: { type: "file"; url: string; filename: string; mime: string }[] = [] if (args.file) { const list = Array.isArray(args.file) ? args.file : [args.file] @@ -468,14 +489,16 @@ export const RunCommand = cmd({ const baseID = args.continue ? (await sdk.session.list()).data?.find((s: any) => !s.parentID)?.id : args.session if (baseID && args.fork) { - const forked = await sdk.session.fork({ sessionID: baseID }) + const forked = await sdk.session.fork(schema && args.attach ? { path: { id: baseID } } : { sessionID: baseID }) return forked.data?.id } if (baseID) return baseID const name = title() - const result = await sdk.session.create({ title: name, permission: rules }) + const result = await sdk.session.create( + schema && args.attach ? { body: { title: name, permission: rules } } : { title: name, permission: rules }, + ) return result.data?.id } @@ -483,12 +506,14 @@ export const RunCommand = cmd({ const cfg = await sdk.config.get() if (!cfg.data) return if (cfg.data.share !== "auto" && !args.share) return - const res = await sdk.session.share({ sessionID }).catch((error: any) => { - if (error instanceof Error && error.message.includes("disabled")) { - UI.println(UI.Style.TEXT_DANGER_BOLD + "! " + error.message) - } - return { error } - }) + const res = await sdk.session + .share(schema && args.attach ? { path: { id: sessionID } } : { sessionID }) + .catch((error: any) => { + if (error instanceof Error && error.message.includes("disabled")) { + UI.println(UI.Style.TEXT_DANGER_BOLD + "! " + error.message) + } + return { error } + }) if (!res.error && "data" in res && res.data?.share?.url) { UI.println(UI.Style.TEXT_INFO_BOLD + "~ " + res.data.share.url) } @@ -551,10 +576,31 @@ export const RunCommand = cmd({ // Keep every failure path on one ordered, idempotent terminal lifecycle. const output = terminal(emit) - function complete(message = error ?? null) { + let structured: OutputSchema.Outcome | undefined + let pending: OutputSchema.Outcome | undefined + let attempts = 0 + async function complete(message = error ?? null) { + if (schema && !structured) { + if (pending?.status === "accepted" && !message && !control.current && schema.validate(pending.value)) { + if (args["output-result"]) await outputResult(args["output-result"], pending.value, () => !!control.current) + structured = pending + if (!emit("structured_output", structured) && !args["output-result"]) + process.stdout.write(JSON.stringify(structured.value, null, 2) + EOL) + } else { + structured = control.current + ? { status: "failed", reason: "aborted", attempts } + : pending?.status === "failed" + ? pending + : { status: "failed", reason: "error", attempts } + emit("structured_output", structured) + error ??= `Structured output failed: ${structured.reason}` + invocation.error(error) + if (!control.current && !process.exitCode) process.exitCode = OUTPUT_FAILED_EXIT + } + } output.complete({ durationMs: Date.now() - startTime, - error: message, + error: error ?? message, }) } @@ -566,6 +612,30 @@ export const RunCommand = cmd({ const toggles = new Map() eventsLoop: for await (const event of events.stream) { + if ( + schema && + event.type === "session.structured_output_rejected" && + event.properties.sessionID === sessionID + ) { + attempts = event.properties.attempt + emit("structured_output_rejected", { + attempt: attempts, + maxAttempts: event.properties.maxAttempts, + errors: event.properties.errors, + }) + continue + } + if ( + schema && + event.type === "session.structured_output" && + event.properties.sessionID === sessionID && + !structured + ) { + pending = OutputSchema.Outcome.parse(event.properties.outcome) + attempts = pending.attempts + continue + } + if (event.type === "session.status") { const status = event.properties.status switch (status.type) { @@ -730,7 +800,10 @@ export const RunCommand = cmd({ // to a non-zero exit code so CI wrappers see the failure instead of a // spuriously-green job. process.exitCode (not process.exit) lets the // loop drain to session.status idle and emit session_complete first. - if (!control.current) process.exitCode = 1 + if (!control.current) { + process.exitCode = 1 + if (props.error.name === "StructuredOutputError") process.exitCode = OUTPUT_FAILED_EXIT + } invocation.error(props.error) const classified = classifySessionError(props.error) // Structured session_error is the telemetry/CI channel for the @@ -864,7 +937,7 @@ export const RunCommand = cmd({ if (aborted) return aborted = true attempt( - () => sdk.session.abort({ sessionID }), + () => sdk.session.abort(schema && args.attach ? { path: { id: sessionID } } : { sessionID }), () => Log.Default.error("session abort failed"), ) } @@ -877,19 +950,19 @@ export const RunCommand = cmd({ } async function expire(signal: Signals.Info) { - complete(error ?? signal.message) + await complete(error ?? signal.message) await invocation.abort(signal.message) await Shutdown.flush() } using control = signals(interrupt, 5_000, expire) - function reject(cause: unknown) { + async function reject(cause: unknown) { const classified = classifySessionError(cause) error ??= classified.message invocation.error(cause) report(classified.reason, classified.code, classified.message) - complete(error) + await complete(error) if (control.current) { Log.Default.error("run failed after signal", { reason: classified.reason, @@ -897,7 +970,7 @@ export const RunCommand = cmd({ }) return } - console.error(cause) + console.error(schema ? "Structured output run failed" : cause) process.exitCode = 1 } @@ -929,7 +1002,9 @@ export const RunCommand = cmd({ }) } - const loopDone = loop().then(() => complete(), reject) + const loopDone = loop() + .then(() => complete()) + .catch(reject) if (control.current) { await loopDone @@ -938,23 +1013,37 @@ export const RunCommand = cmd({ } if (args.command) { - await sdk.session.command({ - sessionID, + const opts = { agent, model: args.model, command: args.command, arguments: message, variant: args.variant, - }) + ...(schema ? { format: schema.format } : {}), + } + const request = sdk.session.command( + schema && args.attach ? { path: { id: sessionID }, body: opts, throwOnError: true } : { sessionID, ...opts }, + ) + await (schema ? request.catch(reject) : request) } else { const model = args.model ? Provider.parseModel(args.model) : undefined - await sdk.session.prompt({ - sessionID, + const opts = { agent, model, variant: args.variant, parts: [...files, { type: "text", text: message }], - }) + ...(schema ? { format: schema.format } : {}), + } + const request = sdk.session.prompt( + schema && args.attach ? { path: { id: sessionID }, body: opts, throwOnError: true } : { sessionID, ...opts }, + ) + await (schema ? request.catch(reject) : request) + } + + if (schema && error) { + abort() + await Shutdown.flush() + return } // Race loopDone against promptResult to handle early prompt failures. @@ -1006,6 +1095,7 @@ export const RunCommand = cmd({ agent: opts.agent, model: opts.model, variant: opts.variant, + ...(opts.format ? { format: opts.format } : {}), }) }, async command(opts: any) { @@ -1016,6 +1106,7 @@ export const RunCommand = cmd({ agent: opts.agent, model: opts.model, variant: opts.variant, + ...(opts.format ? { format: opts.format } : {}), }) }, }, diff --git a/packages/cli/test/cli/run-output-schema.test.ts b/packages/cli/test/cli/run-output-schema.test.ts new file mode 100644 index 00000000..4cf627b9 --- /dev/null +++ b/packages/cli/test/cli/run-output-schema.test.ts @@ -0,0 +1,420 @@ +import { expect, test } from "bun:test" +import path from "path" +import fs from "fs/promises" +import { tmpdir } from "../fixture/fixture" +import { schema } from "../fixture/output-schema" +import { outputResult } from "../../src/cli/cmd/run.output" + +const entry = path.resolve(import.meta.dir, "../../src/index.ts") + +async function run( + options: { + flags?: string[] + json?: boolean + inputs?: readonly string[] + content?: string + existing?: string + steps?: number + timeout?: boolean + abort?: boolean + failure?: boolean + } = {}, +) { + await using tmp = await tmpdir() + const requests: Record[] = [] + let notify: () => void = () => {} + const called = new Promise((resolve) => { + notify = resolve + }) + const server = Bun.serve({ + port: 0, + async fetch(request) { + requests.push((await request.json()) as Record) + notify() + if (options.failure) + return new Response(JSON.stringify({ error: { message: "Fixture provider failed" } }), { + status: 400, + headers: { "Content-Type": "application/json" }, + }) + if (options.abort || options.timeout) + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode( + 'data: {"choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}\n\n', + ), + ) + }, + }), + { headers: { "Content-Type": "text/event-stream" } }, + ) + const args = (options.inputs ?? ['{"result":"accepted"}'])[ + Math.min(requests.length - 1, (options.inputs?.length ?? 1) - 1) + ] + const chunks = [ + { choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }] }, + { + choices: [ + { + index: 0, + delta: + args === "prose" + ? { content: "Plain response" } + : { + tool_calls: [ + { + index: 0, + id: `call_${requests.length}`, + type: "function", + function: { name: "StructuredOutput", arguments: args }, + }, + ], + }, + finish_reason: null, + }, + ], + }, + { + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }, + ] + return new Response( + chunks + .map((chunk) => `data: ${JSON.stringify(chunk)}`) + .concat("data: [DONE]") + .join("\n\n") + "\n\n", + { headers: { "Content-Type": "text/event-stream" } }, + ) + }, + }) + await Bun.write( + path.join(tmp.path, "aictrl.json"), + JSON.stringify({ + enabled_providers: ["alibaba"], + provider: { alibaba: { options: { apiKey: "test-key", baseURL: `${server.url.origin}/v1` } } }, + agent: { title: { disable: true }, ...(options.steps ? { build: { steps: options.steps } } : {}) }, + }), + ) + await Bun.write(path.join(tmp.path, "schema.json"), options.content ?? JSON.stringify(schema)) + if (options.existing) await Bun.write(path.join(tmp.path, "result.json"), options.existing) + const home = path.join(tmp.path, "home") + await fs.mkdir(home) + const proc = Bun.spawn( + [ + "bun", + "run", + "--conditions=browser", + entry, + "run", + ...(options.json === false ? [] : ["--format", "json"]), + "--model", + "alibaba/qwen-plus", + "--title", + "Schema fixture", + ...(options.flags ?? ["--output-schema", "schema.json"]), + "Return a result.", + ], + { + cwd: tmp.path, + env: { + ...process.env, + HOME: home, + XDG_CONFIG_HOME: path.join(home, ".config"), + XDG_DATA_HOME: path.join(home, ".local/share"), + XDG_CACHE_HOME: path.join(home, ".cache"), + AICTRL_TEST_HOME: home, + AICTRL_DISABLE_DEFAULT_PLUGINS: "true", + AICTRL_DISABLE_MODELS_FETCH: "true", + AICTRL_DISABLE_AUTOCOMPACT: "true", + AICTRL_MODEL_STREAM_IDLE_TIMEOUT_MS: options.timeout ? "100" : "0", + }, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }, + ) + const timer = setTimeout(() => proc.kill("SIGKILL"), 20000) + if (options.abort) void called.then(() => proc.kill("SIGTERM")) + try { + const [stdout, stderr, exit] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]) + const result = await Bun.file(path.join(tmp.path, "result.json")) + .text() + .catch(() => undefined) + const files = await fs.readdir(tmp.path) + const events = stdout + .split("\n") + .filter((line) => options.json !== false && line.startsWith("{")) + .map((line) => JSON.parse(line)) as Record[] + return { stdout, stderr, exit, requests, result, events, files } + } finally { + clearTimeout(timer) + proc.kill("SIGKILL") + server.stop(true) + } +} + +for (const [name, options, reason] of [ + ["unreadable", { flags: ["--output-schema", "absent.json"] }, "cannot read"], + ["invalid JSON", { content: "{broken" }, "invalid JSON"], + ["uncompilable", { content: '{"type":"object","unknownKeyword":true}' }, "strict mode"], + ["non-object root", { content: '{"type":"array"}' }, "object"], + ["retries without schema", { flags: ["--output-schema-retries", "1"] }, "require --output-schema"], + ["result without schema", { flags: ["--output-result", "result.json"] }, "require --output-schema"], + ["negative retries", { flags: ["--output-schema", "schema.json", "--output-schema-retries", "-1"] }, "integer >= 0"], + [ + "fractional retries", + { flags: ["--output-schema", "schema.json", "--output-schema-retries", "1.5"] }, + "integer >= 0", + ], + [ + "unsupported draft", + { content: '{"type":"object","$schema":"https://json-schema.org/draft/2019-09/schema"}' }, + "schema", + ], +] as const) { + test(`configuration error ${name} happens before any model request`, async () => { + const result = await run({ ...options, flags: "flags" in options ? [...options.flags] : undefined }) + expect(result.exit, result.stderr).toBe(2) + expect(result.stderr).toContain(reason) + if ("content" in options) expect(result.stderr).toContain("schema.json") + expect(result.requests).toEqual([]) + expect(result.result).toBeUndefined() + expect(result.events.map((event) => event.type)).toEqual([ + "invocation_start", + "invocation_error", + "invocation_complete", + ]) + }, 25000) +} + +test("repair emits bounded events and writes only validated JSON atomically", async () => { + const result = await run({ + inputs: ["{}", '{"result":"accepted"}'], + existing: "old result", + flags: ["--output-schema", "schema.json", "--output-result", "result.json"], + }) + expect(result.exit, result.stderr).toBe(0) + expect(result.result).toBe(JSON.stringify({ result: "accepted" }, null, 2) + "\n") + expect(result.files.filter((file) => file.endsWith(".tmp"))).toEqual([]) + expect(result.events.filter((event) => event.type === "structured_output_rejected")).toMatchObject([ + { attempt: 1, maxAttempts: 3, errors: [{ path: "", keyword: "required" }] }, + ]) + expect(result.events.filter((event) => event.type === "structured_output")).toMatchObject([ + { status: "accepted", attempts: 2, value: { result: "accepted" } }, + ]) + expect(result.events.findIndex((event) => event.type === "structured_output")).toBeLessThan( + result.events.findIndex((event) => event.type === "session_complete"), + ) +}, 25000) + +for (const existing of [undefined, "untouched"]) { + test(`exhaustion preserves ${existing ? "existing" : "absent"} result file`, async () => { + const result = await run({ + existing, + inputs: ["{}"], + flags: ["--output-schema", "schema.json", "--output-schema-retries", "0", "--output-result", "result.json"], + }) + expect(result.exit, result.stderr).toBe(3) + expect(result.requests).toHaveLength(1) + expect(result.result).toBe(existing) + expect(result.events.filter((event) => event.type === "structured_output")).toMatchObject([ + { status: "failed", reason: "exhausted", attempts: 1 }, + ]) + }, 25000) +} + +for (const [name, options, code, reason] of [ + ["missing", { inputs: ["prose"] }, 3, "missing"], + ["step limit", { inputs: ["{}"], steps: 1 }, 3, "step_limit"], + ["provider error", { failure: true }, 1, "error"], + ["timeout", { timeout: true }, 1, "error"], + ["abort", { abort: true }, 143, "aborted"], +] as const) { + test(`CLI ${name} emits one terminal failure and leaves result unchanged`, async () => { + const result = await run({ + ...options, + existing: "untouched", + flags: ["--output-schema", "schema.json", "--output-result", "result.json"], + }) + expect(result.exit, result.stderr).toBe(code) + expect(result.result).toBe("untouched") + expect(result.events.filter((event) => event.type === "structured_output")).toMatchObject([ + { status: "failed", reason }, + ]) + expect(result.events.filter((event) => event.type === "structured_output")).toHaveLength(1) + expect(result.stdout).not.toContain('"status":"accepted"') + }, 25000) +} + +test("default mode prints validated JSON at the end without a result file", async () => { + const result = await run({ json: false }) + expect(result.exit, result.stderr).toBe(0) + expect(result.stdout.trimEnd()).toEndWith(JSON.stringify({ result: "accepted" }, null, 2)) + expect(result.result).toBeUndefined() +}, 25000) + +test("plain run pins unchanged event types and ordering", async () => { + const result = await run({ inputs: ["prose"], flags: [] }) + expect(result.exit, result.stderr).toBe(0) + expect(result.events.map((event) => event.type)).toEqual([ + "invocation_start", + "session_start", + "tool_catalog", + "step_start", + "text", + "step_finish", + "message_complete", + "session_complete", + "invocation_complete", + ]) + expect(result.requests).toHaveLength(1) + expect(result.result).toBeUndefined() +}, 25000) + +test("atomic output cancellation cleans temp and preserves destination", async () => { + await using tmp = await tmpdir() + const file = path.join(tmp.path, "result.json") + await Bun.write(file, "untouched") + await expect(outputResult(file, { result: "accepted" }, () => true)).rejects.toThrow("cancelled") + expect(await Bun.file(file).text()).toBe("untouched") + expect(await fs.readdir(tmp.path)).toEqual(["result.json"]) +}) + +for (const [name, value, failure, code] of [ + ["accepted", { result: "attached" }, false, 0], + ["invalid remote result", {}, false, 3], + ["late session failure", { result: "attached" }, true, 1], +] as const) { + test(`attach ${name} forwards the canonical schema and validates final acceptance`, async () => { + await using tmp = await tmpdir() + await Bun.write(path.join(tmp.path, "schema.json"), JSON.stringify(schema)) + const requests: { url: string; body: Record }[] = [] + let controller: ReadableStreamDefaultController | undefined + const server = Bun.serve({ + port: 0, + async fetch(request) { + const url = new URL(request.url).pathname + if (url === "/event") + return new Response( + new ReadableStream({ + start(value) { + controller = value + controller.enqueue(new TextEncoder().encode(": ready\n\n")) + }, + }), + { headers: { "Content-Type": "text/event-stream" } }, + ) + if (url === "/config") return Response.json({ share: "disabled" }) + if (url.endsWith("/abort")) return Response.json(true) + const body = request.method === "POST" ? ((await request.json()) as Record) : {} + requests.push({ url, body }) + if (url === "/session") return Response.json({ id: "ses_fixture" }) + if (url === "/session/ses_fixture/message") { + for (const event of [ + { + type: "session.structured_output_rejected", + properties: { + sessionID: "ses_fixture", + attempt: 1, + maxAttempts: 5, + errors: [{ path: "", keyword: "required", message: "must have required property 'result'" }], + }, + }, + { + type: "session.structured_output", + properties: { + sessionID: "ses_fixture", + outcome: { status: "accepted", attempts: 2, value }, + }, + }, + ...(failure + ? [ + { + type: "session.error", + properties: { + sessionID: "ses_fixture", + error: { name: "APIError", data: { message: "Late provider failure", isRetryable: false } }, + }, + }, + ] + : []), + { type: "session.status", properties: { sessionID: "ses_fixture", status: { type: "idle" } } }, + ]) + controller!.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(event)}\n\n`)) + return Response.json({}) + } + return new Response("unexpected request", { status: 400 }) + }, + }) + const proc = Bun.spawn( + [ + "bun", + "run", + "--conditions=browser", + entry, + "run", + "--attach", + server.url.origin, + "--format", + "json", + "--output-schema", + "schema.json", + "--output-schema-retries", + "4", + "--output-result", + "result.json", + "Return a result.", + ], + { + cwd: tmp.path, + env: { ...process.env, AICTRL_DISABLE_DEFAULT_PLUGINS: "true", AICTRL_DISABLE_MODELS_FETCH: "true" }, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }, + ) + const timer = setTimeout(() => proc.kill("SIGKILL"), 20000) + try { + const [stdout, stderr, exit] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]) + expect(exit, stderr + stdout).toBe(code) + expect(requests.find((request) => request.url.endsWith("/message"))?.body.format).toEqual({ + type: "json_schema", + schema, + retryCount: 4, + }) + expect(await Bun.file(path.join(tmp.path, "result.json")).exists()).toBe(code === 0) + if (code === 0) expect(await Bun.file(path.join(tmp.path, "result.json")).json()).toEqual(value) + const events = stdout + .split("\n") + .filter((line) => line.startsWith("{")) + .map((line) => JSON.parse(line)) + expect(events.filter((event) => event.type === "structured_output")).toMatchObject([ + code === 0 ? { status: "accepted", attempts: 2, value } : { status: "failed", reason: "error", attempts: 2 }, + ]) + expect(events.filter((event) => event.type === "structured_output")).toHaveLength(1) + } finally { + clearTimeout(timer) + proc.kill("SIGKILL") + server.stop(true) + } + }, 25000) +} + +test("JSON mode without output-result carries the accepted value only in terminal event", async () => { + const result = await run() + expect(result.exit, result.stderr).toBe(0) + expect(result.result).toBeUndefined() + expect(result.events.filter((event) => event.type === "structured_output")).toMatchObject([ + { status: "accepted", attempts: 1, value: { result: "accepted" } }, + ]) +}, 25000) From 7525fab9fbf5b4865a207a02048bfb4a58cce890 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 7 Oct 2026 11:51:45 +0100 Subject: [PATCH 03/11] fix(session): retry missing structured output within the attempt budget --- EVENTS.md | 16 +++-- README.md | 4 +- packages/cli/README.md | 4 +- packages/cli/src/session/prompt.ts | 45 ++++++++------ .../cli/test/session/output-schema.test.ts | 58 +++++++++++++++++-- 5 files changed, 98 insertions(+), 29 deletions(-) diff --git a/EVENTS.md b/EVENTS.md index 4bc0de55..f4dde186 100644 --- a/EVENTS.md +++ b/EVENTS.md @@ -549,7 +549,8 @@ value in the terminal event below. ### `structured_output_rejected` -Emitted once per rejected StructuredOutput attempt, including unparseable JSON: +Emitted once per rejected StructuredOutput attempt, including unparseable JSON +and a model turn that finishes in prose without a final tool call: ```json { @@ -564,7 +565,10 @@ The usual event envelope fields also apply. `attempt` is one-based; `maxAttempts` equals `retryCount + 1`. Diagnostics include at most ten errors, with bounded JSON-pointer paths, keywords and messages. Their encoded payload is under 2 KB and never includes submitted values or raw JSON arguments. -Feedback goes to the model in the same session through the `invalid` tool. +Validation feedback goes to the model in the same session through the `invalid` tool. +A prose-only finish emits the fixed diagnostic +`{ "path": "", "keyword": "missing", "message": "call StructuredOutput with the final result" }` +and receives an ephemeral reminder before the next turn while budget remains. Other tools remain usable until a valid final result; their outputs are not final results. Provider-compatible tool schemas may be transformed, while local validation always uses the canonical schema. @@ -583,8 +587,10 @@ or: { "type": "structured_output", "status": "failed", "reason": "exhausted", "attempts": 3 } ``` -`attempts` counts StructuredOutput calls, including rejected/unparseable ones; -prose and other tool calls do not count. Failure reasons are `exhausted`, `missing`, +`attempts` counts StructuredOutput calls, including rejected/unparseable ones, +and prose-only finishes without a final call. Other tool calls do not count. +When the budget is spent, the last rejection determines the reason: `missing` +for a prose-only finish, otherwise `exhausted`. Failure reasons are `exhausted`, `missing`, `step_limit`, `aborted`, or `error` (including provider failures and stream idle timeouts). Failed events have no `value`. No prose fallback is accepted. Configuration failures happen before a session starts and use the existing @@ -594,4 +600,4 @@ Exit codes are **0** on accepted results, **2** for schema configuration errors, **3** for exhausted attempts, missing output and step limits. Provider errors and timeouts keep **1**; SIGINT and SIGTERM keep **130** and **143**. `StructuredOutputError.data.retries` records the number of corrective -StructuredOutput attempts used (initial attempt excluded). +attempts used (initial attempt excluded), including prose-only corrective turns. diff --git a/README.md b/README.md index 356bd3f2..c74bca05 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,9 @@ formatted mode prints validated JSON at the end; `--format json` emits it in a terminal `structured_output` event. Rejections emit `structured_output_rejected` with bounded JSON-pointer diagnostics. See [EVENTS.md](EVENTS.md) for the contract. No valid result means failure, including missing output and the agent step limit; -there is no prose fallback. Earlier tools are not replayed for corrective attempts. +there is no prose fallback. A prose-only finish consumes one attempt and receives +a reminder to call StructuredOutput while retries remain. Earlier tools are not +replayed for corrective attempts. Exit codes: **0** for accepted results, **2** for schema configuration errors, **3** for exhausted attempts, missing output or a step limit without a result. diff --git a/packages/cli/README.md b/packages/cli/README.md index 3d8dcac3..dd41984e 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -52,7 +52,9 @@ formatted mode prints validated JSON at the end; `--format json` emits it in a terminal `structured_output` event. Rejections emit `structured_output_rejected` with bounded JSON-pointer diagnostics. See [EVENTS.md](../../EVENTS.md) for the contract. No valid result means failure, including missing output and the agent step limit; -there is no prose fallback. Earlier tools are not replayed for corrective attempts. +there is no prose fallback. A prose-only finish consumes one attempt and receives +a reminder to call StructuredOutput while retries remain. Earlier tools are not +replayed for corrective attempts. Exit codes: **0** for accepted results, **2** for schema configuration errors, **3** for exhausted attempts, missing output or a step limit without a result. diff --git a/packages/cli/src/session/prompt.ts b/packages/cli/src/session/prompt.ts index 2548d604..695ff506 100644 --- a/packages/cli/src/session/prompt.ts +++ b/packages/cli/src/session/prompt.ts @@ -330,6 +330,7 @@ export namespace SessionPrompt { let structuredOutput: unknown | undefined let attempts = 0 let rejected = 0 + let reminder = false let contract: | { id: string @@ -369,6 +370,7 @@ export namespace SessionPrompt { contract = { id: lastUser.id, format: lastUser.format, validate: OutputSchema.compile(lastUser.format.schema) } attempts = 0 rejected = 0 + reminder = false structuredOutput = undefined } const lastAssistantMsg = msgs.findLast((msg) => msg.info.id === lastAssistant?.id) @@ -376,7 +378,8 @@ export namespace SessionPrompt { lastAssistant && isModelFinished(lastAssistant.finish) && !hasToolCalls(lastAssistantMsg?.parts ?? []) && - lastUser.id < lastAssistant.id + lastUser.id < lastAssistant.id && + !reminder ) { log.info("exiting loop", { sessionID }) break @@ -703,9 +706,9 @@ export namespace SessionPrompt { messages: msgs, }) - const reject = async (input: string) => { + const reject = async (input: string | OutputSchema.Diagnostic[]) => { if (!contract) throw new Error("Missing output schema") - const errors = OutputSchema.parse(input, contract.validate) + const errors = typeof input === "string" ? OutputSchema.parse(input, contract.validate) : input if (attempts < contract.format.retryCount + 1) { attempts++ rejected++ @@ -780,6 +783,14 @@ export namespace SessionPrompt { system, messages: [ ...MessageV2.toModelMessages(msgs, model), + ...(reminder + ? [ + { + role: "user" as const, + content: "call StructuredOutput with the final result", + }, + ] + : []), ...(isLastStep ? [ { @@ -791,6 +802,7 @@ export namespace SessionPrompt { ], tools, model, + toolChoice: contract ? "required" : undefined, structured: contract ? { reject } : undefined, }) @@ -808,32 +820,29 @@ export namespace SessionPrompt { outcome = { status: "accepted", attempts, value: structuredOutput } break } + structuredOutput = undefined - if (contract && !processor.message.error && rejected > contract.format.retryCount) { - processor.message.error = new MessageV2.StructuredOutputError({ - message: "Structured output corrective attempts exhausted", - retries: Math.max(0, attempts - 1), - }).toObject() - await Session.updateMessage(processor.message) - await Bus.publish(Session.Event.Error, { sessionID, error: processor.message.error }) - outcome = { status: "failed", reason: "exhausted", attempts } - break + reminder = + !!contract && (await missingStructuredOutput(processor.message, () => MessageV2.parts(processor.message.id))) + if (reminder) { + await reject([{ path: "", keyword: "missing", message: "call StructuredOutput with the final result" }]) } - if ( - format.type === "json_schema" && - (await missingStructuredOutput(processor.message, () => MessageV2.parts(processor.message.id))) - ) { + if (contract && !processor.message.error && rejected > contract.format.retryCount) { processor.message.error = new MessageV2.StructuredOutputError({ - message: "Model did not produce structured output", + message: reminder + ? "Model did not produce structured output" + : "Structured output corrective attempts exhausted", retries: Math.max(0, attempts - 1), }).toObject() await Session.updateMessage(processor.message) await Bus.publish(Session.Event.Error, { sessionID, error: processor.message.error }) - outcome = { status: "failed", reason: "missing", attempts } + outcome = { status: "failed", reason: reminder ? "missing" : "exhausted", attempts } break } + if (reminder) continue + if (result === "stop") break if (result === "compact") { await SessionCompaction.create({ diff --git a/packages/cli/test/session/output-schema.test.ts b/packages/cli/test/session/output-schema.test.ts index 7d64ba05..682514b9 100644 --- a/packages/cli/test/session/output-schema.test.ts +++ b/packages/cli/test/session/output-schema.test.ts @@ -255,10 +255,20 @@ test("valid StructuredOutput wins when another tool is called in the same step", }, 15_000) test("model finish without StructuredOutput has an explicit missing outcome", async () => { - const result = await run(["prose"]) + const result = await run(["prose"], 0) + expect(result.requests).toHaveLength(1) + expect(result.requests[0].tool_choice).toBe("required") + expect(result.rejected).toEqual([ + { + sessionID: result.info.sessionID, + attempt: 1, + maxAttempts: 1, + errors: [{ path: "", keyword: "missing", message: "call StructuredOutput with the final result" }], + }, + ]) expect(result.info.structured).toBeUndefined() expect(result.info.error).toMatchObject({ name: "StructuredOutputError", data: { retries: 0 } }) - expect(result.outcomes).toEqual([{ status: "failed", reason: "missing", attempts: 0 }]) + expect(result.outcomes).toEqual([{ status: "failed", reason: "missing", attempts: 1 }]) }, 15_000) test("step cap without a valid result has an explicit failure", async () => { @@ -290,6 +300,7 @@ test("plain session keeps prose and emits no structured events", async () => { expect(result.outcomes).toEqual([]) expect(result.rejected).toEqual([]) expect(result.requests).toHaveLength(1) + expect(result.requests[0].tool_choice).toBe("auto") }, 15_000) test("Ajv verdict parity across hostile corpus without mutating values", () => { @@ -332,8 +343,10 @@ test("draft 2020-12 declaration with a fragment uses the same validator dialect" test("missing result after rejections reports corrective attempts truthfully", async () => { const result = await run(["{}", "{}", "prose"]) expect(result.info.structured).toBeUndefined() - expect(result.info.error).toMatchObject({ name: "StructuredOutputError", data: { retries: 1 } }) - expect(result.outcomes).toEqual([{ status: "failed", reason: "missing", attempts: 2 }]) + expect(result.info.error).toMatchObject({ name: "StructuredOutputError", data: { retries: 2 } }) + expect(result.requests).toHaveLength(3) + expect(result.rejected).toHaveLength(3) + expect(result.outcomes).toEqual([{ status: "failed", reason: "missing", attempts: 3 }]) }, 15_000) test("Gemini tool schema is transformed but canonical numeric enum remains authoritative", async () => { @@ -347,3 +360,40 @@ test("Gemini tool schema is transformed but canonical numeric enum remains autho }) expect(canonical.properties.result).toEqual({ type: "integer", enum: [1, 2] }) }, 15_000) + +test("prose-only finish is a counted corrective attempt before a valid result", async () => { + const result = await run(["prose", '{"result":"repaired"}'], 1) + expect(result.requests).toHaveLength(2) + result.requests.forEach((request) => expect(request.tool_choice).toBe("required")) + expect(result.info.structured).toEqual({ result: "repaired" }) + expect(result.info.error).toBeUndefined() + expect(result.rejected).toEqual([ + { + sessionID: result.info.sessionID, + attempt: 1, + maxAttempts: 2, + errors: [{ path: "", keyword: "missing", message: "call StructuredOutput with the final result" }], + }, + ]) + expect(JSON.stringify(result.requests[1].messages)).toContain("call StructuredOutput with the final result") + expect(result.outcomes).toEqual([{ status: "accepted", attempts: 2, value: { result: "repaired" } }]) +}, 15_000) + +test("a final validation rejection after a missing turn reports exhausted", async () => { + const result = await run(["prose", "{}"], 1) + expect(result.requests).toHaveLength(2) + expect(result.info.structured).toBeUndefined() + expect(result.info.error).toMatchObject({ name: "StructuredOutputError", data: { retries: 1 } }) + expect(result.rejected.map((event) => event.errors[0].keyword)).toEqual(["missing", "required"]) + expect(result.outcomes).toEqual([{ status: "failed", reason: "exhausted", attempts: 2 }]) +}, 15_000) + +test("repeated prose-only finishes spend the configured corrective budget", async () => { + const result = await run(["prose"], 2) + expect(result.requests).toHaveLength(3) + expect(result.rejected.map((event) => event.attempt)).toEqual([1, 2, 3]) + expect(result.rejected.every((event) => event.errors[0].keyword === "missing")).toBe(true) + expect(result.info.structured).toBeUndefined() + expect(result.info.error).toMatchObject({ name: "StructuredOutputError", data: { retries: 2 } }) + expect(result.outcomes).toEqual([{ status: "failed", reason: "missing", attempts: 3 }]) +}, 15_000) From c6b306c607a6e615e9a52d766a95ff77ab98c8ee Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 7 Oct 2026 11:51:55 +0100 Subject: [PATCH 04/11] fix(cli): use legacy attach request shapes consistently --- packages/cli/src/cli/cmd/run.ts | 35 ++-- .../cli/test/cli/run-output-schema.test.ts | 163 +++++++++++++++++- packages/sdk/script/build.ts | 18 +- 3 files changed, 190 insertions(+), 26 deletions(-) diff --git a/packages/cli/src/cli/cmd/run.ts b/packages/cli/src/cli/cmd/run.ts index e4b5f825..d05f1fde 100644 --- a/packages/cli/src/cli/cmd/run.ts +++ b/packages/cli/src/cli/cmd/run.ts @@ -485,20 +485,23 @@ export const RunCommand = cmd({ return message.slice(0, 50) + (message.length > 50 ? "..." : "") } + function input(id?: string, body?: Record) { + if (!args.attach) return { ...(id ? { sessionID: id } : {}), ...body } + return { ...(id ? { path: { id } } : {}), ...(body ? { body } : {}), throwOnError: true } + } + async function session(sdk: any) { const baseID = args.continue ? (await sdk.session.list()).data?.find((s: any) => !s.parentID)?.id : args.session if (baseID && args.fork) { - const forked = await sdk.session.fork(schema && args.attach ? { path: { id: baseID } } : { sessionID: baseID }) + const forked = await sdk.session.fork(input(baseID)) return forked.data?.id } if (baseID) return baseID const name = title() - const result = await sdk.session.create( - schema && args.attach ? { body: { title: name, permission: rules } } : { title: name, permission: rules }, - ) + const result = await sdk.session.create(input(undefined, { title: name, permission: rules })) return result.data?.id } @@ -506,14 +509,12 @@ export const RunCommand = cmd({ const cfg = await sdk.config.get() if (!cfg.data) return if (cfg.data.share !== "auto" && !args.share) return - const res = await sdk.session - .share(schema && args.attach ? { path: { id: sessionID } } : { sessionID }) - .catch((error: any) => { - if (error instanceof Error && error.message.includes("disabled")) { - UI.println(UI.Style.TEXT_DANGER_BOLD + "! " + error.message) - } - return { error } - }) + const res = await sdk.session.share(input(sessionID)).catch((error: any) => { + if (error instanceof Error && error.message.includes("disabled")) { + UI.println(UI.Style.TEXT_DANGER_BOLD + "! " + error.message) + } + return { error } + }) if (!res.error && "data" in res && res.data?.share?.url) { UI.println(UI.Style.TEXT_INFO_BOLD + "~ " + res.data.share.url) } @@ -937,7 +938,7 @@ export const RunCommand = cmd({ if (aborted) return aborted = true attempt( - () => sdk.session.abort(schema && args.attach ? { path: { id: sessionID } } : { sessionID }), + () => sdk.session.abort(input(sessionID)), () => Log.Default.error("session abort failed"), ) } @@ -1021,9 +1022,7 @@ export const RunCommand = cmd({ variant: args.variant, ...(schema ? { format: schema.format } : {}), } - const request = sdk.session.command( - schema && args.attach ? { path: { id: sessionID }, body: opts, throwOnError: true } : { sessionID, ...opts }, - ) + const request = sdk.session.command(input(sessionID, opts)) await (schema ? request.catch(reject) : request) } else { const model = args.model ? Provider.parseModel(args.model) : undefined @@ -1034,9 +1033,7 @@ export const RunCommand = cmd({ parts: [...files, { type: "text", text: message }], ...(schema ? { format: schema.format } : {}), } - const request = sdk.session.prompt( - schema && args.attach ? { path: { id: sessionID }, body: opts, throwOnError: true } : { sessionID, ...opts }, - ) + const request = sdk.session.prompt(input(sessionID, opts)) await (schema ? request.catch(reject) : request) } diff --git a/packages/cli/test/cli/run-output-schema.test.ts b/packages/cli/test/cli/run-output-schema.test.ts index 4cf627b9..34429454 100644 --- a/packages/cli/test/cli/run-output-schema.test.ts +++ b/packages/cli/test/cli/run-output-schema.test.ts @@ -18,6 +18,7 @@ async function run( timeout?: boolean abort?: boolean failure?: boolean + attach?: boolean } = {}, ) { await using tmp = await tmpdir() @@ -101,6 +102,65 @@ async function run( if (options.existing) await Bun.write(path.join(tmp.path, "result.json"), options.existing) const home = path.join(tmp.path, "home") await fs.mkdir(home) + const attached: { url: string; body: Record }[] = [] + // The headless package has no HTTP server; bridge its real session APIs rather + // than synthesizing responses or terminal events for the attach regression. + const bridge = options.attach + ? await (async () => { + const { Instance } = await import("../../src/project/instance") + const { Session } = await import("../../src/session") + const { SessionPrompt } = await import("../../src/session/prompt") + const { Config } = await import("../../src/config/config") + const { GlobalBus } = await import("../../src/bus/global") + return Bun.serve({ + port: 0, + idleTimeout: 0, + async fetch(request) { + const url = new URL(request.url).pathname + if (url === "/event") { + const send = (event: { directory?: string; payload: unknown }) => { + if (event.directory !== tmp.path) return + controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(event.payload)}\n\n`)) + } + let controller: ReadableStreamDefaultController + return new Response( + new ReadableStream({ + start(value) { + controller = value + GlobalBus.on("event", send) + controller.enqueue(new TextEncoder().encode(": ready\n\n")) + }, + cancel() { + GlobalBus.off("event", send) + }, + }), + { headers: { "Content-Type": "text/event-stream" } }, + ) + } + const body = + request.method === "POST" && request.headers.get("content-type")?.includes("json") + ? ((await request.json()) as Record) + : {} + attached.push({ url, body }) + return Instance.provide({ + directory: tmp.path, + fn: async () => { + if (url === "/config") return Response.json(await Config.get()) + if (url === "/session") return Response.json(await Session.create(Session.create.schema.parse(body))) + const id = url.split("/")[2] + if (url.endsWith("/abort")) { + SessionPrompt.cancel(id) + return Response.json(true) + } + const input = SessionPrompt.PromptInput.safeParse({ ...body, sessionID: id }) + if (!url.endsWith("/message") || !input.success) return new Response("Invalid prompt", { status: 400 }) + return Response.json(await SessionPrompt.prompt(input.data)) + }, + }) + }, + }) + })() + : undefined const proc = Bun.spawn( [ "bun", @@ -108,6 +168,7 @@ async function run( "--conditions=browser", entry, "run", + ...(bridge ? ["--attach", bridge.url.origin, "--dir", tmp.path] : []), ...(options.json === false ? [] : ["--format", "json"]), "--model", "alibaba/qwen-plus", @@ -151,11 +212,16 @@ async function run( .split("\n") .filter((line) => options.json !== false && line.startsWith("{")) .map((line) => JSON.parse(line)) as Record[] - return { stdout, stderr, exit, requests, result, events, files } + return { stdout, stderr, exit, requests, attached, result, events, files } } finally { clearTimeout(timer) proc.kill("SIGKILL") server.stop(true) + bridge?.stop(true) + if (bridge) { + const { Instance } = await import("../../src/project/instance") + await Instance.provide({ directory: tmp.path, fn: () => Instance.dispose() }) + } } } @@ -213,6 +279,53 @@ test("repair emits bounded events and writes only validated JSON atomically", as ) }, 25000) +for (const json of [true, false]) { + test(`prose finish is repaired through the CLI in ${json ? "JSON" : "formatted"} mode`, async () => { + const result = await run({ + json, + inputs: ["prose", '{"result":"accepted"}'], + flags: ["--output-schema", "schema.json", "--output-result", "result.json"], + }) + expect(result.exit, result.stderr).toBe(0) + expect(result.requests).toHaveLength(2) + expect(result.requests.map((request) => request.tool_choice)).toEqual(["required", "required"]) + expect(JSON.stringify(result.requests[1].messages)).toContain("call StructuredOutput with the final result") + expect(result.result).toBe(JSON.stringify({ result: "accepted" }, null, 2) + "\n") + if (!json) return + expect(result.events.filter((event) => event.type === "structured_output_rejected")).toMatchObject([ + { + attempt: 1, + maxAttempts: 3, + errors: [{ path: "", keyword: "missing", message: "call StructuredOutput with the final result" }], + }, + ]) + expect(result.events.filter((event) => event.type === "structured_output")).toMatchObject([ + { status: "accepted", attempts: 2, value: { result: "accepted" } }, + ]) + }, 25000) +} + +test("prose finish with zero retries fails after exactly one CLI provider request", async () => { + const result = await run({ + inputs: ["prose"], + flags: ["--output-schema", "schema.json", "--output-schema-retries", "0", "--output-result", "result.json"], + }) + expect(result.exit, result.stderr).toBe(3) + expect(result.requests).toHaveLength(1) + expect(result.requests[0].tool_choice).toBe("required") + expect(result.result).toBeUndefined() + expect(result.events.filter((event) => event.type === "structured_output_rejected")).toMatchObject([ + { + attempt: 1, + maxAttempts: 1, + errors: [{ path: "", keyword: "missing", message: "call StructuredOutput with the final result" }], + }, + ]) + expect(result.events.filter((event) => event.type === "structured_output")).toMatchObject([ + { status: "failed", reason: "missing", attempts: 1 }, + ]) +}, 25000) + for (const existing of [undefined, "untouched"]) { test(`exhaustion preserves ${existing ? "existing" : "absent"} result file`, async () => { const result = await run({ @@ -249,6 +362,11 @@ for (const [name, options, code, reason] of [ ]) expect(result.events.filter((event) => event.type === "structured_output")).toHaveLength(1) expect(result.stdout).not.toContain('"status":"accepted"') + if (name === "missing") { + expect(result.requests).toHaveLength(3) + expect(result.events.filter((event) => event.type === "structured_output_rejected")).toHaveLength(3) + expect(result.events.find((event) => event.type === "structured_output")).toMatchObject({ attempts: 3 }) + } }, 25000) } @@ -274,9 +392,52 @@ test("plain run pins unchanged event types and ordering", async () => { "invocation_complete", ]) expect(result.requests).toHaveLength(1) + expect(result.requests[0].tool_choice).toBe("auto") expect(result.result).toBeUndefined() }, 25000) +test("plain attach runs a prompt through the HTTP session boundary", async () => { + const result = await run({ attach: true, inputs: ["prose"], flags: [] }) + expect(result.attached.find((request) => request.url === "/session")?.body.title).toBe("Schema fixture") + expect(result.attached.find((request) => request.url === "/session")?.body.permission).toEqual([ + { permission: "question", action: "deny", pattern: "*" }, + { permission: "plan_enter", action: "deny", pattern: "*" }, + { permission: "plan_exit", action: "deny", pattern: "*" }, + ]) + expect(result.attached.find((request) => request.url.endsWith("/message"))?.body).toMatchObject({ + model: { providerID: "alibaba", modelID: "qwen-plus" }, + parts: [{ type: "text", text: expect.stringContaining("Return a result.") }], + }) + expect(result.exit, result.stderr + result.stdout).toBe(0) + expect(result.requests).toHaveLength(1) + expect(result.requests[0].tool_choice).toBe("auto") + expect(result.events.filter((event) => event.type === "text")).toMatchObject([{ part: { text: "Plain response" } }]) + expect(result.events.filter((event) => event.type === "structured_output")).toEqual([]) +}, 25000) + +test("schema attach repairs a prose finish through the HTTP session boundary", async () => { + const result = await run({ + attach: true, + inputs: ["prose", '{"result":"accepted"}'], + flags: ["--output-schema", "schema.json", "--output-result", "result.json"], + }) + expect(result.exit, result.stderr + result.stdout).toBe(0) + expect(result.attached.find((request) => request.url.endsWith("/message"))?.body.format).toEqual({ + type: "json_schema", + schema, + retryCount: 2, + }) + expect(result.requests).toHaveLength(2) + expect(result.requests.map((request) => request.tool_choice)).toEqual(["required", "required"]) + expect(result.result).toBe(JSON.stringify({ result: "accepted" }, null, 2) + "\n") + expect(result.events.filter((event) => event.type === "structured_output_rejected")).toMatchObject([ + { attempt: 1, errors: [{ path: "", keyword: "missing", message: "call StructuredOutput with the final result" }] }, + ]) + expect(result.events.filter((event) => event.type === "structured_output")).toMatchObject([ + { status: "accepted", attempts: 2, value: { result: "accepted" } }, + ]) +}, 25000) + test("atomic output cancellation cleans temp and preserves destination", async () => { await using tmp = await tmpdir() const file = path.join(tmp.path, "result.json") diff --git a/packages/sdk/script/build.ts b/packages/sdk/script/build.ts index a562626d..44d71479 100755 --- a/packages/sdk/script/build.ts +++ b/packages/sdk/script/build.ts @@ -38,8 +38,8 @@ await createClient({ ], }) -// The headless generator writes canonical v2 schemas. Keep the legacy HTTP -// client's shared types aligned without regenerating its existing endpoints. +// The headless OpenAPI has no paths, so it cannot regenerate legacy endpoints. +// Keep the legacy HTTP client's shared types aligned with canonical v2 schemas. const legacy = Bun.file("./src/gen/types.gen.ts") let source = await legacy.text() for (const name of [ @@ -54,22 +54,28 @@ for (const name of [ "m", ) const declaration = `export type ${name} = import("../v2/gen/types.gen.js").${name}` - source = pattern.test(source) ? source.replace(pattern, declaration) : source + "\n" + declaration + "\n" + if (!pattern.test(source)) throw new Error(`Legacy ${name} declaration not found`) + source = source.replace(pattern, declaration) } for (const name of ["SessionPromptData", "SessionPromptAsyncData", "SessionCommandData"]) { const start = source.indexOf(`export type ${name} = {`) const end = source.indexOf("\n path:", start) - if (start < 0 || end < 0) throw new Error(`Legacy ${name} declaration not found`) + const next = source.indexOf("\nexport type ", start) + if (start < 0 || end < 0 || (next >= 0 && end >= next)) throw new Error(`Legacy ${name} declaration not found`) const body = source.slice(start, end) - if (!body.includes("format?: OutputFormat")) + if (!body.includes("format?: OutputFormat")) { + if (!body.includes(" body?: {")) throw new Error(`Legacy ${name} body anchor not found`) source = source.slice(0, start) + body.replace(" body?: {", " body?: {\n format?: OutputFormat") + source.slice(end) + } } -if (!source.includes(" | EventSessionStructuredOutput\n")) +if (!source.includes(" | EventSessionStructuredOutput\n")) { + if (!source.includes("export type Event =\n")) throw new Error("Legacy Event declaration not found") source = source.replace( "export type Event =\n", "export type Event =\n | EventSessionStructuredOutput\n | EventSessionStructuredOutputRejected\n", ) +} await legacy.write(source) await $`bun prettier --write src/gen` From 52b94b745017d6db892a0f66f12511aab9df245f Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 7 Oct 2026 12:35:27 +0100 Subject: [PATCH 05/11] fix(provider): handle model catalog response body timeouts --- packages/cli/src/provider/models.ts | 14 ++++---- packages/cli/test/preload.ts | 2 ++ packages/cli/test/provider/models.test.ts | 40 +++++++++++++++++++++++ 3 files changed, 50 insertions(+), 6 deletions(-) create mode 100644 packages/cli/test/provider/models.test.ts diff --git a/packages/cli/src/provider/models.ts b/packages/cli/src/provider/models.ts index b5e319e4..b109f92d 100644 --- a/packages/cli/src/provider/models.ts +++ b/packages/cli/src/provider/models.ts @@ -109,13 +109,15 @@ export namespace ModelsDev { "User-Agent": Installation.USER_AGENT, }, signal: AbortSignal.timeout(10 * 1000), - }).catch((e) => { - log.error("Failed to fetch models.dev", { - error: e, - }) }) - if (result && result.ok) { - await Filesystem.write(filepath, await result.text()) + .then((response) => (response.ok ? response.text() : undefined)) + .catch((e) => { + log.error("Failed to fetch models.dev", { + error: e, + }) + }) + if (result !== undefined) { + await Filesystem.write(filepath, result) ModelsDev.Data.reset() } } diff --git a/packages/cli/test/preload.ts b/packages/cli/test/preload.ts index ae29da0d..58ad7c29 100644 --- a/packages/cli/test/preload.ts +++ b/packages/cli/test/preload.ts @@ -33,6 +33,8 @@ process.env["XDG_CACHE_HOME"] = path.join(dir, "cache") process.env["XDG_CONFIG_HOME"] = path.join(dir, "config") process.env["XDG_STATE_HOME"] = path.join(dir, "state") process.env["AICTRL_MODELS_PATH"] = path.join(import.meta.dir, "tool", "fixtures", "models-api.json") +// Use the fixture catalog without starting a detached live refresh in the test process. +process.env["AICTRL_DISABLE_MODELS_FETCH"] = "true" // Set test home directory to isolate tests from user's actual home directory // This prevents tests from picking up real user configs/skills from ~/.claude/skills diff --git a/packages/cli/test/provider/models.test.ts b/packages/cli/test/provider/models.test.ts new file mode 100644 index 00000000..c744ecba --- /dev/null +++ b/packages/cli/test/provider/models.test.ts @@ -0,0 +1,40 @@ +import { expect, test } from "bun:test" +import path from "path" + +test("model catalog refresh handles a timeout while reading the response body", async () => { + using server = Bun.serve({ + port: 0, + idleTimeout: 0, + fetch() { + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("{")) + }, + }), + { headers: { "Content-Type": "application/json" } }, + ) + }, + }) + const proc = Bun.spawn( + [ + "bun", + "--conditions=browser", + "-e", + `const { ModelsDev } = await import(${JSON.stringify(path.resolve(import.meta.dir, "../../src/provider/models.ts"))}); await ModelsDev.refresh(); console.log("refresh completed")`, + ], + { + env: { ...process.env, AICTRL_DISABLE_MODELS_FETCH: "true", AICTRL_MODELS_URL: server.url.origin }, + stdout: "pipe", + stderr: "pipe", + }, + ) + using cleanup = { [Symbol.dispose]: () => proc.kill() } + const [stdout, stderr, exit] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]) + expect(exit, stderr).toBe(0) + expect(stdout).toContain("refresh completed") +}, 15000) From 20df10f76e219cddf05b4fb10dbcf137808b0600 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 7 Oct 2026 12:35:39 +0100 Subject: [PATCH 06/11] fix(session): preserve structured contracts and retry budgets --- EVENTS.md | 28 ++- README.md | 4 + bun.lock | 2 +- packages/cli/README.md | 4 + packages/cli/package.json | 2 +- packages/cli/src/cli/cmd/run.invocation.ts | 7 +- packages/cli/src/cli/cmd/run.output.ts | 12 + packages/cli/src/cli/cmd/run.ts | 6 +- packages/cli/src/session/llm.ts | 2 +- packages/cli/src/session/output-schema.ts | 6 +- packages/cli/src/session/processor.ts | 2 + packages/cli/src/session/prompt.ts | 82 +++++-- .../cli/test/cli/run-output-schema.test.ts | 58 ++++- .../cli/test/session/output-schema.test.ts | 229 +++++++++++++++++- 14 files changed, 394 insertions(+), 50 deletions(-) diff --git a/EVENTS.md b/EVENTS.md index f4dde186..4229767b 100644 --- a/EVENTS.md +++ b/EVENTS.md @@ -543,7 +543,7 @@ does not coerce types, insert defaults or remove extra properties. Zero permits exactly one attempt. `--output-result ` writes only accepted JSON using two-space indentation and a trailing newline, with a temporary file in the destination directory followed by an atomic rename. Failures preserve the -file. Both flags require `--output-schema`; the destination directory must exist. +file. Both flags require `--output-schema`; the destination directory must exist and be writable at configuration time. Without `--output-result`, formatted mode prints final JSON; NDJSON includes the value in the terminal event below. @@ -564,7 +564,10 @@ and a model turn that finishes in prose without a final tool call: The usual event envelope fields also apply. `attempt` is one-based; `maxAttempts` equals `retryCount + 1`. Diagnostics include at most ten errors, with bounded JSON-pointer paths, keywords and messages. Their encoded payload -is under 2 KB and never includes submitted values or raw JSON arguments. +is under 2 KB and never includes submitted values or raw JSON arguments. A `path` +can contain property names submitted by the model (for example, in map or +`patternProperties` schemas), never their values. Each path segment is clipped +to 64 characters; paths also have an overall length bound. Validation feedback goes to the model in the same session through the `invalid` tool. A prose-only finish emits the fixed diagnostic `{ "path": "", "keyword": "missing", "message": "call StructuredOutput with the final result" }` @@ -589,12 +592,25 @@ or: `attempts` counts StructuredOutput calls, including rejected/unparseable ones, and prose-only finishes without a final call. Other tool calls do not count. +Compaction turns do not consume a missing-result attempt. Discarded provider +streams restore the budget and captured result; their rejection events remain +as telemetry. A valid call in a parallel step wins regardless of call order if +that step started with budget remaining, and its attempt count stays within the +configured maximum. When the budget is spent, the last rejection determines the reason: `missing` -for a prose-only finish, otherwise `exhausted`. Failure reasons are `exhausted`, `missing`, -`step_limit`, `aborted`, or `error` (including provider failures and stream idle -timeouts). Failed events have no `value`. No prose fallback is accepted. +for a prose-only finish, otherwise `exhausted`. Failure reasons are: + +- `exhausted`: invalid or unparseable calls spent the corrective budget. +- `missing`: prose finishes spent the budget, or the session ended normally + without an accepted result (including headless permission rejection). +- `step_limit`: the agent reached its configured step cap without a result. +- `aborted`: the invocation was cancelled, including SIGINT/SIGTERM. +- `error`: a provider or session error, including stream idle timeout or an + invalid accepted result received from a remote session. + +Failed events have no `value`. No prose fallback is accepted. Configuration failures happen before a session starts and use the existing -`invocation_error`/`invocation_complete` envelope, with code `OUTPUT_SCHEMA_CONFIG`. +`invocation_error`/`invocation_complete` envelope, with code `OUTPUT_SCHEMA_CONFIG` and the specific cause clipped to 300 characters. Exit codes are **0** on accepted results, **2** for schema configuration errors, **3** for exhausted attempts, missing output and step limits. Provider errors and diff --git a/README.md b/README.md index c74bca05..dd4ebbeb 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,10 @@ aictrl run --output-schema result.schema.json --output-schema-retries 2 \ indented with two spaces and a trailing newline. Failures preserve an existing file and leave an absent file absent. The destination directory must exist. +Ajv strict mode rejects unknown schema keywords and unregistered `format` +values such as `date-time` and `uri` as configuration errors (exit **2**). +The result parent directory must exist and be writable at configuration time. + The retries and result flags require `--output-schema`. Without a result file, formatted mode prints validated JSON at the end; `--format json` emits it in a terminal `structured_output` event. Rejections emit `structured_output_rejected` diff --git a/bun.lock b/bun.lock index 24a7ba27..f02ab309 100644 --- a/bun.lock +++ b/bun.lock @@ -50,7 +50,7 @@ "@standard-schema/spec": "1.0.0", "@zip.js/zip.js": "2.7.62", "ai": "catalog:", - "ajv": "^8", + "ajv": "8.18.0", "decimal.js": "10.5.0", "diff": "catalog:", "drizzle-orm": "1.0.0-beta.12-a5629fb", diff --git a/packages/cli/README.md b/packages/cli/README.md index dd41984e..5645525c 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -47,6 +47,10 @@ aictrl run --output-schema result.schema.json --output-schema-retries 2 \ indented with two spaces and a trailing newline. Failures preserve an existing file and leave an absent file absent. The destination directory must exist. +Ajv strict mode rejects unknown schema keywords and unregistered `format` +values such as `date-time` and `uri` as configuration errors (exit **2**). +The result parent directory must exist and be writable at configuration time. + The retries and result flags require `--output-schema`. Without a result file, formatted mode prints validated JSON at the end; `--format json` emits it in a terminal `structured_output` event. Rejections emit `structured_output_rejected` diff --git a/packages/cli/package.json b/packages/cli/package.json index 430e56bf..3a271b72 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -102,7 +102,7 @@ "@standard-schema/spec": "1.0.0", "@zip.js/zip.js": "2.7.62", "ai": "catalog:", - "ajv": "^8", + "ajv": "8.18.0", "decimal.js": "10.5.0", "diff": "catalog:", "drizzle-orm": "1.0.0-beta.12-a5629fb", diff --git a/packages/cli/src/cli/cmd/run.invocation.ts b/packages/cli/src/cli/cmd/run.invocation.ts index b2df3cb0..c57e3619 100644 --- a/packages/cli/src/cli/cmd/run.invocation.ts +++ b/packages/cli/src/cli/cmd/run.invocation.ts @@ -27,14 +27,17 @@ export function createRunInvocation(enabled: boolean) { emit("invocation_start") - function error(_error: unknown, code = `INVOCATION_${phase.toUpperCase()}_FAILED`) { + function error(cause: unknown, code = `INVOCATION_${phase.toUpperCase()}_FAILED`) { if (!id || failed || completed) return failed = true if (sessionID) return emit("invocation_error", { phase, code, - message: `Invocation failed during ${phase}`, + message: + code === "OUTPUT_SCHEMA_CONFIG" + ? (cause instanceof Error ? cause.message : String(cause)).slice(0, 300) + : `Invocation failed during ${phase}`, }) } diff --git a/packages/cli/src/cli/cmd/run.output.ts b/packages/cli/src/cli/cmd/run.output.ts index 0e386cba..aefebaba 100644 --- a/packages/cli/src/cli/cmd/run.output.ts +++ b/packages/cli/src/cli/cmd/run.output.ts @@ -35,6 +35,18 @@ export async function outputSchema(input: { schema?: string; retries?: number; r throw new Error(`${input.schema}: ${error instanceof Error ? error.message : "cannot compile schema"}`) } })() + if (input.result) { + const directory = path.dirname(input.result) + const writable = await fs + .stat(directory) + .then(async (stat) => { + if (!stat.isDirectory()) return false + await fs.access(directory, fs.constants.W_OK | fs.constants.X_OK) + return true + }) + .catch(() => false) + if (!writable) throw new Error(`${input.result}: output-result parent directory must exist and be writable`) + } return { format: { type: "json_schema" as const, schema: canonical, retryCount: input.retries ?? 2 }, validate } } diff --git a/packages/cli/src/cli/cmd/run.ts b/packages/cli/src/cli/cmd/run.ts index d05f1fde..d805c24c 100644 --- a/packages/cli/src/cli/cmd/run.ts +++ b/packages/cli/src/cli/cmd/run.ts @@ -592,7 +592,11 @@ export const RunCommand = cmd({ ? { status: "failed", reason: "aborted", attempts } : pending?.status === "failed" ? pending - : { status: "failed", reason: "error", attempts } + : { + status: "failed", + reason: message || pending?.status === "accepted" ? "error" : "missing", + attempts, + } emit("structured_output", structured) error ??= `Structured output failed: ${structured.reason}` invocation.error(error) diff --git a/packages/cli/src/session/llm.ts b/packages/cli/src/session/llm.ts index e4b49e3d..4f4bf660 100644 --- a/packages/cli/src/session/llm.ts +++ b/packages/cli/src/session/llm.ts @@ -40,7 +40,7 @@ export namespace LLM { tools: Record retries?: number toolChoice?: "auto" | "required" | "none" - structured?: { reject: (input: string) => Promise } + structured?: { reject: (input: string) => Promise; checkpoint: () => () => void } } export type StreamOutput = StreamTextResult diff --git a/packages/cli/src/session/output-schema.ts b/packages/cli/src/session/output-schema.ts index 279a48fe..6b804cdd 100644 --- a/packages/cli/src/session/output-schema.ts +++ b/packages/cli/src/session/output-schema.ts @@ -38,7 +38,11 @@ export namespace OutputSchema { export function diagnostics(errors?: ErrorObject[] | null): Diagnostic[] { return (errors ?? []).slice(0, 10).reduce((result, error) => { const item = { - path: error.instancePath.slice(0, 96), + path: error.instancePath + .split("/") + .map((segment) => segment.slice(0, 64)) + .join("/") + .slice(0, 96), keyword: error.keyword.slice(0, 32), message: (error.message ?? "invalid value").slice(0, 96), } diff --git a/packages/cli/src/session/processor.ts b/packages/cli/src/session/processor.ts index 20b51311..cedaf380 100644 --- a/packages/cli/src/session/processor.ts +++ b/packages/cli/src/session/processor.ts @@ -82,6 +82,7 @@ export namespace SessionProcessor { needsCompaction = false const shouldBreak = (await Config.get()).experimental?.continue_loop_on_deny !== true while (true) { + const reset = streamInput.structured?.checkpoint() const parts = new Set() const texts: MessageV2.TextPart[] = [] const delivered = new Set() @@ -477,6 +478,7 @@ export namespace SessionProcessor { } const retry = SessionRetry.retryable(error) if (retry !== undefined) { + reset?.() // Keep the failed attempt's file changes available to revert tooling. await patch() retried.push( diff --git a/packages/cli/src/session/prompt.ts b/packages/cli/src/session/prompt.ts index 695ff506..6a0a70c0 100644 --- a/packages/cli/src/session/prompt.ts +++ b/packages/cli/src/session/prompt.ts @@ -331,14 +331,15 @@ export namespace SessionPrompt { let attempts = 0 let rejected = 0 let reminder = false + let user: string | undefined let contract: | { - id: string format: MessageV2.OutputFormat & { type: "json_schema" } validate: ReturnType } | undefined let outcome: OutputSchema.Outcome | undefined + let failed = false let step = 0 const session = await Session.get(sessionID) @@ -366,8 +367,21 @@ export namespace SessionPrompt { } if (!lastUser) throw new Error("No user message found in stream. This should never happen.") - if (lastUser.format?.type === "json_schema" && contract?.id !== lastUser.id) { - contract = { id: lastUser.id, format: lastUser.format, validate: OutputSchema.compile(lastUser.format.schema) } + const request = msgs.findLast( + (msg) => + msg.info.role === "user" && + (!msg.parts.length || + !msg.parts.every( + (part) => + part.type === "compaction" || part.type === "subtask" || (part.type === "text" && part.synthetic), + )), + )?.info as MessageV2.User | undefined + if (request && user !== request.id) { + user = request.id + contract = + request.format?.type === "json_schema" + ? { format: request.format, validate: OutputSchema.compile(request.format.schema) } + : undefined attempts = 0 rejected = 0 reminder = false @@ -632,7 +646,10 @@ export namespace SessionPrompt { sessionID, auto: task.auto, }) - if (result === "stop") break + if (result === "stop") { + failed = true + break + } continue } @@ -722,6 +739,7 @@ export namespace SessionPrompt { return OutputSchema.error(errors).message } + const available = !!contract && attempts < contract.format.retryCount + 1 if (contract) { tools["StructuredOutput"] = createStructuredOutputTool({ schema: contract.format.schema, @@ -729,9 +747,8 @@ export namespace SessionPrompt { model, async onSuccess(output) { if (structuredOutput !== undefined) return - if (attempts >= contract!.format.retryCount + 1) - throw new Error("StructuredOutput attempt budget exhausted") - attempts++ + if (!available) throw new Error("StructuredOutput attempt budget exhausted") + attempts = Math.min(attempts + 1, contract!.format.retryCount + 1) structuredOutput = output }, }) @@ -770,8 +787,7 @@ export namespace SessionPrompt { // Build system prompt, adding structured output instruction if needed const system = [...(await SystemPrompt.environment(model)), ...(await InstructionPrompt.system())] - const format = lastUser.format ?? { type: "text" } - if (format.type === "json_schema") { + if (contract) { system.push(STRUCTURED_OUTPUT_SYSTEM_PROMPT) } @@ -803,9 +819,23 @@ export namespace SessionPrompt { tools, model, toolChoice: contract ? "required" : undefined, - structured: contract ? { reject } : undefined, + structured: contract + ? { + reject, + checkpoint() { + const before = { attempts, rejected, structuredOutput } + return () => { + attempts = before.attempts + rejected = before.rejected + structuredOutput = before.structuredOutput + } + }, + } + : undefined, }) + failed = !!processor.message.error + // If structured output was captured, save it and exit immediately // This takes priority because the StructuredOutput tool was called successfully if ( @@ -822,6 +852,16 @@ export namespace SessionPrompt { } structuredOutput = undefined + if (result === "compact") { + await SessionCompaction.create({ + sessionID, + agent: lastUser.agent, + model: lastUser.model, + auto: true, + }) + continue + } + reminder = !!contract && (await missingStructuredOutput(processor.message, () => MessageV2.parts(processor.message.id))) if (reminder) { @@ -844,21 +884,17 @@ export namespace SessionPrompt { if (reminder) continue if (result === "stop") break - if (result === "compact") { - await SessionCompaction.create({ - sessionID, - agent: lastUser.agent, - model: lastUser.model, - auto: true, - }) - } continue } await SessionCompaction.prune({ sessionID }) if (contract) { await Bus.publish(Session.Event.StructuredOutput, { sessionID, - outcome: outcome ?? { status: "failed", reason: abort.aborted ? "aborted" : "error", attempts }, + outcome: outcome ?? { + status: "failed", + reason: abort.aborted ? "aborted" : failed ? "error" : "missing", + attempts, + }, }) } for await (const item of MessageV2.stream(sessionID)) { @@ -1082,10 +1118,10 @@ export namespace SessionPrompt { validate?: ReturnType model?: Provider.Model onSuccess: (output: unknown) => void | Promise - }): AITool { + }): AITool & { id: "StructuredOutput" } { const validate = input.validate ?? OutputSchema.compile(input.schema) const schema = Object.fromEntries(Object.entries(input.schema).filter(([key]) => key !== "$schema")) - return tool({ + const result = tool({ description: STRUCTURED_OUTPUT_DESCRIPTION, inputSchema: jsonSchema(input.model ? ProviderTransform.schema(input.model, schema) : schema, { validate(value) { @@ -1106,9 +1142,9 @@ export namespace SessionPrompt { toModelOutput(result) { return { type: "text", value: result.output } }, - // Retain the internal identifier used by tool consumers. - ...{ id: "StructuredOutput" }, }) + // Retain the internal identifier used by tool consumers. + return Object.assign(result, { id: "StructuredOutput" as const }) } async function createUserMessage(input: PromptInput) { diff --git a/packages/cli/test/cli/run-output-schema.test.ts b/packages/cli/test/cli/run-output-schema.test.ts index 34429454..ee52e470 100644 --- a/packages/cli/test/cli/run-output-schema.test.ts +++ b/packages/cli/test/cli/run-output-schema.test.ts @@ -19,6 +19,8 @@ async function run( abort?: boolean failure?: boolean attach?: boolean + locked?: boolean + denied?: boolean } = {}, ) { await using tmp = await tmpdir() @@ -68,7 +70,9 @@ async function run( index: 0, id: `call_${requests.length}`, type: "function", - function: { name: "StructuredOutput", arguments: args }, + function: options.denied + ? { name: "read", arguments: JSON.stringify({ filePath: "/outside-fixture.txt" }) } + : { name: "StructuredOutput", arguments: args }, }, ], }, @@ -77,7 +81,7 @@ async function run( ], }, { - choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + choices: [{ index: 0, delta: {}, finish_reason: options.denied ? "tool-calls" : "stop" }], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, }, ] @@ -99,6 +103,7 @@ async function run( }), ) await Bun.write(path.join(tmp.path, "schema.json"), options.content ?? JSON.stringify(schema)) + if (options.locked) await fs.mkdir(path.join(tmp.path, "locked"), { mode: 0o555 }) if (options.existing) await Bun.write(path.join(tmp.path, "result.json"), options.existing) const home = path.join(tmp.path, "home") await fs.mkdir(home) @@ -230,6 +235,26 @@ for (const [name, options, reason] of [ ["invalid JSON", { content: "{broken" }, "invalid JSON"], ["uncompilable", { content: '{"type":"object","unknownKeyword":true}' }, "strict mode"], ["non-object root", { content: '{"type":"array"}' }, "object"], + [ + "unknown format", + { content: '{"type":"object","properties":{"result":{"type":"string","format":"date-time"}}}' }, + "unknown format", + ], + [ + "missing result parent", + { flags: ["--output-schema", "schema.json", "--output-result", "absent/result.json"] }, + "parent directory must exist and be writable", + ], + [ + "file as result parent", + { flags: ["--output-schema", "schema.json", "--output-result", "schema.json/result.json"] }, + "parent directory must exist and be writable", + ], + [ + "unwritable result parent", + { locked: true, flags: ["--output-schema", "schema.json", "--output-result", "locked/result.json"] }, + "parent directory must exist and be writable", + ], ["retries without schema", { flags: ["--output-schema-retries", "1"] }, "require --output-schema"], ["result without schema", { flags: ["--output-result", "result.json"] }, "require --output-schema"], ["negative retries", { flags: ["--output-schema", "schema.json", "--output-schema-retries", "-1"] }, "integer >= 0"], @@ -248,6 +273,9 @@ for (const [name, options, reason] of [ const result = await run({ ...options, flags: "flags" in options ? [...options.flags] : undefined }) expect(result.exit, result.stderr).toBe(2) expect(result.stderr).toContain(reason) + const event = result.events.find((event) => event.type === "invocation_error") + expect((event!.message as string).length).toBeLessThanOrEqual(300) + expect(event).toMatchObject({ code: "OUTPUT_SCHEMA_CONFIG", message: expect.stringContaining(reason) }) if ("content" in options) expect(result.stderr).toContain("schema.json") expect(result.requests).toEqual([]) expect(result.result).toBeUndefined() @@ -579,3 +607,29 @@ test("JSON mode without output-result carries the accepted value only in termina { status: "accepted", attempts: 1, value: { result: "accepted" } }, ]) }, 25000) + +test("headless permission rejection reports missing output with no session error", async () => { + const result = await run({ + denied: true, + existing: "untouched", + flags: ["--output-schema", "schema.json", "--output-result", "result.json"], + }) + expect(result.exit, result.stderr + result.stdout).toBe(3) + expect(result.requests).toHaveLength(1) + expect(result.result).toBe("untouched") + expect(result.events.filter((event) => event.type === "permission_rejected")).toHaveLength(1) + expect(result.events.filter((event) => event.type === "session_error")).toEqual([]) + expect(result.events.filter((event) => event.type === "structured_output")).toMatchObject([ + { status: "failed", reason: "missing", attempts: 0 }, + ]) +}, 25000) + +test("schema config NDJSON clips the specific compilation cause", async () => { + const result = await run({ content: JSON.stringify({ type: "object", ["unknown".repeat(100)]: true }) }) + expect(result.exit).toBe(2) + expect(result.requests).toEqual([]) + const event = result.events.find((event) => event.type === "invocation_error") + expect(event?.code).toBe("OUTPUT_SCHEMA_CONFIG") + expect(event?.message).toStartWith("schema.json: strict mode: unknown keyword:") + expect((event?.message as string).length).toBe(300) +}) diff --git a/packages/cli/test/session/output-schema.test.ts b/packages/cli/test/session/output-schema.test.ts index 682514b9..53bb9920 100644 --- a/packages/cli/test/session/output-schema.test.ts +++ b/packages/cli/test/session/output-schema.test.ts @@ -1,10 +1,14 @@ -import { expect, test } from "bun:test" +import { expect, spyOn, test } from "bun:test" +import { APICallError } from "ai" +import { LLM } from "../../src/session/llm" +import { PermissionNext } from "../../src/permission/next" import Ajv from "ajv" import Ajv2020 from "ajv/dist/2020" import { Instance } from "../../src/project/instance" import { Session } from "../../src/session" import { SessionPrompt } from "../../src/session/prompt" import { Bus } from "../../src/bus" +import { MessageV2 } from "../../src/session/message-v2" import { OutputSchema } from "../../src/session/output-schema" import { tmpdir } from "../fixture/fixture" import { schema, corpus } from "../fixture/output-schema" @@ -22,9 +26,21 @@ async function run( other?: boolean plain?: boolean gemini?: boolean + compact?: boolean + parallel?: string[] + retry?: boolean + queued?: boolean + followup?: boolean + denied?: boolean } = {}, ) { const requests: Record[] = [] + let directory = "" + let sessionID = "" + let notify: () => void = () => {} + const delivered = new Promise((resolve) => { + notify = resolve + }) const server = Bun.serve({ port: 0, async fetch(request) { @@ -51,15 +67,32 @@ async function run( }), { headers: { "Content-Type": "text/event-stream" } }, ) + if (options.queued && requests.length === 1) { + // Persist a genuine new user message while the current model turn runs. + await Instance.provide({ + directory, + fn: () => + SessionPrompt.prompt({ + sessionID, + noReply: true, + model: { providerID: "alibaba", modelID: "qwen-plus" }, + parts: [{ type: "text", text: "A plain follow-up." }], + }), + }) + } const args = inputs[Math.min(requests.length - 1, inputs.length - 1)] - const calls = [ - { - index: 0, - id: `call_${requests.length}`, - type: "function", - function: { name: "StructuredOutput", arguments: args }, - }, - ] + const calls = (options.parallel ?? [args]).map((args, index) => ({ + index, + id: `call_${requests.length}_${index}`, + type: "function", + function: { name: "StructuredOutput", arguments: args }, + })) + if ((options.queued || options.denied) && requests.length === 1) { + calls[0].function = { + name: "read", + arguments: JSON.stringify({ filePath: options.denied ? "/outside-fixture.txt" : "aictrl.json" }), + } + } const chunks = [ { choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }] }, { @@ -67,7 +100,7 @@ async function run( { index: 0, delta: - args === "prose" + args === "prose" && !((options.queued || options.denied) && requests.length === 1) ? { content: "No final tool call" } : { tool_calls: options.other @@ -87,10 +120,40 @@ async function run( ], }, { - choices: [{ index: 0, delta: {}, finish_reason: options.finish ?? "stop" }], - usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + choices: [ + { + index: 0, + delta: {}, + finish_reason: + (options.queued || options.denied) && requests.length === 1 ? "tool-calls" : (options.finish ?? "stop"), + }, + ], + usage: { + prompt_tokens: options.compact && requests.length === 1 ? 1000000 : 1, + completion_tokens: 1, + total_tokens: options.compact && requests.length === 1 ? 1000001 : 2, + }, }, ] + if (options.retry && requests.length === 1) { + return new Response( + new ReadableStream({ + async start(controller) { + chunks + .slice(0, 2) + .forEach((chunk) => controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(chunk)}\n\n`))) + await delivered + controller.enqueue( + new TextEncoder().encode( + `data: ${JSON.stringify({ error: { message: JSON.stringify({ type: "error", error: { type: "too_many_requests" } }) } })}\n\ndata: [DONE]\n\n`, + ), + ) + controller.close() + }, + }), + { headers: { "Content-Type": "text/event-stream" } }, + ) + } return new Response( chunks .map((chunk) => `data: ${JSON.stringify(chunk)}`) @@ -100,10 +163,45 @@ async function run( ) }, }) + // SSE errors are text; convert just the fixture's error into the typed + // retryable failure a provider throws after partially delivering a stream. + const original = LLM.stream + const stream = options.retry + ? spyOn(LLM, "stream").mockImplementation(async (input) => { + const result = await original(input) + return new Proxy(result, { + get(target, key, receiver) { + if (key !== "fullStream") return Reflect.get(target, key, receiver) + return target.fullStream.pipeThrough( + new TransformStream({ + transform(chunk, controller) { + controller.enqueue( + chunk.type === "error" + ? { + type: "error", + error: new APICallError({ + message: "Fixture stream interrupted", + url: server.url.href, + requestBodyValues: {}, + statusCode: 503, + isRetryable: true, + responseHeaders: { "retry-after-ms": "1" }, + }), + } + : chunk, + ) + }, + }), + ) + }, + }) + }) + : undefined try { await using tmp = await tmpdir({ config: { ...(options.steps ? { agent: { build: { steps: options.steps } } } : {}), + ...(options.compact ? { compaction: { auto: true } } : {}), enabled_providers: options.gemini ? ["fixture"] : ["alibaba"], provider: options.gemini ? { @@ -119,7 +217,9 @@ async function run( return await Instance.provide({ directory: tmp.path, fn: async () => { + directory = tmp.path const session = await Session.create({ title: "Schema boundary fixture" }) + sessionID = session.id const rejected: { sessionID: string attempt: number @@ -128,6 +228,19 @@ async function run( }[] = [] const outcomes: OutputSchema.Outcome[] = [] const unsubscribe = [ + Bus.subscribe(PermissionNext.Event.Asked, async (event) => { + if (!options.denied || event.properties.sessionID !== session.id) return + await PermissionNext.reply({ requestID: event.properties.id, reply: "reject" }) + }), + Bus.subscribe(MessageV2.Event.PartUpdated, (event) => { + const part = event.properties.part + if ( + part.sessionID === session.id && + part.type === "tool" && + (part.state.status === "completed" || part.state.status === "error") + ) + notify() + }), Bus.subscribe(Session.Event.StructuredOutputRejected, (event) => { rejected.push(event.properties) }), @@ -149,9 +262,17 @@ async function run( ? {} : { format: { type: "json_schema" as const, schema: options.schema ?? schema, retryCount } }), }) + const followup = options.followup + ? await SessionPrompt.prompt({ + sessionID: session.id, + model: { providerID: "alibaba", modelID: "qwen-plus" }, + parts: [{ type: "text", text: "A plain follow-up." }], + }) + : undefined if (result.info.role !== "assistant") throw new Error("Expected assistant") return { info: result.info, + followup, requests, rejected, outcomes, @@ -166,6 +287,7 @@ async function run( }, }) } finally { + stream?.mockRestore() server.stop(true) } } @@ -397,3 +519,86 @@ test("repeated prose-only finishes spend the configured corrective budget", asyn expect(result.info.error).toMatchObject({ name: "StructuredOutputError", data: { retries: 2 } }) expect(result.outcomes).toEqual([{ status: "failed", reason: "missing", attempts: 3 }]) }, 15_000) + +test("compaction precedes missing reminders and preserves the schema contract", async () => { + const result = await run(["prose", "prose", '{"result":"accepted"}'], 0, { compact: true }) + expect( + result.requests, + JSON.stringify({ requests: result.requests, messages: result.messages, outcomes: result.outcomes }), + ).toHaveLength(3) + expect(result.messages.some((message) => message.parts.some((part) => part.type === "compaction"))).toBe(true) + expect(result.requests[1].tools).toBeUndefined() + expect(result.requests[2].tool_choice).toBe("required") + expect(JSON.stringify(result.requests[2].tools)).toContain("StructuredOutput") + expect(JSON.stringify(result.requests[2].messages)).toContain("The user has requested structured output") + expect(result.rejected).toEqual([]) + expect(result.outcomes).toEqual([{ status: "accepted", attempts: 1, value: { result: "accepted" } }]) +}, 15000) + +for (const queued of [false, true]) { + test(`plain follow-up clears the schema contract ${queued ? "during" : "after"} a run`, async () => { + const result = await run([queued ? "prose" : '{"result":"accepted"}', "prose"], 0, { queued, followup: !queued }) + expect(result.requests, JSON.stringify(result.info)).toHaveLength(2) + expect(result.requests[0].tool_choice).toBe("required") + expect(result.requests[1].tool_choice).toBe("auto") + expect(JSON.stringify(result.requests[1].tools)).not.toContain("StructuredOutput") + // The old schema instruction can remain in history only as ordinary conversation. + const messages = result.requests[1].messages as { role: string; content: unknown }[] + expect(JSON.stringify(messages.filter((message) => message.role === "system"))).not.toContain( + "The user has requested structured output", + ) + expect(result.rejected).toEqual([]) + }, 15000) +} + +for (const input of ["{}", '{"result":"discarded"}']) { + test(`provider stream retry restores ${input === "{}" ? "rejected budget" : "captured result"}`, async () => { + const invalid = input === "{}" + const result = await run( + invalid ? [input, "{}", '{"result":"accepted"}'] : [input, '{"result":"accepted"}'], + invalid ? 1 : 0, + { retry: true }, + ) + expect(result.requests, JSON.stringify(result.info)).toHaveLength(invalid ? 3 : 2) + expect(result.info.error).toBeUndefined() + expect(result.info.structured).toEqual({ result: "accepted" }) + expect(result.outcomes).toEqual([{ status: "accepted", attempts: invalid ? 2 : 1, value: { result: "accepted" } }]) + // Discarded rejections stay as telemetry, while the clean stream starts at attempt 1. + expect(result.rejected.map((event) => event.attempt)).toEqual(invalid ? [1, 1] : []) + expect(JSON.stringify(result.requests[1])).not.toContain("discarded") + }, 15000) +} + +for (const parallel of [ + ["{}", '{"result":"accepted"}'], + ['{"result":"accepted"}', "{}"], +]) { + test(`valid parallel call wins with zero retries: ${parallel[0] === "{}" ? "invalid first" : "valid first"}`, async () => { + const result = await run(["unused"], 0, { parallel }) + expect(result.requests).toHaveLength(1) + expect(result.info.error).toBeUndefined() + expect(result.info.structured).toEqual({ result: "accepted" }) + expect(result.outcomes).toEqual([{ status: "accepted", attempts: 1, value: { result: "accepted" } }]) + }, 15000) +} + +test("permission denial without a provider error reports missing output", async () => { + const result = await run(["unused"], 0, { denied: true }) + expect(result.requests, JSON.stringify(result.info)).toHaveLength(1) + expect(result.info.error).toBeUndefined() + expect(result.info.structured).toBeUndefined() + expect(result.outcomes).toEqual([{ status: "failed", reason: "missing", attempts: 0 }]) +}, 15000) + +test("diagnostic paths clip every model-chosen property segment without including values", () => { + const validate = OutputSchema.compile({ + type: "object", + additionalProperties: { type: "object", additionalProperties: { type: "integer" } }, + }) + const key = "k".repeat(100) + expect(validate({ [key]: { [key]: "submitted-private-value" } })).toBe(false) + const errors = OutputSchema.diagnostics(validate.errors) + expect(errors[0].path.split("/").every((segment) => segment.length <= 64)).toBe(true) + expect(errors[0].path).toStartWith("/" + "k".repeat(64) + "/") + expect(JSON.stringify(errors)).not.toContain("submitted-private-value") +}) From 286c329f218bc82e8aef32c9b99dd39acd2e2a94 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 7 Oct 2026 13:23:12 +0100 Subject: [PATCH 07/11] fix(cli): address review on #129 for structured output failures --- packages/cli/src/cli/cmd/run.invocation.ts | 3 +- packages/cli/src/cli/cmd/run.output.ts | 5 +- packages/cli/src/cli/cmd/run.ts | 20 ++-- packages/cli/src/session/llm.ts | 6 +- packages/cli/src/session/output-schema.ts | 12 +++ packages/cli/src/session/processor.ts | 6 +- packages/cli/src/session/prompt.ts | 16 +++- .../cli/test/cli/run-output-schema.test.ts | 94 ++++++++++++++----- .../cli/test/session/output-schema.test.ts | 47 +++++++++- 9 files changed, 162 insertions(+), 47 deletions(-) diff --git a/packages/cli/src/cli/cmd/run.invocation.ts b/packages/cli/src/cli/cmd/run.invocation.ts index c57e3619..082af77f 100644 --- a/packages/cli/src/cli/cmd/run.invocation.ts +++ b/packages/cli/src/cli/cmd/run.invocation.ts @@ -1,5 +1,6 @@ import { Stdout } from "../stdout" import { SCHEMA_VERSION } from "./run.errors" +import { OUTPUT_SCHEMA_CONFIG } from "./run.output" export type RunInvocationPhase = "validation" | "stdin" | "bootstrap" | "session" @@ -35,7 +36,7 @@ export function createRunInvocation(enabled: boolean) { phase, code, message: - code === "OUTPUT_SCHEMA_CONFIG" + code === OUTPUT_SCHEMA_CONFIG ? (cause instanceof Error ? cause.message : String(cause)).slice(0, 300) : `Invocation failed during ${phase}`, }) diff --git a/packages/cli/src/cli/cmd/run.output.ts b/packages/cli/src/cli/cmd/run.output.ts index aefebaba..5f930791 100644 --- a/packages/cli/src/cli/cmd/run.output.ts +++ b/packages/cli/src/cli/cmd/run.output.ts @@ -1,7 +1,8 @@ import path from "path" import fs from "fs/promises" -import { OutputSchema } from "../../session/output-schema" +import { OutputSchema } from "@/session/output-schema" +export const OUTPUT_SCHEMA_CONFIG = "OUTPUT_SCHEMA_CONFIG" export const OUTPUT_CONFIG_EXIT = 2 export const OUTPUT_FAILED_EXIT = 3 @@ -12,7 +13,7 @@ export async function outputSchema(input: { schema?: string; retries?: number; r return } if (input.retries !== undefined && (!Number.isSafeInteger(input.retries) || input.retries < 0)) - throw new Error(`${input.schema}: --output-schema-retries must be an integer >= 0`) + throw new Error("--output-schema-retries must be an integer >= 0") const text = await Bun.file(input.schema) .text() .catch(() => { diff --git a/packages/cli/src/cli/cmd/run.ts b/packages/cli/src/cli/cmd/run.ts index d805c24c..af627850 100644 --- a/packages/cli/src/cli/cmd/run.ts +++ b/packages/cli/src/cli/cmd/run.ts @@ -1,5 +1,5 @@ -import { outputSchema, outputResult, OUTPUT_CONFIG_EXIT, OUTPUT_FAILED_EXIT } from "./run.output" -import { OutputSchema } from "../../session/output-schema" +import { outputSchema, outputResult, OUTPUT_SCHEMA_CONFIG, OUTPUT_CONFIG_EXIT, OUTPUT_FAILED_EXIT } from "./run.output" +import { OutputSchema } from "@/session/output-schema" import type { Argv } from "yargs" import path from "path" import { pathToFileURL } from "bun" @@ -390,7 +390,7 @@ export const RunCommand = cmd({ handler: async (args) => { const invocation = createRunInvocation(args.format === "json") - async function fail(message: string, code: string, exit = 1) { + async function fail(message: string, code: string, exit = 1): Promise { UI.error(message) await invocation.abort(message, code) process.exit(exit) @@ -415,13 +415,12 @@ export const RunCommand = cmd({ schema: args["output-schema"], retries: args["output-schema-retries"], result: args["output-result"], - }).catch(async (error: unknown) => { - await fail( + }).catch((error: unknown): Promise => { + return fail( error instanceof Error ? error.message : "Invalid output schema", - "OUTPUT_SCHEMA_CONFIG", + OUTPUT_SCHEMA_CONFIG, OUTPUT_CONFIG_EXIT, ) - return undefined }) const files: { type: "file"; url: string; filename: string; mime: string }[] = [] @@ -636,7 +635,8 @@ export const RunCommand = cmd({ event.properties.sessionID === sessionID && !structured ) { - pending = OutputSchema.Outcome.parse(event.properties.outcome) + const outcome = OutputSchema.Outcome.safeParse(event.properties.outcome) + pending = outcome.success ? outcome.data : { status: "failed", reason: "error", attempts } attempts = pending.attempts continue } @@ -975,8 +975,8 @@ export const RunCommand = cmd({ }) return } - console.error(schema ? "Structured output run failed" : cause) - process.exitCode = 1 + console.error(schema ? JSON.stringify(classified) : cause) + if (process.exitCode !== OUTPUT_FAILED_EXIT) process.exitCode = 1 } // Emit the resolved tool catalog (builtin + MCP tools) and available diff --git a/packages/cli/src/session/llm.ts b/packages/cli/src/session/llm.ts index 4f4bf660..e6b5d128 100644 --- a/packages/cli/src/session/llm.ts +++ b/packages/cli/src/session/llm.ts @@ -1,6 +1,7 @@ import { Installation } from "@/installation" import { Provider } from "@/provider/provider" import { Log } from "@/util/log" +import { OutputSchema } from "./output-schema" import { streamText, wrapLanguageModel, @@ -40,7 +41,7 @@ export namespace LLM { tools: Record retries?: number toolChoice?: "auto" | "required" | "none" - structured?: { reject: (input: string) => Promise; checkpoint: () => () => void } + structured?: { reject: (input: string) => Promise; checkpoint: () => () => void; repairs: Set } } export type StreamOutput = StreamTextResult @@ -174,11 +175,12 @@ export namespace LLM { return streamText({ onError(error) { l.error("stream error", { - error: input.structured ? "Structured output stream failed" : error, + error: input.structured ? OutputSchema.summary(error.error) : error, }) }, async experimental_repairToolCall(failed) { if (failed.toolCall.toolName === "StructuredOutput" && input.structured) { + input.structured.repairs.add(failed.toolCall.toolCallId) return { ...failed.toolCall, toolName: "invalid", diff --git a/packages/cli/src/session/output-schema.ts b/packages/cli/src/session/output-schema.ts index 6b804cdd..72eeff76 100644 --- a/packages/cli/src/session/output-schema.ts +++ b/packages/cli/src/session/output-schema.ts @@ -62,6 +62,18 @@ export namespace OutputSchema { } } + export function summary(error: unknown) { + const value = error && typeof error === "object" ? error : {} + return { + message: "Structured output stream failed", + ...("name" in value && typeof value.name === "string" ? { name: value.name.slice(0, 96) } : {}), + ...("statusCode" in value && typeof value.statusCode === "number" && Number.isFinite(value.statusCode) + ? { statusCode: value.statusCode } + : {}), + ...("isRetryable" in value && typeof value.isRetryable === "boolean" ? { isRetryable: value.isRetryable } : {}), + } + } + export function error(errors: Diagnostic[]) { return new Error(`StructuredOutput rejected: ${JSON.stringify(errors)}`) } diff --git a/packages/cli/src/session/processor.ts b/packages/cli/src/session/processor.ts index cedaf380..efcf04ab 100644 --- a/packages/cli/src/session/processor.ts +++ b/packages/cli/src/session/processor.ts @@ -1,3 +1,4 @@ +import { OutputSchema } from "./output-schema" import { MessageV2 } from "./message-v2" import { Log } from "@/util/log" import { Identifier } from "@/id/id" @@ -235,8 +236,7 @@ export namespace SessionProcessor { if ( !( streamInput.structured && - (value.toolName === "StructuredOutput" || - (value.toolName === "invalid" && value.input?.tool === "StructuredOutput")) + (value.toolName === "StructuredOutput" || streamInput.structured.repairs.has(value.toolCallId)) ) && lastThree.length === DOOM_LOOP_THRESHOLD && lastThree.every( @@ -466,7 +466,7 @@ export namespace SessionProcessor { } } catch (e: any) { log.error("process", { - error: streamInput.structured ? "Structured output stream failed" : e, + error: streamInput.structured ? OutputSchema.summary(e) : e, stack: streamInput.structured ? undefined : JSON.stringify(e.stack), }) const error = MessageV2.APIError.isInstance(e) diff --git a/packages/cli/src/session/prompt.ts b/packages/cli/src/session/prompt.ts index 6a0a70c0..97c8c2b4 100644 --- a/packages/cli/src/session/prompt.ts +++ b/packages/cli/src/session/prompt.ts @@ -723,9 +723,8 @@ export namespace SessionPrompt { messages: msgs, }) - const reject = async (input: string | OutputSchema.Diagnostic[]) => { + const rejection = async (errors: OutputSchema.Diagnostic[]) => { if (!contract) throw new Error("Missing output schema") - const errors = typeof input === "string" ? OutputSchema.parse(input, contract.validate) : input if (attempts < contract.format.retryCount + 1) { attempts++ rejected++ @@ -739,16 +738,22 @@ export namespace SessionPrompt { return OutputSchema.error(errors).message } + const reject = (raw: string) => { + if (!contract) throw new Error("Missing output schema") + return rejection(OutputSchema.parse(raw, contract.validate)) + } + const available = !!contract && attempts < contract.format.retryCount + 1 if (contract) { + const format = contract.format tools["StructuredOutput"] = createStructuredOutputTool({ - schema: contract.format.schema, + schema: format.schema, validate: contract.validate, model, async onSuccess(output) { if (structuredOutput !== undefined) return if (!available) throw new Error("StructuredOutput attempt budget exhausted") - attempts = Math.min(attempts + 1, contract!.format.retryCount + 1) + attempts = Math.min(attempts + 1, format.retryCount + 1) structuredOutput = output }, }) @@ -822,6 +827,7 @@ export namespace SessionPrompt { structured: contract ? { reject, + repairs: new Set(), checkpoint() { const before = { attempts, rejected, structuredOutput } return () => { @@ -865,7 +871,7 @@ export namespace SessionPrompt { reminder = !!contract && (await missingStructuredOutput(processor.message, () => MessageV2.parts(processor.message.id))) if (reminder) { - await reject([{ path: "", keyword: "missing", message: "call StructuredOutput with the final result" }]) + await rejection([{ path: "", keyword: "missing", message: "call StructuredOutput with the final result" }]) } if (contract && !processor.message.error && rejected > contract.format.retryCount) { diff --git a/packages/cli/test/cli/run-output-schema.test.ts b/packages/cli/test/cli/run-output-schema.test.ts index ee52e470..4502cf1e 100644 --- a/packages/cli/test/cli/run-output-schema.test.ts +++ b/packages/cli/test/cli/run-output-schema.test.ts @@ -269,22 +269,30 @@ for (const [name, options, reason] of [ "schema", ], ] as const) { - test(`configuration error ${name} happens before any model request`, async () => { - const result = await run({ ...options, flags: "flags" in options ? [...options.flags] : undefined }) - expect(result.exit, result.stderr).toBe(2) - expect(result.stderr).toContain(reason) - const event = result.events.find((event) => event.type === "invocation_error") - expect((event!.message as string).length).toBeLessThanOrEqual(300) - expect(event).toMatchObject({ code: "OUTPUT_SCHEMA_CONFIG", message: expect.stringContaining(reason) }) - if ("content" in options) expect(result.stderr).toContain("schema.json") - expect(result.requests).toEqual([]) - expect(result.result).toBeUndefined() - expect(result.events.map((event) => event.type)).toEqual([ - "invocation_start", - "invocation_error", - "invocation_complete", - ]) - }, 25000) + // Root ignores directory mode bits, so it cannot exercise this permission failure. + const check = name === "unwritable result parent" && process.getuid?.() === 0 ? test.skip : test + check( + `configuration error ${name} happens before any model request`, + async () => { + const result = await run({ ...options, flags: "flags" in options ? [...options.flags] : undefined }) + expect(result.exit, result.stderr).toBe(2) + expect(result.stderr).toContain(reason) + const event = result.events.find((event) => event.type === "invocation_error") + expect((event!.message as string).length).toBeLessThanOrEqual(300) + if (name === "negative retries" || name === "fractional retries") + expect(event?.message).toBe("--output-schema-retries must be an integer >= 0") + expect(event).toMatchObject({ code: "OUTPUT_SCHEMA_CONFIG", message: expect.stringContaining(reason) }) + if ("content" in options) expect(result.stderr).toContain("schema.json") + expect(result.requests).toEqual([]) + expect(result.result).toBeUndefined() + expect(result.events.map((event) => event.type)).toEqual([ + "invocation_start", + "invocation_error", + "invocation_complete", + ]) + }, + 25000, + ) } test("repair emits bounded events and writes only validated JSON atomically", async () => { @@ -479,6 +487,9 @@ for (const [name, value, failure, code] of [ ["accepted", { result: "attached" }, false, 0], ["invalid remote result", {}, false, 3], ["late session failure", { result: "attached" }, true, 1], + ["malformed outcome", { result: "attached" }, false, 3], + ["structured failure followed by loop rejection", {}, false, 3], + ["provider rejection", { result: "submitted-secret" }, false, 1], ] as const) { test(`attach ${name} forwards the canonical schema and validates final acceptance`, async () => { await using tmp = await tmpdir() @@ -487,6 +498,7 @@ for (const [name, value, failure, code] of [ let controller: ReadableStreamDefaultController | undefined const server = Bun.serve({ port: 0, + idleTimeout: 0, async fetch(request) { const url = new URL(request.url).pathname if (url === "/event") @@ -519,23 +531,52 @@ for (const [name, value, failure, code] of [ type: "session.structured_output", properties: { sessionID: "ses_fixture", - outcome: { status: "accepted", attempts: 2, value }, + outcome: + name === "malformed outcome" + ? { status: "unexpected", attempts: "bad", value } + : { status: "accepted", attempts: 2, value }, }, }, - ...(failure + ...(failure || name === "provider rejection" ? [ { type: "session.error", properties: { sessionID: "ses_fixture", - error: { name: "APIError", data: { message: "Late provider failure", isRetryable: false } }, + error: { + name: "APIError", + data: { + message: "Late provider failure", + isRetryable: false, + ...(name === "provider rejection" ? { statusCode: 400 } : {}), + }, + }, }, }, ] : []), - { type: "session.status", properties: { sessionID: "ses_fixture", status: { type: "idle" } } }, + ...(name === "structured failure followed by loop rejection" + ? [ + { + type: "session.error", + properties: { + sessionID: "ses_fixture", + error: { name: "StructuredOutputError", data: { message: "No valid result", retries: 1 } }, + }, + }, + // A malformed later event forces the CLI event loop to reject. + { type: "session.status", properties: { sessionID: "ses_fixture", status: null } }, + ] + : name === "provider rejection" + ? [] + : [{ type: "session.status", properties: { sessionID: "ses_fixture", status: { type: "idle" } } }]), ]) controller!.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(event)}\n\n`)) + if (name === "provider rejection") + return Response.json( + { name: "APIError", data: { message: "Fixture provider failed", statusCode: 400, isRetryable: false } }, + { status: 400 }, + ) return Response.json({}) } return new Response("unexpected request", { status: 400 }) @@ -576,6 +617,11 @@ for (const [name, value, failure, code] of [ proc.exited, ]) expect(exit, stderr + stdout).toBe(code) + if (name === "provider rejection") { + expect(stderr).toContain('"code":"400"') + expect(stderr).toContain('"reason":"unknown"') + expect(stdout + stderr).not.toContain("submitted-secret") + } expect(requests.find((request) => request.url.endsWith("/message"))?.body.format).toEqual({ type: "json_schema", schema, @@ -588,9 +634,13 @@ for (const [name, value, failure, code] of [ .filter((line) => line.startsWith("{")) .map((line) => JSON.parse(line)) expect(events.filter((event) => event.type === "structured_output")).toMatchObject([ - code === 0 ? { status: "accepted", attempts: 2, value } : { status: "failed", reason: "error", attempts: 2 }, + code === 0 + ? { status: "accepted", attempts: 2, value } + : { status: "failed", reason: "error", attempts: name === "malformed outcome" ? 1 : 2 }, ]) expect(events.filter((event) => event.type === "structured_output")).toHaveLength(1) + expect(events.filter((event) => event.type === "session_complete")).toHaveLength(1) + if (name === "malformed outcome") expect(events.filter((event) => event.type === "session_error")).toEqual([]) } finally { clearTimeout(timer) proc.kill("SIGKILL") @@ -632,4 +682,4 @@ test("schema config NDJSON clips the specific compilation cause", async () => { expect(event?.code).toBe("OUTPUT_SCHEMA_CONFIG") expect(event?.message).toStartWith("schema.json: strict mode: unknown keyword:") expect((event?.message as string).length).toBe(300) -}) +}, 25000) diff --git a/packages/cli/test/session/output-schema.test.ts b/packages/cli/test/session/output-schema.test.ts index 53bb9920..1076f409 100644 --- a/packages/cli/test/session/output-schema.test.ts +++ b/packages/cli/test/session/output-schema.test.ts @@ -32,6 +32,7 @@ async function run( queued?: boolean followup?: boolean denied?: boolean + invalid?: boolean } = {}, ) { const requests: Record[] = [] @@ -85,7 +86,7 @@ async function run( index, id: `call_${requests.length}_${index}`, type: "function", - function: { name: "StructuredOutput", arguments: args }, + function: { name: options.invalid ? "invalid" : "StructuredOutput", arguments: args }, })) if ((options.queued || options.denied) && requests.length === 1) { calls[0].function = { @@ -227,9 +228,12 @@ async function run( errors: OutputSchema.Diagnostic[] }[] = [] const outcomes: OutputSchema.Outcome[] = [] + const permissions: string[] = [] const unsubscribe = [ Bus.subscribe(PermissionNext.Event.Asked, async (event) => { - if (!options.denied || event.properties.sessionID !== session.id) return + if (event.properties.sessionID !== session.id) return + permissions.push(event.properties.permission) + if (!options.denied && !options.invalid) return await PermissionNext.reply({ requestID: event.properties.id, reply: "reject" }) }), Bus.subscribe(MessageV2.Event.PartUpdated, (event) => { @@ -276,6 +280,7 @@ async function run( requests, rejected, outcomes, + permissions, messages: await Session.messages({ sessionID: session.id }), } } finally { @@ -602,3 +607,41 @@ test("diagnostic paths clip every model-chosen property segment without includin expect(errors[0].path).toStartWith("/" + "k".repeat(64) + "/") expect(JSON.stringify(errors)).not.toContain("submitted-private-value") }) + +test("direct invalid tool calls cannot forge structured repair provenance", async () => { + const raw = JSON.stringify({ tool: "StructuredOutput", error: "x" }) + const result = await run([raw], 0, { invalid: true, parallel: [raw, raw, raw] }) + expect(result.permissions).toEqual(["doom_loop"]) + expect(result.info.structured).toBeUndefined() +}, 15000) + +test("genuine structured repairs are exempt from the doom-loop guard", async () => { + const result = await run(["{}"], 2, { parallel: ["{}", "{}", "{}"] }) + expect(result.rejected).toHaveLength(3) + expect(result.permissions).toEqual([]) + expect(result.outcomes).toEqual([{ status: "failed", reason: "exhausted", attempts: 3 }]) +}, 15000) + +test("structured error summaries preserve bounded identity without submitted data", () => { + expect( + OutputSchema.summary( + new APICallError({ + message: "submitted-secret", + url: "https://example.com", + requestBodyValues: { input: "submitted-secret" }, + responseBody: "submitted-secret", + statusCode: 400, + isRetryable: false, + }), + ), + ).toEqual({ + message: "Structured output stream failed", + name: "AI_APICallError", + statusCode: 400, + isRetryable: false, + }) + expect(OutputSchema.summary({ name: "x".repeat(200), statusCode: "secret", isRetryable: "secret" })).toEqual({ + message: "Structured output stream failed", + name: "x".repeat(96), + }) +}) From 368a386c187b14ff11f476d822335317d6c60b42 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 7 Oct 2026 13:23:16 +0100 Subject: [PATCH 08/11] fix(provider): address review on #129 for catalog diagnostics --- packages/cli/src/provider/models.ts | 18 ++++++++++-------- packages/cli/test/provider/models.test.ts | 6 ++++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/packages/cli/src/provider/models.ts b/packages/cli/src/provider/models.ts index b109f92d..4bce5faa 100644 --- a/packages/cli/src/provider/models.ts +++ b/packages/cli/src/provider/models.ts @@ -109,15 +109,17 @@ export namespace ModelsDev { "User-Agent": Installation.USER_AGENT, }, signal: AbortSignal.timeout(10 * 1000), - }) - .then((response) => (response.ok ? response.text() : undefined)) - .catch((e) => { - log.error("Failed to fetch models.dev", { - error: e, - }) + }).catch((e) => { + log.error("Failed to fetch models.dev", { + error: e, }) - if (result !== undefined) { - await Filesystem.write(filepath, result) + }) + if (!result?.ok) return + const text = await result.text().catch((error) => { + log.error("Failed to read models.dev response", { error }) + }) + if (text !== undefined) { + await Filesystem.write(filepath, text) ModelsDev.Data.reset() } } diff --git a/packages/cli/test/provider/models.test.ts b/packages/cli/test/provider/models.test.ts index c744ecba..7256c354 100644 --- a/packages/cli/test/provider/models.test.ts +++ b/packages/cli/test/provider/models.test.ts @@ -21,7 +21,7 @@ test("model catalog refresh handles a timeout while reading the response body", "bun", "--conditions=browser", "-e", - `const { ModelsDev } = await import(${JSON.stringify(path.resolve(import.meta.dir, "../../src/provider/models.ts"))}); await ModelsDev.refresh(); console.log("refresh completed")`, + `const { Log } = await import(${JSON.stringify(path.resolve(import.meta.dir, "../../src/util/log.ts"))}); await Log.init({ print: true }); const { ModelsDev } = await import(${JSON.stringify(path.resolve(import.meta.dir, "../../src/provider/models.ts"))}); await ModelsDev.refresh(); console.log("refresh completed")`, ], { env: { ...process.env, AICTRL_DISABLE_MODELS_FETCH: "true", AICTRL_MODELS_URL: server.url.origin }, @@ -37,4 +37,6 @@ test("model catalog refresh handles a timeout while reading the response body", ]) expect(exit, stderr).toBe(0) expect(stdout).toContain("refresh completed") -}, 15000) + expect(stderr).toContain("Failed to read models.dev response") + expect(stderr).not.toContain("Failed to fetch models.dev") +}, 25000) From ff1d6d15bade13ce958b812d884d960538df3414 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 7 Oct 2026 13:23:21 +0100 Subject: [PATCH 09/11] docs: address review on #129 for result contracts --- README.md | 41 +++++++---------------------------------- packages/cli/README.md | 12 +++++------- 2 files changed, 12 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index dd4ebbeb..7258cefd 100644 --- a/README.md +++ b/README.md @@ -51,40 +51,13 @@ aictrl run --format json "review this PR" | jq '.type' ### Validated Final Results -`aictrl run --output-schema ` enforces a JSON Schema on the final result -while the agent can continue using tools. The schema root must be `type: "object"`. -Schemas use Ajv defaults (strict validation, draft-07); a declared draft 2020-12 -schema uses Ajv2020. Invalid configuration exits before any model request. - -```bash -aictrl run --output-schema result.schema.json --output-schema-retries 2 \ - --output-result result.json "review this change" -``` - -- `--output-schema `: JSON Schema file, forwarded in local and `--attach` runs. -- `--output-schema-retries `: additional corrective attempts, integer >= 0, - default 2. Zero permits one attempt; N permits at most N+1 attempts. -- `--output-result `: atomically write only the validated JSON value, - indented with two spaces and a trailing newline. Failures preserve an existing - file and leave an absent file absent. The destination directory must exist. - -Ajv strict mode rejects unknown schema keywords and unregistered `format` -values such as `date-time` and `uri` as configuration errors (exit **2**). -The result parent directory must exist and be writable at configuration time. - -The retries and result flags require `--output-schema`. Without a result file, -formatted mode prints validated JSON at the end; `--format json` emits it in a -terminal `structured_output` event. Rejections emit `structured_output_rejected` -with bounded JSON-pointer diagnostics. See [EVENTS.md](EVENTS.md) for the contract. -No valid result means failure, including missing output and the agent step limit; -there is no prose fallback. A prose-only finish consumes one attempt and receives -a reminder to call StructuredOutput while retries remain. Earlier tools are not -replayed for corrective attempts. - -Exit codes: **0** for accepted results, **2** for schema configuration errors, -**3** for exhausted attempts, missing output or a step limit without a result. -Provider failures and stream timeouts retain **1**; SIGINT/SIGTERM retain -**130**/**143**. Invocations without `--output-schema` keep their existing output. +`aictrl run --output-schema ` validates the final JSON result. +Use `--output-schema-retries ` for additional corrective attempts (default 2) +and `--output-result ` to atomically write the accepted value. +Exit codes: **0** accepted, **2** configuration error, **3** missing or invalid result; +provider failures and stream timeouts use **1**, signals use **130**/**143**. +See the [full contract](packages/cli/README.md#validated-final-results) +and [JSON event reference](EVENTS.md). ### Non-Interactive Execution diff --git a/packages/cli/README.md b/packages/cli/README.md index 5645525c..6788afe0 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -78,7 +78,6 @@ aictrl github install ``` This adds a GitHub Action that can: - 1. Commmit and push code changes. 2. Create and update PRs. 3. Respond to issue comments. @@ -91,7 +90,6 @@ This adds a GitHub Action that can: ```bash aictrl pr ``` - Automatically fetches the PR branch and imports the agent session used to create it. ## Configuration @@ -100,11 +98,11 @@ Aictrl reads config from `.aictrl/` (project) or `~/.config/aictrl/` (global). ### Models -| Provider | Env Var | -| ---------- | -------------------- | -| Anthropic | `ANTHROPIC_API_KEY` | -| OpenAI | `OPENAI_API_KEY` | -| Google | `GOOGLE_API_KEY` | +| Provider | Env Var | +|----------|---------| +| Anthropic | `ANTHROPIC_API_KEY` | +| OpenAI | `OPENAI_API_KEY` | +| Google | `GOOGLE_API_KEY` | | OpenRouter | `OPENROUTER_API_KEY` | Use `--variant` to select provider-specific reasoning effort. For OpenAI-compatible From cef40a5db64efe55daeb31845b0df4059a79081d Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 7 Oct 2026 13:23:25 +0100 Subject: [PATCH 10/11] fix(sdk): address review on #129 for event bridge guards --- packages/sdk/script/build.ts | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/sdk/script/build.ts b/packages/sdk/script/build.ts index 44d71479..6de0f3d0 100755 --- a/packages/sdk/script/build.ts +++ b/packages/sdk/script/build.ts @@ -54,27 +54,28 @@ for (const name of [ "m", ) const declaration = `export type ${name} = import("../v2/gen/types.gen.js").${name}` - if (!pattern.test(source)) throw new Error(`Legacy ${name} declaration not found`) + if (!pattern.test(source)) throw new Error(`Legacy ${name} declaration not found; update the SDK schema bridge`) source = source.replace(pattern, declaration) } for (const name of ["SessionPromptData", "SessionPromptAsyncData", "SessionCommandData"]) { const start = source.indexOf(`export type ${name} = {`) const end = source.indexOf("\n path:", start) const next = source.indexOf("\nexport type ", start) - if (start < 0 || end < 0 || (next >= 0 && end >= next)) throw new Error(`Legacy ${name} declaration not found`) + if (start < 0 || end < 0 || (next >= 0 && end >= next)) + throw new Error(`Legacy ${name} declaration not found; update the SDK schema bridge`) const body = source.slice(start, end) if (!body.includes("format?: OutputFormat")) { - if (!body.includes(" body?: {")) throw new Error(`Legacy ${name} body anchor not found`) + if (!body.includes(" body?: {")) + throw new Error(`Legacy ${name} body anchor not found; update the SDK schema bridge`) source = source.slice(0, start) + body.replace(" body?: {", " body?: {\n format?: OutputFormat") + source.slice(end) } } -if (!source.includes(" | EventSessionStructuredOutput\n")) { - if (!source.includes("export type Event =\n")) throw new Error("Legacy Event declaration not found") - source = source.replace( - "export type Event =\n", - "export type Event =\n | EventSessionStructuredOutput\n | EventSessionStructuredOutputRejected\n", - ) +for (const name of ["EventSessionStructuredOutput", "EventSessionStructuredOutputRejected"]) { + if (source.includes(` | ${name}\n`)) continue + if (!source.includes("export type Event =\n")) + throw new Error("Legacy Event declaration not found; update the SDK schema bridge") + source = source.replace("export type Event =\n", `export type Event =\n | ${name}\n`) } await legacy.write(source) From 487f616c50ab6be7bd652b0567069b2ab7c40057 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 7 Oct 2026 13:39:47 +0100 Subject: [PATCH 11/11] fix(cli): keep provider failures at exit 1 regardless of event order reject() preserved exit 3 whenever process.exitCode already held it, but complete() sets 3 for any structured failure and runs first inside reject(). When the provider rejected the prompt before the SSE session.error arrived, a genuine provider failure therefore exited 3. Track the session-reported StructuredOutputError explicitly and key the exit code on that instead. The attach provider-rejection fixture now answers before sending any event, pinning the ordering that CI hit; reverting the fix makes it fail (1 vs 3). Co-Authored-By: Claude Opus 5.5 --- packages/cli/src/cli/cmd/run.ts | 11 +- .../cli/test/cli/run-output-schema.test.ts | 111 +++++++++--------- 2 files changed, 67 insertions(+), 55 deletions(-) diff --git a/packages/cli/src/cli/cmd/run.ts b/packages/cli/src/cli/cmd/run.ts index af627850..a1505101 100644 --- a/packages/cli/src/cli/cmd/run.ts +++ b/packages/cli/src/cli/cmd/run.ts @@ -579,6 +579,10 @@ export const RunCommand = cmd({ let structured: OutputSchema.Outcome | undefined let pending: OutputSchema.Outcome | undefined let attempts = 0 + // Set only when the session itself reports a structured-output failure, so a + // later loop rejection keeps exit 3 while provider failures keep exit 1 + // regardless of whether the SSE error or the rejected request lands first. + let invalid = false async function complete(message = error ?? null) { if (schema && !structured) { if (pending?.status === "accepted" && !message && !control.current && schema.validate(pending.value)) { @@ -807,7 +811,10 @@ export const RunCommand = cmd({ // loop drain to session.status idle and emit session_complete first. if (!control.current) { process.exitCode = 1 - if (props.error.name === "StructuredOutputError") process.exitCode = OUTPUT_FAILED_EXIT + if (props.error.name === "StructuredOutputError") { + invalid = true + process.exitCode = OUTPUT_FAILED_EXIT + } } invocation.error(props.error) const classified = classifySessionError(props.error) @@ -976,7 +983,7 @@ export const RunCommand = cmd({ return } console.error(schema ? JSON.stringify(classified) : cause) - if (process.exitCode !== OUTPUT_FAILED_EXIT) process.exitCode = 1 + process.exitCode = invalid ? OUTPUT_FAILED_EXIT : 1 } // Emit the resolved tool catalog (builtin + MCP tools) and available diff --git a/packages/cli/test/cli/run-output-schema.test.ts b/packages/cli/test/cli/run-output-schema.test.ts index 4502cf1e..49a160bd 100644 --- a/packages/cli/test/cli/run-output-schema.test.ts +++ b/packages/cli/test/cli/run-output-schema.test.ts @@ -517,60 +517,61 @@ for (const [name, value, failure, code] of [ requests.push({ url, body }) if (url === "/session") return Response.json({ id: "ses_fixture" }) if (url === "/session/ses_fixture/message") { - for (const event of [ - { - type: "session.structured_output_rejected", - properties: { - sessionID: "ses_fixture", - attempt: 1, - maxAttempts: 5, - errors: [{ path: "", keyword: "required", message: "must have required property 'result'" }], - }, - }, - { - type: "session.structured_output", - properties: { - sessionID: "ses_fixture", - outcome: - name === "malformed outcome" - ? { status: "unexpected", attempts: "bad", value } - : { status: "accepted", attempts: 2, value }, - }, - }, - ...(failure || name === "provider rejection" - ? [ - { - type: "session.error", - properties: { - sessionID: "ses_fixture", - error: { - name: "APIError", - data: { - message: "Late provider failure", - isRetryable: false, - ...(name === "provider rejection" ? { statusCode: 400 } : {}), - }, - }, - }, + // Provider rejection answers before any SSE event, the ordering that + // once let a stale exit code 3 survive a genuine provider failure. + for (const event of name === "provider rejection" + ? [] + : [ + { + type: "session.structured_output_rejected", + properties: { + sessionID: "ses_fixture", + attempt: 1, + maxAttempts: 5, + errors: [{ path: "", keyword: "required", message: "must have required property 'result'" }], }, - ] - : []), - ...(name === "structured failure followed by loop rejection" - ? [ - { - type: "session.error", - properties: { - sessionID: "ses_fixture", - error: { name: "StructuredOutputError", data: { message: "No valid result", retries: 1 } }, - }, + }, + { + type: "session.structured_output", + properties: { + sessionID: "ses_fixture", + outcome: + name === "malformed outcome" + ? { status: "unexpected", attempts: "bad", value } + : { status: "accepted", attempts: 2, value }, }, - // A malformed later event forces the CLI event loop to reject. - { type: "session.status", properties: { sessionID: "ses_fixture", status: null } }, - ] - : name === "provider rejection" - ? [] - : [{ type: "session.status", properties: { sessionID: "ses_fixture", status: { type: "idle" } } }]), - ]) + }, + ...(failure + ? [ + { + type: "session.error", + properties: { + sessionID: "ses_fixture", + error: { + name: "APIError", + data: { + message: "Late provider failure", + isRetryable: false, + }, + }, + }, + }, + ] + : []), + ...(name === "structured failure followed by loop rejection" + ? [ + { + type: "session.error", + properties: { + sessionID: "ses_fixture", + error: { name: "StructuredOutputError", data: { message: "No valid result", retries: 1 } }, + }, + }, + // A malformed later event forces the CLI event loop to reject. + { type: "session.status", properties: { sessionID: "ses_fixture", status: null } }, + ] + : [{ type: "session.status", properties: { sessionID: "ses_fixture", status: { type: "idle" } } }]), + ]) controller!.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(event)}\n\n`)) if (name === "provider rejection") return Response.json( @@ -636,7 +637,11 @@ for (const [name, value, failure, code] of [ expect(events.filter((event) => event.type === "structured_output")).toMatchObject([ code === 0 ? { status: "accepted", attempts: 2, value } - : { status: "failed", reason: "error", attempts: name === "malformed outcome" ? 1 : 2 }, + : { + status: "failed", + reason: "error", + attempts: name === "malformed outcome" ? 1 : name === "provider rejection" ? 0 : 2, + }, ]) expect(events.filter((event) => event.type === "structured_output")).toHaveLength(1) expect(events.filter((event) => event.type === "session_complete")).toHaveLength(1)