From 2c58138b502e95096aa65c77c435ad5da1a22d5b Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 7 Oct 2026 14:18:56 +0100 Subject: [PATCH 1/8] fix(schema): bound schemas and corrective retry configuration --- packages/cli/src/cli/cmd/run.output.ts | 7 +++--- packages/cli/src/session/message-v2.ts | 2 +- packages/cli/src/session/output-schema.ts | 5 +++- .../cli/test/cli/run-output-schema.test.ts | 23 +++++++++++++++---- packages/cli/test/provider/models.test.ts | 8 ++++++- .../cli/test/session/output-schema.test.ts | 21 +++++++++++++++++ 6 files changed, 55 insertions(+), 11 deletions(-) diff --git a/packages/cli/src/cli/cmd/run.output.ts b/packages/cli/src/cli/cmd/run.output.ts index 5f930791..86dbe98b 100644 --- a/packages/cli/src/cli/cmd/run.output.ts +++ b/packages/cli/src/cli/cmd/run.output.ts @@ -1,4 +1,5 @@ import path from "path" +// Bun content I/O preserves createPath: false; fs/promises supplies metadata and atomic rename/removal. import fs from "fs/promises" import { OutputSchema } from "@/session/output-schema" @@ -12,8 +13,8 @@ export async function outputSchema(input: { schema?: string; retries?: number; r throw new Error("--output-schema-retries and --output-result require --output-schema") return } - if (input.retries !== undefined && (!Number.isSafeInteger(input.retries) || input.retries < 0)) - throw new Error("--output-schema-retries must be an integer >= 0") + if (input.retries !== undefined && (!Number.isSafeInteger(input.retries) || input.retries < 0 || input.retries > 10)) + throw new Error("--output-schema-retries must be an integer between 0 and 10") const text = await Bun.file(input.schema) .text() .catch(() => { @@ -27,7 +28,7 @@ export async function outputSchema(input: { schema?: string; retries?: number; r } })() if (!schema || typeof schema !== "object" || Array.isArray(schema)) - throw new Error(`${input.schema}: schema root must have type "object"`) + throw new Error(`${input.schema}: schema root must be a JSON object`) const canonical = schema as Record const validate = (() => { try { diff --git a/packages/cli/src/session/message-v2.ts b/packages/cli/src/session/message-v2.ts index 972f6f37..bbca5fc7 100644 --- a/packages/cli/src/session/message-v2.ts +++ b/packages/cli/src/session/message-v2.ts @@ -83,7 +83,7 @@ export namespace MessageV2 { .object({ type: z.literal("json_schema"), schema: z.record(z.string(), z.any()).meta({ ref: "JSONSchema" }), - retryCount: z.number().int().min(0).default(2), + retryCount: z.number().int().min(0).max(10).default(2), }) .meta({ ref: "OutputFormatJsonSchema", diff --git a/packages/cli/src/session/output-schema.ts b/packages/cli/src/session/output-schema.ts index 72eeff76..ef6d5a1e 100644 --- a/packages/cli/src/session/output-schema.ts +++ b/packages/cli/src/session/output-schema.ts @@ -1,6 +1,7 @@ import Ajv, { type ErrorObject } from "ajv" import Ajv2020 from "ajv/dist/2020" import z from "zod" +import { createHash } from "crypto" export namespace OutputSchema { export const Diagnostic = z.object({ path: z.string(), keyword: z.string(), message: z.string() }) @@ -21,7 +22,9 @@ export namespace OutputSchema { export function compile(schema: Record) { if (schema.type !== "object") throw new Error('schema root must have type "object"') - const key = JSON.stringify(schema) + const text = JSON.stringify(schema) + if (Buffer.byteLength(text) > 64 * 1024) throw new Error("serialized schema must not exceed 64 KiB") + const key = createHash("sha256").update(text).digest("hex") const cached = cache.get(key) if (cached) return cached const ajv = diff --git a/packages/cli/test/cli/run-output-schema.test.ts b/packages/cli/test/cli/run-output-schema.test.ts index 49a160bd..f02e672b 100644 --- a/packages/cli/test/cli/run-output-schema.test.ts +++ b/packages/cli/test/cli/run-output-schema.test.ts @@ -234,7 +234,11 @@ for (const [name, options, reason] of [ ["unreadable", { flags: ["--output-schema", "absent.json"] }, "cannot read"], ["invalid JSON", { content: "{broken" }, "invalid JSON"], ["uncompilable", { content: '{"type":"object","unknownKeyword":true}' }, "strict mode"], - ["non-object root", { content: '{"type":"array"}' }, "object"], + ["non-object root", { content: '{"type":"array"}' }, 'schema root must have type "object"'], + ["array JSON root", { content: "[]" }, "schema root must be a JSON object"], + ["null JSON root", { content: "null" }, "schema root must be a JSON object"], + ["scalar JSON root", { content: '"schema"' }, "schema root must be a JSON object"], + ["oversized schema", { content: JSON.stringify({ type: "object", description: "x".repeat(64 * 1024) }) }, "64 KiB"], [ "unknown format", { content: '{"type":"object","properties":{"result":{"type":"string","format":"date-time"}}}' }, @@ -257,11 +261,20 @@ for (const [name, options, reason] of [ ], ["retries without schema", { flags: ["--output-schema-retries", "1"] }, "require --output-schema"], ["result without schema", { flags: ["--output-result", "result.json"] }, "require --output-schema"], - ["negative retries", { flags: ["--output-schema", "schema.json", "--output-schema-retries", "-1"] }, "integer >= 0"], + [ + "excessive retries", + { flags: ["--output-schema", "schema.json", "--output-schema-retries", "11"] }, + "integer between 0 and 10", + ], + [ + "negative retries", + { flags: ["--output-schema", "schema.json", "--output-schema-retries", "-1"] }, + "integer between 0 and 10", + ], [ "fractional retries", { flags: ["--output-schema", "schema.json", "--output-schema-retries", "1.5"] }, - "integer >= 0", + "integer between 0 and 10", ], [ "unsupported draft", @@ -279,8 +292,8 @@ for (const [name, options, reason] of [ expect(result.stderr).toContain(reason) const event = result.events.find((event) => event.type === "invocation_error") expect((event!.message as string).length).toBeLessThanOrEqual(300) - if (name === "negative retries" || name === "fractional retries") - expect(event?.message).toBe("--output-schema-retries must be an integer >= 0") + if (name === "negative retries" || name === "fractional retries" || name === "excessive retries") + expect(event?.message).toBe("--output-schema-retries must be an integer between 0 and 10") expect(event).toMatchObject({ code: "OUTPUT_SCHEMA_CONFIG", message: expect.stringContaining(reason) }) if ("content" in options) expect(result.stderr).toContain("schema.json") expect(result.requests).toEqual([]) diff --git a/packages/cli/test/provider/models.test.ts b/packages/cli/test/provider/models.test.ts index 7256c354..75324219 100644 --- a/packages/cli/test/provider/models.test.ts +++ b/packages/cli/test/provider/models.test.ts @@ -29,7 +29,13 @@ test("model catalog refresh handles a timeout while reading the response body", stderr: "pipe", }, ) - using cleanup = { [Symbol.dispose]: () => proc.kill() } + const timer = setTimeout(() => proc.kill("SIGKILL"), 20000) + using cleanup = { + [Symbol.dispose]() { + clearTimeout(timer) + proc.kill("SIGKILL") + }, + } const [stdout, stderr, exit] = await Promise.all([ new Response(proc.stdout).text(), new Response(proc.stderr).text(), diff --git a/packages/cli/test/session/output-schema.test.ts b/packages/cli/test/session/output-schema.test.ts index 1076f409..c4c98ada 100644 --- a/packages/cli/test/session/output-schema.test.ts +++ b/packages/cli/test/session/output-schema.test.ts @@ -645,3 +645,24 @@ test("structured error summaries preserve bounded identity without submitted dat name: "x".repeat(96), }) }) + +test("serialized schema limit counts bytes and accepts exactly 64 KiB", () => { + const schema = { type: "object", description: "" } + const size = Buffer.byteLength(JSON.stringify(schema)) + schema.description = "x".repeat(64 * 1024 - size) + expect(OutputSchema.compile(schema)({})).toBe(true) + expect(() => OutputSchema.compile({ ...schema, description: schema.description + "x" })).toThrow("64 KiB") + expect(() => OutputSchema.compile({ ...schema, description: "é".repeat(33 * 1024) })).toThrow("64 KiB") + expect(OutputSchema.compile(schema)).toBe(OutputSchema.compile(JSON.parse(JSON.stringify(schema)))) +}) + +test("prompt path rejects an oversized schema before any provider request", async () => { + await expect(run(["unused"], 0, { schema: { type: "object", description: "x".repeat(64 * 1024) } })).rejects.toThrow( + "64 KiB", + ) +}) + +test("retryCount accepts the upper bound and rejects values above ten", () => { + expect(MessageV2.OutputFormatJsonSchema.parse({ type: "json_schema", schema, retryCount: 10 }).retryCount).toBe(10) + expect(() => MessageV2.OutputFormatJsonSchema.parse({ type: "json_schema", schema, retryCount: 11 })).toThrow() +}) From 2e438e27fd2e6934b2a25d9094d6e262a5e3a0c8 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 7 Oct 2026 14:20:34 +0100 Subject: [PATCH 2/8] fix(run): settle schema failures and count every rejected attempt --- EVENTS.md | 18 +++--- README.md | 4 +- packages/cli/README.md | 8 ++- packages/cli/src/cli/cmd/run.ts | 56 ++++++++++++++----- packages/cli/src/session/prompt.ts | 20 +++---- packages/cli/test/cli/run-event-race.test.ts | 8 +-- .../cli/test/cli/run-output-schema.test.ts | 31 ++++++---- .../cli/test/session/output-schema.test.ts | 14 ++++- 8 files changed, 109 insertions(+), 50 deletions(-) diff --git a/EVENTS.md b/EVENTS.md index 4229767b..22e044a3 100644 --- a/EVENTS.md +++ b/EVENTS.md @@ -539,9 +539,10 @@ The schema is validated before any model request, with Ajv defaults and `allErrors: true` (draft-07, or Ajv2020 for a declared draft 2020-12). Validation does not coerce types, insert defaults or remove extra properties. -`--output-schema-retries ` permits N additional corrective attempts (default 2). -Zero permits exactly one attempt. `--output-result ` writes only accepted -JSON using two-space indentation and a trailing newline, with a temporary file +`--output-schema-retries ` permits N additional corrective turns (0–10, default 2). +Zero permits exactly one model request. A step can contain multiple StructuredOutput +calls; every rejected call is reported and counted even after the budget is spent. +`--output-result ` writes only accepted JSON using two-space indentation and a trailing newline, with a temporary file in the destination directory followed by an atomic rename. Failures preserve the file. Both flags require `--output-schema`; the destination directory must exist and be writable at configuration time. Without `--output-result`, formatted mode prints final JSON; NDJSON includes the @@ -562,7 +563,8 @@ and a model turn that finishes in prose without a final tool call: ``` The usual event envelope fields also apply. `attempt` is one-based; -`maxAttempts` equals `retryCount + 1`. Diagnostics include at most ten errors, +`maxAttempts` equals `retryCount + 1`, the corrective turn budget; `attempt` can +exceed it when a step contains multiple calls. Diagnostics include at most ten errors, with bounded JSON-pointer paths, keywords and messages. Their encoded payload is under 2 KB and never includes submitted values or raw JSON arguments. A `path` can contain property names submitted by the model (for example, in map or @@ -578,7 +580,8 @@ validation always uses the canonical schema. ### `structured_output` -Exactly one terminal event per schema-enabled session, before `session_complete`: +Exactly one terminal event per schema-enabled prompt run, before `session_complete`. +A headless `aictrl run` has one prompt run: ```json { "type": "structured_output", "status": "accepted", "attempts": 2, "value": { "result": "ok" } } @@ -595,8 +598,9 @@ and prose-only finishes without a final call. Other tool calls do not count. Compaction turns do not consume a missing-result attempt. Discarded provider streams restore the budget and captured result; their rejection events remain as telemetry. A valid call in a parallel step wins regardless of call order if -that step started with budget remaining, and its attempt count stays within the -configured maximum. +that step started with budget remaining. The attempt count includes every call +in that step and can exceed `maxAttempts`; the N+1 bound applies to corrective +turns/requests. When the budget is spent, the last rejection determines the reason: `missing` for a prose-only finish, otherwise `exhausted`. Failure reasons are: diff --git a/README.md b/README.md index 7258cefd..af14b63f 100644 --- a/README.md +++ b/README.md @@ -52,8 +52,10 @@ aictrl run --format json "review this PR" | jq '.type' ### Validated Final Results `aictrl run --output-schema ` validates the final JSON result. -Use `--output-schema-retries ` for additional corrective attempts (default 2) +Use `--output-schema-retries ` for additional corrective turns (0–10, default 2) and `--output-result ` to atomically write the accepted value. +Schema mode emits exactly one terminal `structured_output` event per prompt run; +a headless `aictrl run` has one prompt run. Exit codes: **0** accepted, **2** configuration error, **3** missing or invalid result; provider failures and stream timeouts use **1**, signals use **130**/**143**. See the [full contract](packages/cli/README.md#validated-final-results) diff --git a/packages/cli/README.md b/packages/cli/README.md index 6788afe0..85cf9aeb 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -41,8 +41,9 @@ aictrl run --output-schema result.schema.json --output-schema-retries 2 \ ``` - `--output-schema `: JSON Schema file, forwarded in local and `--attach` runs. -- `--output-schema-retries `: additional corrective attempts, integer >= 0, - default 2. Zero permits one attempt; N permits at most N+1 attempts. +- `--output-schema-retries `: additional corrective turns, integer from 0 to 10, + default 2. Zero permits one model request; N permits at most N+1 corrective + requests. Each rejected call is counted, including multiple calls in one step. - `--output-result `: atomically write only the validated JSON value, indented with two spaces and a trailing newline. Failures preserve an existing file and leave an absent file absent. The destination directory must exist. @@ -53,7 +54,8 @@ The result parent directory must exist and be writable at configuration time. The retries and result flags require `--output-schema`. Without a result file, formatted mode prints validated JSON at the end; `--format json` emits it in a -terminal `structured_output` event. Rejections emit `structured_output_rejected` +terminal `structured_output` event, exactly one per prompt run. A headless +`aictrl run` has one prompt run. Rejections emit `structured_output_rejected` with bounded JSON-pointer diagnostics. See [EVENTS.md](../../EVENTS.md) for the contract. No valid result means failure, including missing output and the agent step limit; there is no prose fallback. A prose-only finish consumes one attempt and receives diff --git a/packages/cli/src/cli/cmd/run.ts b/packages/cli/src/cli/cmd/run.ts index a1505101..b8dd5cc0 100644 --- a/packages/cli/src/cli/cmd/run.ts +++ b/packages/cli/src/cli/cmd/run.ts @@ -350,7 +350,7 @@ export const RunCommand = cmd({ describe: "format: default (formatted) or json (raw JSON events)", }) .option("output-schema", { type: "string", describe: "JSON Schema file for the final result (object root)" }) - .option("output-schema-retries", { type: "number", describe: "additional corrective attempts (default: 2)" }) + .option("output-schema-retries", { type: "number", describe: "additional corrective turns (0-10, default: 2)" }) .option("output-result", { type: "string", describe: "atomically write only the validated JSON result to this file", @@ -561,7 +561,8 @@ export const RunCommand = cmd({ return false } - const events = await sdk.event.subscribe() + const subscription = new AbortController() + const events = await sdk.event.subscribe({ signal: subscription.signal }) let error: string | undefined const startTime = Date.now() const childSessions = new Set() @@ -1014,12 +1015,21 @@ export const RunCommand = cmd({ }) } - const loopDone = loop() - .then(() => complete()) - .catch(reject) + const loopDone = loop().then( + () => ({ ok: true as const }), + (cause: unknown) => ({ ok: false as const, cause }), + ) + const finish = async (result: Awaited) => { + if (result.ok) return complete() + return reject(result.cause) + } + let failure: { cause: unknown } | undefined + const failed = (cause: unknown) => { + failure = { cause } + } if (control.current) { - await loopDone + await finish(await loopDone) await Shutdown.flush() return } @@ -1034,7 +1044,7 @@ export const RunCommand = cmd({ ...(schema ? { format: schema.format } : {}), } const request = sdk.session.command(input(sessionID, opts)) - await (schema ? request.catch(reject) : request) + await (schema ? request.catch(failed) : request) } else { const model = args.model ? Provider.parseModel(args.model) : undefined const opts = { @@ -1045,11 +1055,15 @@ export const RunCommand = cmd({ ...(schema ? { format: schema.format } : {}), } const request = sdk.session.prompt(input(sessionID, opts)) - await (schema ? request.catch(reject) : request) + await (schema ? request.catch(failed) : request) } - if (schema && error) { + if (schema && (failure || error)) { abort() + subscription.abort() + const result = await loopDone + if (failure) await reject(failure.cause) + if (!failure) await finish(result) await Shutdown.flush() return } @@ -1059,7 +1073,19 @@ export const RunCommand = cmd({ // (e.g., Session.get() fails, model not found), no session.status idle event // is emitted, so loopDone would hang forever. Racing ensures we surface // the error and exit. - await Promise.race([loopDone, promptResult.then(() => loopDone, reject)]) + const result = await Promise.race([ + loopDone, + promptResult.then( + () => loopDone, + (cause: unknown) => ({ ok: false as const, cause }), + ), + ]) + if (schema && !result.ok) { + abort() + subscription.abort() + await loopDone + } + await finish(result) await Shutdown.flush() } @@ -1124,7 +1150,7 @@ export const RunCommand = cmd({ }, }, event: { - async subscribe() { + async subscribe(opts: { signal: AbortSignal }) { const queue: any[] = [] let resolve: (() => void) | null = null const handler = (event: { payload: any }) => { @@ -1132,19 +1158,23 @@ export const RunCommand = cmd({ resolve?.() } GlobalBus.on("event", handler) + const stop = () => resolve?.() + opts.signal.addEventListener("abort", stop) const stream = (async function* () { try { - while (true) { - while (queue.length > 0) { + while (!opts.signal.aborted) { + while (queue.length > 0 && !opts.signal.aborted) { yield queue.shift() } + if (opts.signal.aborted) break await new Promise((r) => { resolve = r }) } } finally { GlobalBus.off("event", handler) + opts.signal.removeEventListener("abort", stop) } })() return { stream } diff --git a/packages/cli/src/session/prompt.ts b/packages/cli/src/session/prompt.ts index 97c8c2b4..378235b9 100644 --- a/packages/cli/src/session/prompt.ts +++ b/packages/cli/src/session/prompt.ts @@ -725,16 +725,14 @@ export namespace SessionPrompt { const rejection = async (errors: OutputSchema.Diagnostic[]) => { if (!contract) throw new Error("Missing output schema") - if (attempts < contract.format.retryCount + 1) { - attempts++ - rejected++ - await Bus.publish(Session.Event.StructuredOutputRejected, { - sessionID, - attempt: attempts, - maxAttempts: contract.format.retryCount + 1, - errors, - }) - } + attempts++ + rejected++ + await Bus.publish(Session.Event.StructuredOutputRejected, { + sessionID, + attempt: attempts, + maxAttempts: contract.format.retryCount + 1, + errors, + }) return OutputSchema.error(errors).message } @@ -753,7 +751,7 @@ export namespace SessionPrompt { async onSuccess(output) { if (structuredOutput !== undefined) return if (!available) throw new Error("StructuredOutput attempt budget exhausted") - attempts = Math.min(attempts + 1, format.retryCount + 1) + attempts++ structuredOutput = output }, }) diff --git a/packages/cli/test/cli/run-event-race.test.ts b/packages/cli/test/cli/run-event-race.test.ts index 9bece209..6f066a77 100644 --- a/packages/cli/test/cli/run-event-race.test.ts +++ b/packages/cli/test/cli/run-event-race.test.ts @@ -7,10 +7,10 @@ describe("run.ts GlobalBus listener race (OPS-05)", () => { test("GlobalBus.on is registered before async generator in subscribe", async () => { const source = await Bun.file(RUN_SRC).text() // Find the subscribe method in the local SDK object - const subscribeIdx = source.indexOf("async subscribe()") + const subscribeIdx = source.indexOf("async subscribe(") expect(subscribeIdx).toBeGreaterThan(-1) // Extract the subscribe method body - const subscribeBody = source.slice(subscribeIdx, subscribeIdx + 600) + const subscribeBody = source.slice(subscribeIdx, source.indexOf("return { stream }", subscribeIdx)) // GlobalBus.on must appear BEFORE the generator definition const onIdx = subscribeBody.indexOf('GlobalBus.on("event"') const generatorIdx = subscribeBody.indexOf("(async function* ()") @@ -21,9 +21,9 @@ describe("run.ts GlobalBus listener race (OPS-05)", () => { test("GlobalBus.off cleanup is still in generator finally block", async () => { const source = await Bun.file(RUN_SRC).text() - const subscribeIdx = source.indexOf("async subscribe()") + const subscribeIdx = source.indexOf("async subscribe(") expect(subscribeIdx).toBeGreaterThan(-1) - const subscribeBody = source.slice(subscribeIdx, subscribeIdx + 800) + const subscribeBody = source.slice(subscribeIdx, source.indexOf("return { stream }", subscribeIdx)) // Generator must still have finally with GlobalBus.off expect(subscribeBody).toContain("finally") expect(subscribeBody).toMatch(/GlobalBus\.off\("event"/) diff --git a/packages/cli/test/cli/run-output-schema.test.ts b/packages/cli/test/cli/run-output-schema.test.ts index f02e672b..7ecd5456 100644 --- a/packages/cli/test/cli/run-output-schema.test.ts +++ b/packages/cli/test/cli/run-output-schema.test.ts @@ -503,12 +503,17 @@ for (const [name, value, failure, code] of [ ["malformed outcome", { result: "attached" }, false, 3], ["structured failure followed by loop rejection", {}, false, 3], ["provider rejection", { result: "submitted-secret" }, false, 1], + ["provider rejection after loop failure", { result: "submitted-secret" }, false, 1], ] as const) { test(`attach ${name} forwards the canonical schema and validates final acceptance`, async () => { await using tmp = await tmpdir() await Bun.write(path.join(tmp.path, "schema.json"), JSON.stringify(schema)) + const rejection = name.startsWith("provider rejection") const requests: { url: string; body: Record }[] = [] - let controller: ReadableStreamDefaultController | undefined + let ready: (value: ReadableStreamDefaultController) => void = () => {} + const connected = new Promise>((resolve) => { + ready = resolve + }) const server = Bun.serve({ port: 0, idleTimeout: 0, @@ -518,8 +523,8 @@ for (const [name, value, failure, code] of [ return new Response( new ReadableStream({ start(value) { - controller = value - controller.enqueue(new TextEncoder().encode(": ready\n\n")) + value.enqueue(new TextEncoder().encode(": ready\n\n")) + ready(value) }, }), { headers: { "Content-Type": "text/event-stream" } }, @@ -530,10 +535,13 @@ for (const [name, value, failure, code] of [ requests.push({ url, body }) if (url === "/session") return Response.json({ id: "ses_fixture" }) if (url === "/session/ses_fixture/message") { + const controller = await connected // Provider rejection answers before any SSE event, the ordering that // once let a stale exit code 3 survive a genuine provider failure. - for (const event of name === "provider rejection" - ? [] + for (const event of rejection + ? name === "provider rejection after loop failure" + ? [{ type: "session.status", properties: { sessionID: "ses_fixture", status: null } }] + : [] : [ { type: "session.structured_output_rejected", @@ -585,12 +593,14 @@ for (const [name, value, failure, code] of [ ] : [{ type: "session.status", properties: { sessionID: "ses_fixture", status: { type: "idle" } } }]), ]) - controller!.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(event)}\n\n`)) - if (name === "provider rejection") + controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(event)}\n\n`)) + if (rejection) { + if (name === "provider rejection after loop failure") await Bun.sleep(25) return Response.json( { name: "APIError", data: { message: "Fixture provider failed", statusCode: 400, isRetryable: false } }, { status: 400 }, ) + } return Response.json({}) } return new Response("unexpected request", { status: 400 }) @@ -631,8 +641,9 @@ for (const [name, value, failure, code] of [ proc.exited, ]) expect(exit, stderr + stdout).toBe(code) - if (name === "provider rejection") { - expect(stderr).toContain('"code":"400"') + if (rejection) { + expect(stderr.split("\n").filter((line) => line.startsWith("{") && line.includes('"reason":'))).toHaveLength(1) + expect(stderr.split("\n").filter((line) => line.includes('"code":"400"'))).toHaveLength(1) expect(stderr).toContain('"reason":"unknown"') expect(stdout + stderr).not.toContain("submitted-secret") } @@ -653,7 +664,7 @@ for (const [name, value, failure, code] of [ : { status: "failed", reason: "error", - attempts: name === "malformed outcome" ? 1 : name === "provider rejection" ? 0 : 2, + attempts: name === "malformed outcome" ? 1 : rejection ? 0 : 2, }, ]) expect(events.filter((event) => event.type === "structured_output")).toHaveLength(1) diff --git a/packages/cli/test/session/output-schema.test.ts b/packages/cli/test/session/output-schema.test.ts index c4c98ada..e52d5adf 100644 --- a/packages/cli/test/session/output-schema.test.ts +++ b/packages/cli/test/session/output-schema.test.ts @@ -583,7 +583,8 @@ for (const parallel of [ expect(result.requests).toHaveLength(1) expect(result.info.error).toBeUndefined() expect(result.info.structured).toEqual({ result: "accepted" }) - expect(result.outcomes).toEqual([{ status: "accepted", attempts: 1, value: { result: "accepted" } }]) + expect(result.outcomes).toEqual([{ status: "accepted", attempts: 2, value: { result: "accepted" } }]) + expect(result.rejected).toMatchObject([{ attempt: parallel[0] === "{}" ? 1 : 2, maxAttempts: 1 }]) }, 15000) } @@ -666,3 +667,14 @@ test("retryCount accepts the upper bound and rejects values above ten", () => { expect(MessageV2.OutputFormatJsonSchema.parse({ type: "json_schema", schema, retryCount: 10 }).retryCount).toBe(10) expect(() => MessageV2.OutputFormatJsonSchema.parse({ type: "json_schema", schema, retryCount: 11 })).toThrow() }) + +test("every parallel rejection is counted after the corrective budget is spent", async () => { + const result = await run(["unused"], 0, { parallel: ["{}", "{}", "{}"] }) + expect(result.requests).toHaveLength(1) + expect(result.rejected).toMatchObject([ + { attempt: 1, maxAttempts: 1 }, + { attempt: 2, maxAttempts: 1 }, + { attempt: 3, maxAttempts: 1 }, + ]) + expect(result.outcomes).toEqual([{ status: "failed", reason: "exhausted", attempts: 3 }]) +}) From 2cef742d5fff1614e3727662b030ac3714c1fee6 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 7 Oct 2026 14:21:06 +0100 Subject: [PATCH 3/8] fix(provider): preserve output constraints and force corrective tool choice --- packages/cli/src/session/llm.ts | 3 ++- packages/cli/src/session/prompt.ts | 10 ++++++-- .../cli/test/cli/run-output-schema.test.ts | 10 ++++++-- .../cli/test/session/output-schema.test.ts | 24 ++++++++++++++++++- 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/packages/cli/src/session/llm.ts b/packages/cli/src/session/llm.ts index e6b5d128..64a6fe6b 100644 --- a/packages/cli/src/session/llm.ts +++ b/packages/cli/src/session/llm.ts @@ -9,6 +9,7 @@ import { type StreamTextResult, type Tool, type ToolSet, + type ToolChoice, tool, jsonSchema, } from "ai" @@ -40,7 +41,7 @@ export namespace LLM { small?: boolean tools: Record retries?: number - toolChoice?: "auto" | "required" | "none" + toolChoice?: ToolChoice structured?: { reject: (input: string) => Promise; checkpoint: () => () => void; repairs: Set } } diff --git a/packages/cli/src/session/prompt.ts b/packages/cli/src/session/prompt.ts index 378235b9..174942d1 100644 --- a/packages/cli/src/session/prompt.ts +++ b/packages/cli/src/session/prompt.ts @@ -821,7 +821,7 @@ export namespace SessionPrompt { ], tools, model, - toolChoice: contract ? "required" : undefined, + toolChoice: contract ? (reminder ? { type: "tool", toolName: "StructuredOutput" } : "required") : undefined, structured: contract ? { reject, @@ -1125,8 +1125,14 @@ export namespace SessionPrompt { }): AITool & { id: "StructuredOutput" } { const validate = input.validate ?? OutputSchema.compile(input.schema) const schema = Object.fromEntries(Object.entries(input.schema).filter(([key]) => key !== "$schema")) + const canonical = JSON.stringify(input.schema) + const bytes = Buffer.from(canonical) + const description = + bytes.length <= 8 * 1024 + ? `Canonical JSON Schema: ${canonical}` + : `Canonical JSON Schema (truncated; the validator enforces the full schema): ${new TextDecoder().decode(bytes.subarray(0, 8 * 1024), { stream: true })}` const result = tool({ - description: STRUCTURED_OUTPUT_DESCRIPTION, + description: `${STRUCTURED_OUTPUT_DESCRIPTION}\n\n${description}`, inputSchema: jsonSchema(input.model ? ProviderTransform.schema(input.model, schema) : schema, { validate(value) { return validate(value) diff --git a/packages/cli/test/cli/run-output-schema.test.ts b/packages/cli/test/cli/run-output-schema.test.ts index 7ecd5456..664ec188 100644 --- a/packages/cli/test/cli/run-output-schema.test.ts +++ b/packages/cli/test/cli/run-output-schema.test.ts @@ -337,7 +337,10 @@ for (const json of [true, false]) { }) expect(result.exit, result.stderr).toBe(0) expect(result.requests).toHaveLength(2) - expect(result.requests.map((request) => request.tool_choice)).toEqual(["required", "required"]) + expect(result.requests.map((request) => request.tool_choice)).toEqual([ + "required", + { type: "function", function: { name: "StructuredOutput" } }, + ]) expect(JSON.stringify(result.requests[1].messages)).toContain("call StructuredOutput with the final result") expect(result.result).toBe(JSON.stringify({ result: "accepted" }, null, 2) + "\n") if (!json) return @@ -477,7 +480,10 @@ test("schema attach repairs a prose finish through the HTTP session boundary", a retryCount: 2, }) expect(result.requests).toHaveLength(2) - expect(result.requests.map((request) => request.tool_choice)).toEqual(["required", "required"]) + expect(result.requests.map((request) => request.tool_choice)).toEqual([ + "required", + { type: "function", function: { name: "StructuredOutput" } }, + ]) expect(result.result).toBe(JSON.stringify({ result: "accepted" }, null, 2) + "\n") expect(result.events.filter((event) => event.type === "structured_output_rejected")).toMatchObject([ { attempt: 1, errors: [{ path: "", keyword: "missing", message: "call StructuredOutput with the final result" }] }, diff --git a/packages/cli/test/session/output-schema.test.ts b/packages/cli/test/session/output-schema.test.ts index e52d5adf..a4d4c07a 100644 --- a/packages/cli/test/session/output-schema.test.ts +++ b/packages/cli/test/session/output-schema.test.ts @@ -491,7 +491,8 @@ test("Gemini tool schema is transformed but canonical numeric enum remains autho test("prose-only finish is a counted corrective attempt before a valid result", async () => { const result = await run(["prose", '{"result":"repaired"}'], 1) expect(result.requests).toHaveLength(2) - result.requests.forEach((request) => expect(request.tool_choice).toBe("required")) + expect(result.requests[0].tool_choice).toBe("required") + expect(result.requests[1].tool_choice).toEqual({ type: "function", function: { name: "StructuredOutput" } }) expect(result.info.structured).toEqual({ result: "repaired" }) expect(result.info.error).toBeUndefined() expect(result.rejected).toEqual([ @@ -678,3 +679,24 @@ test("every parallel rejection is counted after the corrective budget is spent", ]) expect(result.outcomes).toEqual([{ status: "failed", reason: "exhausted", attempts: 3 }]) }) + +test("provider tool description includes canonical pattern constraints", async () => { + const canonical = { + type: "object", + properties: { result: { type: "string", pattern: "^OK-[0-9]{4}$" } }, + required: ["result"], + additionalProperties: false, + } + const result = await run(['{"result":"OK-1234"}'], 0, { schema: canonical, gemini: true }) + const tools = result.requests[0].tools as { function: { name: string; description: string } }[] + expect(tools.find((tool) => tool.function.name === "StructuredOutput")?.function.description).toContain( + JSON.stringify(canonical), + ) +}) + +test("tool description caps the canonical schema at 8 KiB and explains truncation", () => { + const canonical = { type: "object", description: "x".repeat(9 * 1024) } + const tool = SessionPrompt.createStructuredOutputTool({ schema: canonical, onSuccess() {} }) + expect(tool.description).toContain("truncated; the validator enforces the full schema") + expect(tool.description?.split(": ").at(-1)).toBe(JSON.stringify(canonical).slice(0, 8 * 1024)) +}) From 7184b474fd034dca15204de4c24d5db751dfe70e Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 7 Oct 2026 14:44:28 +0100 Subject: [PATCH 4/8] fix(run): report resolved output-schema paths `aictrl run` changes into --dir before reading --output-schema, so relative schema and result paths resolve against --dir (as --file paths already do). A relative path given from the shell's directory then failed with a bare "schema.json: cannot read schema file". Resolve both paths up front and name the absolute path in every configuration error, and document the behaviour. Co-Authored-By: Claude Opus 5.5 --- packages/cli/README.md | 2 ++ packages/cli/src/cli/cmd/run.output.ts | 18 +++++++++++------- .../cli/test/cli/run-output-schema.test.ts | 10 +++++++++- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/packages/cli/README.md b/packages/cli/README.md index 85cf9aeb..8177f69f 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -41,6 +41,8 @@ aictrl run --output-schema result.schema.json --output-schema-retries 2 \ ``` - `--output-schema `: JSON Schema file, forwarded in local and `--attach` runs. + Relative `--output-schema` and `--output-result` paths resolve against `--dir` + when it is given (as `--file` paths do); configuration errors print the resolved path. - `--output-schema-retries `: additional corrective turns, integer from 0 to 10, default 2. Zero permits one model request; N permits at most N+1 corrective requests. Each rejected call is counted, including multiple calls in one step. diff --git a/packages/cli/src/cli/cmd/run.output.ts b/packages/cli/src/cli/cmd/run.output.ts index 86dbe98b..3c90bcb6 100644 --- a/packages/cli/src/cli/cmd/run.output.ts +++ b/packages/cli/src/cli/cmd/run.output.ts @@ -15,30 +15,34 @@ export async function outputSchema(input: { schema?: string; retries?: number; r } if (input.retries !== undefined && (!Number.isSafeInteger(input.retries) || input.retries < 0 || input.retries > 10)) throw new Error("--output-schema-retries must be an integer between 0 and 10") - const text = await Bun.file(input.schema) + // Relative paths resolve against --dir (run changes into it first), like --file. + // Report the resolved path so a misplaced relative path is obvious. + const file = path.resolve(input.schema) + const text = await Bun.file(file) .text() .catch(() => { - throw new Error(`${input.schema}: cannot read schema file`) + throw new Error(`${file}: cannot read schema file`) }) const schema: unknown = (() => { try { return JSON.parse(text) } catch { - throw new Error(`${input.schema}: invalid JSON`) + throw new Error(`${file}: invalid JSON`) } })() if (!schema || typeof schema !== "object" || Array.isArray(schema)) - throw new Error(`${input.schema}: schema root must be a JSON object`) + throw new Error(`${file}: schema root must be a JSON object`) const canonical = schema as Record const validate = (() => { try { return OutputSchema.compile(canonical) } catch (error) { - throw new Error(`${input.schema}: ${error instanceof Error ? error.message : "cannot compile schema"}`) + throw new Error(`${file}: ${error instanceof Error ? error.message : "cannot compile schema"}`) } })() if (input.result) { - const directory = path.dirname(input.result) + const result = path.resolve(input.result) + const directory = path.dirname(result) const writable = await fs .stat(directory) .then(async (stat) => { @@ -47,7 +51,7 @@ export async function outputSchema(input: { schema?: string; retries?: number; r return true }) .catch(() => false) - if (!writable) throw new Error(`${input.result}: output-result parent directory must exist and be writable`) + if (!writable) throw new Error(`${result}: output-result parent directory must exist and be writable`) } return { format: { type: "json_schema" as const, schema: canonical, retryCount: input.retries ?? 2 }, validate } } diff --git a/packages/cli/test/cli/run-output-schema.test.ts b/packages/cli/test/cli/run-output-schema.test.ts index 664ec188..7f180966 100644 --- a/packages/cli/test/cli/run-output-schema.test.ts +++ b/packages/cli/test/cli/run-output-schema.test.ts @@ -232,6 +232,13 @@ async function run( for (const [name, options, reason] of [ ["unreadable", { flags: ["--output-schema", "absent.json"] }, "cannot read"], + [ + // run changes into --dir first, so relative paths resolve there (as --file does) + // and the error names the resolved path rather than the bare argument. + "relative schema under --dir", + { flags: ["--dir", "home", "--output-schema", "schema.json"] }, + `${path.sep}home${path.sep}schema.json: cannot read schema file`, + ], ["invalid JSON", { content: "{broken" }, "invalid JSON"], ["uncompilable", { content: '{"type":"object","unknownKeyword":true}' }, "strict mode"], ["non-object root", { content: '{"type":"array"}' }, 'schema root must have type "object"'], @@ -715,6 +722,7 @@ test("schema config NDJSON clips the specific compilation cause", async () => { expect(result.requests).toEqual([]) const event = result.events.find((event) => event.type === "invocation_error") expect(event?.code).toBe("OUTPUT_SCHEMA_CONFIG") - expect(event?.message).toStartWith("schema.json: strict mode: unknown keyword:") + expect(event?.message).toStartWith(path.sep) + expect(event?.message).toContain(`${path.sep}schema.json: strict mode: unknown keyword:`) expect((event?.message as string).length).toBe(300) }, 25000) From d9fecedfa6987fa362da125015dd8bd372bed9da Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 7 Oct 2026 15:19:14 +0100 Subject: [PATCH 5/8] fix(run): address review on #134 for schema bounds and path diagnostics - Bound schema structure (depth 64, 10k nested objects) as well as bytes: ~2,000 nested levels fit in 16 KB yet overflow Ajv's recursive codegen with a RangeError. The check is iterative. - Lead configuration errors with the reason and put the resolved path last, so the 300-char clip drops path text rather than the cause. - Rename the event-stream AbortController to subscriptionAbort. - Platform-neutral clip assertion; re-wrap EVENTS.md; move the shared path-resolution note out of a single flag's bullet. Co-Authored-By: Claude Opus 5.5 --- EVENTS.md | 9 ++++++--- packages/cli/README.md | 5 +++-- packages/cli/src/cli/cmd/run.output.ts | 13 +++++++------ packages/cli/src/cli/cmd/run.ts | 8 ++++---- packages/cli/src/session/output-schema.ts | 18 ++++++++++++++++++ .../cli/test/cli/run-output-schema.test.ts | 17 ++++++++++++++--- 6 files changed, 52 insertions(+), 18 deletions(-) diff --git a/EVENTS.md b/EVENTS.md index 22e044a3..fdecb3c8 100644 --- a/EVENTS.md +++ b/EVENTS.md @@ -542,9 +542,12 @@ does not coerce types, insert defaults or remove extra properties. `--output-schema-retries ` permits N additional corrective turns (0–10, default 2). Zero permits exactly one model request. A step can contain multiple StructuredOutput calls; every rejected call is reported and counted even after the budget is spent. -`--output-result ` writes only accepted JSON using two-space indentation and a trailing newline, with a temporary file -in the destination directory followed by an atomic rename. Failures preserve the -file. Both flags require `--output-schema`; the destination directory must exist and be writable at configuration time. +`--output-result ` writes only accepted JSON using two-space indentation and +a trailing newline, with a temporary file in the destination directory followed +by an atomic rename. Failures preserve the file. Both flags require +`--output-schema`; the destination directory must exist and be writable at +configuration time. Relative paths resolve against `--dir` when it is given, and +configuration errors name the resolved path after the reason. Without `--output-result`, formatted mode prints final JSON; NDJSON includes the value in the terminal event below. diff --git a/packages/cli/README.md b/packages/cli/README.md index 8177f69f..db6a63c2 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -41,8 +41,6 @@ aictrl run --output-schema result.schema.json --output-schema-retries 2 \ ``` - `--output-schema `: JSON Schema file, forwarded in local and `--attach` runs. - Relative `--output-schema` and `--output-result` paths resolve against `--dir` - when it is given (as `--file` paths do); configuration errors print the resolved path. - `--output-schema-retries `: additional corrective turns, integer from 0 to 10, default 2. Zero permits one model request; N permits at most N+1 corrective requests. Each rejected call is counted, including multiple calls in one step. @@ -50,6 +48,9 @@ aictrl run --output-schema result.schema.json --output-schema-retries 2 \ indented with two spaces and a trailing newline. Failures preserve an existing file and leave an absent file absent. The destination directory must exist. +Relative `--output-schema` and `--output-result` paths resolve against `--dir` when +it is given (as `--file` paths do); configuration errors print the resolved path. + Ajv strict mode rejects unknown schema keywords and unregistered `format` values such as `date-time` and `uri` as configuration errors (exit **2**). The result parent directory must exist and be writable at configuration time. diff --git a/packages/cli/src/cli/cmd/run.output.ts b/packages/cli/src/cli/cmd/run.output.ts index 3c90bcb6..a01b50c7 100644 --- a/packages/cli/src/cli/cmd/run.output.ts +++ b/packages/cli/src/cli/cmd/run.output.ts @@ -16,28 +16,29 @@ export async function outputSchema(input: { schema?: string; retries?: number; r if (input.retries !== undefined && (!Number.isSafeInteger(input.retries) || input.retries < 0 || input.retries > 10)) throw new Error("--output-schema-retries must be an integer between 0 and 10") // Relative paths resolve against --dir (run changes into it first), like --file. - // Report the resolved path so a misplaced relative path is obvious. + // Report the resolved path after the reason, so clipping (300 chars) drops path + // text rather than the cause. const file = path.resolve(input.schema) const text = await Bun.file(file) .text() .catch(() => { - throw new Error(`${file}: cannot read schema file`) + throw new Error(`cannot read schema file (${file})`) }) const schema: unknown = (() => { try { return JSON.parse(text) } catch { - throw new Error(`${file}: invalid JSON`) + throw new Error(`invalid JSON (${file})`) } })() if (!schema || typeof schema !== "object" || Array.isArray(schema)) - throw new Error(`${file}: schema root must be a JSON object`) + throw new Error(`schema root must be a JSON object (${file})`) const canonical = schema as Record const validate = (() => { try { return OutputSchema.compile(canonical) } catch (error) { - throw new Error(`${file}: ${error instanceof Error ? error.message : "cannot compile schema"}`) + throw new Error(`${error instanceof Error ? error.message : "cannot compile schema"} (${file})`) } })() if (input.result) { @@ -51,7 +52,7 @@ export async function outputSchema(input: { schema?: string; retries?: number; r return true }) .catch(() => false) - if (!writable) throw new Error(`${result}: output-result parent directory must exist and be writable`) + if (!writable) throw new Error(`output-result parent directory must exist and be writable (${result})`) } return { format: { type: "json_schema" as const, schema: canonical, retryCount: input.retries ?? 2 }, validate } } diff --git a/packages/cli/src/cli/cmd/run.ts b/packages/cli/src/cli/cmd/run.ts index b8dd5cc0..af3f95f5 100644 --- a/packages/cli/src/cli/cmd/run.ts +++ b/packages/cli/src/cli/cmd/run.ts @@ -561,8 +561,8 @@ export const RunCommand = cmd({ return false } - const subscription = new AbortController() - const events = await sdk.event.subscribe({ signal: subscription.signal }) + const subscriptionAbort = new AbortController() + const events = await sdk.event.subscribe({ signal: subscriptionAbort.signal }) let error: string | undefined const startTime = Date.now() const childSessions = new Set() @@ -1060,7 +1060,7 @@ export const RunCommand = cmd({ if (schema && (failure || error)) { abort() - subscription.abort() + subscriptionAbort.abort() const result = await loopDone if (failure) await reject(failure.cause) if (!failure) await finish(result) @@ -1082,7 +1082,7 @@ export const RunCommand = cmd({ ]) if (schema && !result.ok) { abort() - subscription.abort() + subscriptionAbort.abort() await loopDone } await finish(result) diff --git a/packages/cli/src/session/output-schema.ts b/packages/cli/src/session/output-schema.ts index ef6d5a1e..240d20ca 100644 --- a/packages/cli/src/session/output-schema.ts +++ b/packages/cli/src/session/output-schema.ts @@ -20,10 +20,28 @@ export namespace OutputSchema { // acceptance, including schemas cloned by Zod or loaded from session storage. const cache = new Map>() + // Ajv codegen recurses per nesting level: ~2,000 levels fit in 16 KB yet overflow + // the stack, so bound structure as well as bytes. Iterative, so the check itself + // cannot overflow. + const MAX_DEPTH = 64 + const MAX_NODES = 10_000 + function bound(schema: unknown) { + const stack: [unknown, number][] = [[schema, 0]] + let nodes = 0 + while (stack.length) { + const [value, depth] = stack.pop()! + if (!value || typeof value !== "object") continue + if (depth > MAX_DEPTH) throw new Error(`schema nesting must not exceed ${MAX_DEPTH} levels`) + if (++nodes > MAX_NODES) throw new Error(`schema must not exceed ${MAX_NODES} nested objects`) + for (const child of Object.values(value)) stack.push([child, depth + 1]) + } + } + export function compile(schema: Record) { if (schema.type !== "object") throw new Error('schema root must have type "object"') const text = JSON.stringify(schema) if (Buffer.byteLength(text) > 64 * 1024) throw new Error("serialized schema must not exceed 64 KiB") + bound(schema) const key = createHash("sha256").update(text).digest("hex") const cached = cache.get(key) if (cached) return cached diff --git a/packages/cli/test/cli/run-output-schema.test.ts b/packages/cli/test/cli/run-output-schema.test.ts index 7f180966..50f6e02d 100644 --- a/packages/cli/test/cli/run-output-schema.test.ts +++ b/packages/cli/test/cli/run-output-schema.test.ts @@ -237,7 +237,7 @@ for (const [name, options, reason] of [ // and the error names the resolved path rather than the bare argument. "relative schema under --dir", { flags: ["--dir", "home", "--output-schema", "schema.json"] }, - `${path.sep}home${path.sep}schema.json: cannot read schema file`, + `${path.sep}home${path.sep}schema.json)`, ], ["invalid JSON", { content: "{broken" }, "invalid JSON"], ["uncompilable", { content: '{"type":"object","unknownKeyword":true}' }, "strict mode"], @@ -246,6 +246,17 @@ for (const [name, options, reason] of [ ["null JSON root", { content: "null" }, "schema root must be a JSON object"], ["scalar JSON root", { content: '"schema"' }, "schema root must be a JSON object"], ["oversized schema", { content: JSON.stringify({ type: "object", description: "x".repeat(64 * 1024) }) }, "64 KiB"], + [ + // ~2,000 levels is only ~16 KB but overflows Ajv's recursive codegen. + "deeply nested schema", + { + content: JSON.stringify({ + type: "object", + properties: { a: Array.from({ length: 2000 }).reduce((inner) => ({ not: inner }), { type: "string" }) }, + }), + }, + "nesting must not exceed 64 levels", + ], [ "unknown format", { content: '{"type":"object","properties":{"result":{"type":"string","format":"date-time"}}}' }, @@ -722,7 +733,7 @@ test("schema config NDJSON clips the specific compilation cause", async () => { expect(result.requests).toEqual([]) const event = result.events.find((event) => event.type === "invocation_error") expect(event?.code).toBe("OUTPUT_SCHEMA_CONFIG") - expect(event?.message).toStartWith(path.sep) - expect(event?.message).toContain(`${path.sep}schema.json: strict mode: unknown keyword:`) + // The reason leads so the 300-char clip removes path text, never the cause. + expect(event?.message).toStartWith("strict mode: unknown keyword:") expect((event?.message as string).length).toBe(300) }, 25000) From 4487718cbb744f6ad5e9762909c42cb8447542c2 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 7 Oct 2026 15:27:21 +0100 Subject: [PATCH 6/8] fix(session): enforce schema bounds and preserve replay compatibility --- packages/cli/src/session/message-v2.ts | 7 ++- packages/cli/src/session/output-schema.ts | 13 ++++-- packages/cli/src/session/prompt.ts | 12 ++--- .../cli/test/session/output-schema.test.ts | 44 +++++++++++++++---- .../test/session/structured-output.test.ts | 2 + 5 files changed, 59 insertions(+), 19 deletions(-) diff --git a/packages/cli/src/session/message-v2.ts b/packages/cli/src/session/message-v2.ts index bbca5fc7..18a0dbe1 100644 --- a/packages/cli/src/session/message-v2.ts +++ b/packages/cli/src/session/message-v2.ts @@ -83,7 +83,12 @@ export namespace MessageV2 { .object({ type: z.literal("json_schema"), schema: z.record(z.string(), z.any()).meta({ ref: "JSONSchema" }), - retryCount: z.number().int().min(0).max(10).default(2), + retryCount: z + .number() + .int() + .min(0) + .transform((n) => Math.min(n, 10)) + .default(2), }) .meta({ ref: "OutputFormatJsonSchema", diff --git a/packages/cli/src/session/output-schema.ts b/packages/cli/src/session/output-schema.ts index 240d20ca..0a6c05ba 100644 --- a/packages/cli/src/session/output-schema.ts +++ b/packages/cli/src/session/output-schema.ts @@ -37,12 +37,17 @@ export namespace OutputSchema { } } + export function canonical(schema: Record) { + bound(schema) + const text = JSON.stringify(schema) + const bytes = Buffer.byteLength(text) + if (bytes > 64 * 1024) throw new Error("serialized schema must not exceed 64 KiB") + return { text, bytes } + } + export function compile(schema: Record) { if (schema.type !== "object") throw new Error('schema root must have type "object"') - const text = JSON.stringify(schema) - if (Buffer.byteLength(text) > 64 * 1024) throw new Error("serialized schema must not exceed 64 KiB") - bound(schema) - const key = createHash("sha256").update(text).digest("hex") + const key = createHash("sha256").update(canonical(schema).text).digest("hex") const cached = cache.get(key) if (cached) return cached const ajv = diff --git a/packages/cli/src/session/prompt.ts b/packages/cli/src/session/prompt.ts index 174942d1..9e8bb72a 100644 --- a/packages/cli/src/session/prompt.ts +++ b/packages/cli/src/session/prompt.ts @@ -750,7 +750,7 @@ export namespace SessionPrompt { model, async onSuccess(output) { if (structuredOutput !== undefined) return - if (!available) throw new Error("StructuredOutput attempt budget exhausted") + if (!available) throw new Error("StructuredOutput corrective turn budget exhausted") attempts++ structuredOutput = output }, @@ -1123,14 +1123,14 @@ export namespace SessionPrompt { model?: Provider.Model onSuccess: (output: unknown) => void | Promise }): AITool & { id: "StructuredOutput" } { + OutputSchema.canonical(input.schema) const validate = input.validate ?? OutputSchema.compile(input.schema) const schema = Object.fromEntries(Object.entries(input.schema).filter(([key]) => key !== "$schema")) - const canonical = JSON.stringify(input.schema) - const bytes = Buffer.from(canonical) + const canonical = OutputSchema.canonical(schema) const description = - bytes.length <= 8 * 1024 - ? `Canonical JSON Schema: ${canonical}` - : `Canonical JSON Schema (truncated; the validator enforces the full schema): ${new TextDecoder().decode(bytes.subarray(0, 8 * 1024), { stream: true })}` + canonical.bytes <= 8 * 1024 + ? `Canonical JSON Schema: ${canonical.text}` + : `Canonical JSON Schema (truncated; the validator enforces the full schema): ${new TextDecoder().decode(Buffer.from(canonical.text).subarray(0, 8 * 1024))}` const result = tool({ description: `${STRUCTURED_OUTPUT_DESCRIPTION}\n\n${description}`, inputSchema: jsonSchema(input.model ? ProviderTransform.schema(input.model, schema) : schema, { diff --git a/packages/cli/test/session/output-schema.test.ts b/packages/cli/test/session/output-schema.test.ts index a4d4c07a..b010a590 100644 --- a/packages/cli/test/session/output-schema.test.ts +++ b/packages/cli/test/session/output-schema.test.ts @@ -664,9 +664,9 @@ test("prompt path rejects an oversized schema before any provider request", asyn ) }) -test("retryCount accepts the upper bound and rejects values above ten", () => { +test("retryCount accepts the upper bound and clamps legacy values above ten", () => { expect(MessageV2.OutputFormatJsonSchema.parse({ type: "json_schema", schema, retryCount: 10 }).retryCount).toBe(10) - expect(() => MessageV2.OutputFormatJsonSchema.parse({ type: "json_schema", schema, retryCount: 11 })).toThrow() + expect(MessageV2.OutputFormatJsonSchema.parse({ type: "json_schema", schema, retryCount: 50 }).retryCount).toBe(10) }) test("every parallel rejection is counted after the corrective budget is spent", async () => { @@ -694,9 +694,37 @@ test("provider tool description includes canonical pattern constraints", async ( ) }) -test("tool description caps the canonical schema at 8 KiB and explains truncation", () => { - const canonical = { type: "object", description: "x".repeat(9 * 1024) } - const tool = SessionPrompt.createStructuredOutputTool({ schema: canonical, onSuccess() {} }) - expect(tool.description).toContain("truncated; the validator enforces the full schema") - expect(tool.description?.split(": ").at(-1)).toBe(JSON.stringify(canonical).slice(0, 8 * 1024)) -}) +for (const [name, description] of [ + ["ASCII", "x".repeat(9 * 1024)], + ["multibyte", "x" + "€".repeat(4 * 1024)], +]) { + test(`tool description caps ${name} schema at 8 KiB and explains truncation`, () => { + const canonical = { type: "object", description } + const tool = SessionPrompt.createStructuredOutputTool({ schema: canonical, onSuccess() {} }) + const text = JSON.stringify(canonical) + const expected = new TextDecoder().decode(Buffer.from(text).subarray(0, 8 * 1024)) + expect(tool.description).toContain("truncated; the validator enforces the full schema") + expect(tool.description?.split(": ").at(-1)).toBe(expected) + if (name === "multibyte") expect(expected).toEndWith("�") + }) +} + +for (const [name, schema, reason] of [ + ["oversized", { type: "object", description: "é".repeat(33 * 1024) }, "64 KiB"], + [ + "over-deep", + { type: "object", properties: { a: Array.from({ length: 65 }).reduce((inner) => ({ not: inner }), {}) } }, + "nesting must not exceed 64 levels", + ], + ["too many objects", { type: "object", examples: Array.from({ length: 10_000 }, () => ({})) }, "10000"], +] as const) { + test(`custom validator cannot bypass ${name} schema bounds`, () => { + expect(() => + SessionPrompt.createStructuredOutputTool({ + schema, + validate: OutputSchema.compile({ type: "object" }), + onSuccess() {}, + }), + ).toThrow(reason) + }) +} diff --git a/packages/cli/test/session/structured-output.test.ts b/packages/cli/test/session/structured-output.test.ts index 7d1ca1f5..de8f5cbc 100644 --- a/packages/cli/test/session/structured-output.test.ts +++ b/packages/cli/test/session/structured-output.test.ts @@ -213,6 +213,8 @@ describe("structured-output.createStructuredOutputTool", () => { // AI SDK wraps schema in { jsonSchema: {...} } const inputSchema = tool.inputSchema as any expect(inputSchema.jsonSchema?.$schema).toBeUndefined() + expect(tool.description).toContain(`Canonical JSON Schema: ${JSON.stringify(inputSchema.jsonSchema)}`) + expect(tool.description).not.toContain("$schema") }) test("execute calls onSuccess with valid args", async () => { From 0f53783d5eed53b3cae2c1b0f0d657cb9ed993c2 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 7 Oct 2026 15:27:26 +0100 Subject: [PATCH 7/8] fix(run): drain queued telemetry before terminal failure --- packages/cli/src/cli/cmd/run.ts | 9 +++-- packages/cli/test/cli/run-event-race.test.ts | 8 +++- .../cli/test/cli/run-output-schema.test.ts | 40 +++++++++++++++++++ 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/cli/cmd/run.ts b/packages/cli/src/cli/cmd/run.ts index af3f95f5..32966551 100644 --- a/packages/cli/src/cli/cmd/run.ts +++ b/packages/cli/src/cli/cmd/run.ts @@ -1158,13 +1158,16 @@ export const RunCommand = cmd({ resolve?.() } GlobalBus.on("event", handler) - const stop = () => resolve?.() + const stop = () => { + GlobalBus.off("event", handler) + resolve?.() + } opts.signal.addEventListener("abort", stop) const stream = (async function* () { try { - while (!opts.signal.aborted) { - while (queue.length > 0 && !opts.signal.aborted) { + while (true) { + while (queue.length > 0) { yield queue.shift() } if (opts.signal.aborted) break diff --git a/packages/cli/test/cli/run-event-race.test.ts b/packages/cli/test/cli/run-event-race.test.ts index 6f066a77..55f9c427 100644 --- a/packages/cli/test/cli/run-event-race.test.ts +++ b/packages/cli/test/cli/run-event-race.test.ts @@ -10,7 +10,9 @@ describe("run.ts GlobalBus listener race (OPS-05)", () => { const subscribeIdx = source.indexOf("async subscribe(") expect(subscribeIdx).toBeGreaterThan(-1) // Extract the subscribe method body - const subscribeBody = source.slice(subscribeIdx, source.indexOf("return { stream }", subscribeIdx)) + const end = source.indexOf("return { stream }", subscribeIdx) + expect(end).toBeGreaterThan(-1) + const subscribeBody = source.slice(subscribeIdx, end) // GlobalBus.on must appear BEFORE the generator definition const onIdx = subscribeBody.indexOf('GlobalBus.on("event"') const generatorIdx = subscribeBody.indexOf("(async function* ()") @@ -23,7 +25,9 @@ describe("run.ts GlobalBus listener race (OPS-05)", () => { const source = await Bun.file(RUN_SRC).text() const subscribeIdx = source.indexOf("async subscribe(") expect(subscribeIdx).toBeGreaterThan(-1) - const subscribeBody = source.slice(subscribeIdx, source.indexOf("return { stream }", subscribeIdx)) + const end = source.indexOf("return { stream }", subscribeIdx) + expect(end).toBeGreaterThan(-1) + const subscribeBody = source.slice(subscribeIdx, end) // Generator must still have finally with GlobalBus.off expect(subscribeBody).toContain("finally") expect(subscribeBody).toMatch(/GlobalBus\.off\("event"/) diff --git a/packages/cli/test/cli/run-output-schema.test.ts b/packages/cli/test/cli/run-output-schema.test.ts index 50f6e02d..7882c4c8 100644 --- a/packages/cli/test/cli/run-output-schema.test.ts +++ b/packages/cli/test/cli/run-output-schema.test.ts @@ -18,6 +18,7 @@ async function run( timeout?: boolean abort?: boolean failure?: boolean + queued?: boolean attach?: boolean locked?: boolean denied?: boolean @@ -107,6 +108,26 @@ async function run( if (options.existing) await Bun.write(path.join(tmp.path, "result.json"), options.existing) const home = path.join(tmp.path, "home") await fs.mkdir(home) + if (options.queued) { + // Inject a synchronous provider failure after a burst of real bus events so + // the local subscription still has a rejection queued when it is aborted. + await Bun.write( + path.join(tmp.path, "failure.ts"), + `import { SessionPrompt } from ${JSON.stringify(path.resolve(import.meta.dir, "../../src/session/prompt.ts"))} +import { GlobalBus } from ${JSON.stringify(path.resolve(import.meta.dir, "../../src/bus/global.ts"))} +import { MessageV2 } from ${JSON.stringify(path.resolve(import.meta.dir, "../../src/session/message-v2.ts"))} +SessionPrompt.prompt = async (input) => { + for (let n = 0; n < 32; n++) GlobalBus.emit("event", { payload: { type: "fixture.queued", properties: {} } }) + GlobalBus.emit("event", { payload: { + type: "session.structured_output_rejected", + properties: { sessionID: input.sessionID, attempt: 1, maxAttempts: 3, + errors: [{ path: "", keyword: "required", message: "must have required property 'result'" }] } + } }) + throw new MessageV2.APIError({ message: "Fixture provider failed", statusCode: 400, isRetryable: false }) +} +`, + ) + } const attached: { url: string; body: Record }[] = [] // The headless package has no HTTP server; bridge its real session APIs rather // than synthesizing responses or terminal events for the attach regression. @@ -171,6 +192,7 @@ async function run( "bun", "run", "--conditions=browser", + ...(options.queued ? ["--preload", path.join(tmp.path, "failure.ts")] : []), entry, "run", ...(bridge ? ["--attach", bridge.url.origin, "--dir", tmp.path] : []), @@ -737,3 +759,21 @@ test("schema config NDJSON clips the specific compilation cause", async () => { expect(event?.message).toStartWith("strict mode: unknown keyword:") expect((event?.message as string).length).toBe(300) }, 25000) + +test("local abort drains rejection queued immediately before provider failure", async () => { + const result = await run({ queued: true }) + expect(result.exit, result.stderr + result.stdout).toBe(1) + expect(result.events.filter((event) => event.type === "structured_output_rejected")).toMatchObject([ + { attempt: 1, maxAttempts: 3, errors: [{ keyword: "required" }] }, + ]) + expect(result.events.filter((event) => event.type === "structured_output")).toMatchObject([ + { status: "failed", reason: "error", attempts: 1 }, + ]) + expect(result.events.filter((event) => event.type === "structured_output")).toHaveLength(1) + expect(result.events.filter((event) => event.type === "session_complete")).toHaveLength(1) + expect(result.stderr.split("\n").filter((line) => line.startsWith("{") && line.includes('"reason":'))).toHaveLength(1) + const types = result.events.map((event) => event.type) + expect(types.indexOf("structured_output_rejected")).toBeLessThan(types.indexOf("structured_output")) + expect(types.indexOf("structured_output")).toBeLessThan(types.indexOf("session_complete")) + expect(types.at(-1)).toBe("invocation_complete") +}, 25000) From 1f1bf14c8e125873a8a957b780dd04231cc0557d Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 7 Oct 2026 15:27:30 +0100 Subject: [PATCH 8/8] docs: clarify structured output trust and limits --- EVENTS.md | 8 ++++++-- README.md | 3 +++ packages/cli/README.md | 12 ++++++++---- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/EVENTS.md b/EVENTS.md index fdecb3c8..99563e8d 100644 --- a/EVENTS.md +++ b/EVENTS.md @@ -538,9 +538,13 @@ These events are additive in schema version `"1"` and emitted only when The schema is validated before any model request, with Ajv defaults and `allErrors: true` (draft-07, or Ajv2020 for a declared draft 2020-12). Validation does not coerce types, insert defaults or remove extra properties. +Serialized schemas are limited to 64 KiB, depth 64 and 10,000 nested objects. +The schema file is trusted instruction input: annotations (`description`, `title`, +`examples`) are shown to the model in the StructuredOutput tool description. `--output-schema-retries ` permits N additional corrective turns (0–10, default 2). -Zero permits exactly one model request. A step can contain multiple StructuredOutput +Zero permits only the initial turn; N permits the initial turn and at most N +corrective turns. A step can contain multiple StructuredOutput calls; every rejected call is reported and counted even after the budget is spent. `--output-result ` writes only accepted JSON using two-space indentation and a trailing newline, with a temporary file in the destination directory followed @@ -603,7 +607,7 @@ streams restore the budget and captured result; their rejection events remain as telemetry. A valid call in a parallel step wins regardless of call order if that step started with budget remaining. The attempt count includes every call in that step and can exceed `maxAttempts`; the N+1 bound applies to corrective -turns/requests. +turns. When the budget is spent, the last rejection determines the reason: `missing` for a prose-only finish, otherwise `exhausted`. Failure reasons are: diff --git a/README.md b/README.md index af14b63f..e90d8f6f 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,9 @@ aictrl run --format json "review this PR" | jq '.type' `aictrl run --output-schema ` validates the final JSON result. Use `--output-schema-retries ` for additional corrective turns (0–10, default 2) and `--output-result ` to atomically write the accepted value. +Zero retries permits only the initial turn; N permits the initial turn and at +most N corrective turns. Serialized schemas are limited to 64 KiB, depth 64 and 10,000 +nested objects. Schema mode emits exactly one terminal `structured_output` event per prompt run; a headless `aictrl run` has one prompt run. Exit codes: **0** accepted, **2** configuration error, **3** missing or invalid result; diff --git a/packages/cli/README.md b/packages/cli/README.md index db6a63c2..812e2c48 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -34,6 +34,7 @@ aictrl run --format json "scan for secrets" | jq '.properties.part.text' while the agent can continue using tools. The schema root must be `type: "object"`. Schemas use Ajv defaults (strict validation, draft-07); a declared draft 2020-12 schema uses Ajv2020. Invalid configuration exits before any model request. +Serialized schemas are limited to 64 KiB, depth 64 and 10,000 nested objects. ```bash aictrl run --output-schema result.schema.json --output-schema-retries 2 \ @@ -41,9 +42,12 @@ aictrl run --output-schema result.schema.json --output-schema-retries 2 \ ``` - `--output-schema `: JSON Schema file, forwarded in local and `--attach` runs. + The file is trusted instruction input: annotations (`description`, `title`, + `examples`) are shown to the model in the StructuredOutput tool description. - `--output-schema-retries `: additional corrective turns, integer from 0 to 10, - default 2. Zero permits one model request; N permits at most N+1 corrective - requests. Each rejected call is counted, including multiple calls in one step. + default 2. Zero permits only the initial turn; N permits the initial turn and + at most N corrective turns. Each rejected call is counted, including multiple + calls in one step. - `--output-result `: atomically write only the validated JSON value, indented with two spaces and a trailing newline. Failures preserve an existing file and leave an absent file absent. The destination directory must exist. @@ -63,10 +67,10 @@ with bounded JSON-pointer diagnostics. See [EVENTS.md](../../EVENTS.md) for the No valid result means failure, including missing output and the agent step limit; there is no prose fallback. A prose-only finish consumes one attempt and receives a reminder to call StructuredOutput while retries remain. Earlier tools are not -replayed for corrective attempts. +replayed for corrective turns. Exit codes: **0** for accepted results, **2** for schema configuration errors, -**3** for exhausted attempts, missing output or a step limit without a result. +**3** for exhausted corrective turns, missing output or a step limit without a result. Provider failures and stream timeouts retain **1**; SIGINT/SIGTERM retain **130**/**143**. Invocations without `--output-schema` keep their existing output.