diff --git a/packages/platform-apple/src/snapshot-source/__tests__/exec-timeout-fixture.ts b/packages/platform-apple/src/snapshot-source/__tests__/exec-timeout-fixture.ts new file mode 100644 index 0000000000..98eaaa1ea8 --- /dev/null +++ b/packages/platform-apple/src/snapshot-source/__tests__/exec-timeout-fixture.ts @@ -0,0 +1,22 @@ +import { runCmd } from '@agent-device/host-kit/command'; +import type { AppError } from '@agent-device/kernel/errors'; + +/** Long enough that the kill always lands first, short enough to stay out of the test budget. */ +const STALL_TIMEOUT_MS = 5; + +/** + * The failure a probe or a compile hits: the real error `exec.ts` raises when it kills a command at + * the budget it was handed. Hand-building that shape would let the probes keep classifying a kill + * that exec.ts no longer reports as one. + */ +export async function execKillTimeoutError(): Promise { + // `timeoutMs` bounds the sleep well below its own request, so exec kills it rather than waiting. + return await runCmd(process.execPath, ['-e', 'setTimeout(() => {}, 30_000)'], { + timeoutMs: STALL_TIMEOUT_MS, + }).then( + () => { + throw new Error('the fixture command answered instead of being killed at its timeout'); + }, + (error: unknown) => error as AppError, + ); +} diff --git a/packages/platform-apple/src/snapshot-source/adapter.test.ts b/packages/platform-apple/src/snapshot-source/adapter.test.ts index c6cdab1f88..5c4f9435fa 100644 --- a/packages/platform-apple/src/snapshot-source/adapter.test.ts +++ b/packages/platform-apple/src/snapshot-source/adapter.test.ts @@ -51,7 +51,8 @@ test('the Simulator AX source returns raw acquisition facts and discloses unsupp hint, }); assert.equal(fixture.builds, 1); - assert.equal(fixture.runs, 6); + // Four identity probes and one clang build: the identity read execs one Xcode-owned binary. + assert.equal(fixture.runs, 5); assert.equal(result.stage, 'acquired'); assert.equal(result.acquisition.producer, 'simulator-ax-bridge'); assert.equal(result.acquisition.intent, 'full'); @@ -92,7 +93,7 @@ test('the Simulator AX source returns raw acquisition facts and discloses unsupp }); assert.equal(outcome.stage, 'failed'); if (outcome.stage === 'failed') assert.equal(outcome.failure.kind, 'stale-target'); - assert.equal(fixture.runs, 6); + assert.equal(fixture.runs, 5); } finally { await source.close(); await rm(root, { recursive: true, force: true }); diff --git a/packages/platform-apple/src/snapshot-source/cache-identity.test.ts b/packages/platform-apple/src/snapshot-source/cache-identity.test.ts index fc71799fd0..f1b1a10998 100644 --- a/packages/platform-apple/src/snapshot-source/cache-identity.test.ts +++ b/packages/platform-apple/src/snapshot-source/cache-identity.test.ts @@ -5,25 +5,29 @@ import { createSnapshotSourceHost } from './host.ts'; import { readSnapshotSourceToolchain } from './cache-identity.ts'; import { createSnapshotSourceDeadline, type SnapshotSourceDeadline } from './deadline.ts'; import { SnapshotSourceError } from './errors.ts'; -import type { ExecOptions, ExecResult } from '@agent-device/host-kit/command'; +import { execKillTimeoutError } from './__tests__/exec-timeout-fixture.ts'; +import { + isCommandTimeoutError, + type ExecOptions, + type ExecResult, +} from '@agent-device/host-kit/command'; import type { SnapshotSourceHost } from './types.ts'; -// Apple's syspolicyd signature scan blocks the first xcodebuild/xcrun exec -// after a fresh macOS host boots for roughly 18 to 19 seconds; the immediate -// next exec of the same tool is instant (#2422). These cases exercise the -// resulting one-retry policy, and the deadline that bounds it, without waiting -// on a real cold-start stall: the fake clock only moves when a probe actually -// blocks for the timeout it was handed, so a case that claims the budget was -// spent had to spend it. +// Apple's syspolicyd signature scan blocks the first exec of an Xcode-owned tool after a fresh +// macOS host boots for roughly 18 to 19 seconds; the immediate next exec of the same tool is +// instant (#2422). These cases exercise the resulting one-retry policy, and the deadline that +// bounds it, without waiting on a real cold-start stall: the fake clock only moves when a probe +// actually blocks for the timeout it was handed, so a case that claims the budget was spent had to +// spend it. test('a cold-start toolchain probe recovers on retry, and the retry gets only what the stall left', async () => { const clock = { nowMs: 0 }; const timeouts: number[] = []; let calls = 0; - const host = fakeToolchainHost((command, args, options) => { + const host = fakeToolchainHost(async (command, args, options) => { calls += 1; timeouts.push(options.timeoutMs ?? 0); - if (calls === 1) throw blockForWholeTimeout(clock, command, options); + if (calls === 1) throw await blockForWholeTimeout(clock, options); return toolchainAnswer(command, args); }); @@ -36,28 +40,47 @@ test('a cold-start toolchain probe recovers on retry, and the retry gets only wh assert.equal(identity.xcode, 'Xcode 26.2\nBuild version 17C52'); assert.equal(identity.macosBuild, '24G90'); assert.equal(identity.architecture, 'arm64'); - assert.equal(identity.simulatorSdk, '26.2'); + assert.equal(identity.simulatorRuntime, 'iOS 26.2'); // The stalled first attempt is capped at the 30 s per-probe ceiling; the // retry runs on the 10 s the shared deadline has left, not a second 30 s. assert.deepEqual(timeouts.slice(0, 2), [30_000, 10_000]); assert.equal(clock.nowMs, 30_000); - // 5 baseline probes (xcodebuild, sw_vers x2, uname, xcrun) plus the one + // 4 baseline probes (xcodebuild, sw_vers x2, uname) plus the one // retry that recovered the first, timed-out call. - assert.equal(calls, 6); + assert.equal(calls, 5); }); -test('a toolchain host that never returns still fails at the deadline with the same timeout error', async () => { +// The identity read is allowed to exec one Xcode-owned binary. The Simulator SDK a second +// `xcrun` probe used to report ships inside the selected `Xcode.app`, so it cannot move under a +// `xcodebuild -version` build that already pins it -- and every extra Xcode-owned exec is another +// toolchain the job can wait on and fail against (#2712). +test('the toolchain identity execs one Xcode-owned binary, and no xcrun', async () => { + const clock = { nowMs: 0 }; + const probed: string[] = []; + const host = fakeToolchainHost((command, args) => { + probed.push(command); + return toolchainAnswer(command, args); + }); + + await readSnapshotSourceToolchain(host, 'iOS 26.2', fakeClockDeadline(120_000, clock)); + + assert.deepEqual(probed, ['xcodebuild', 'sw_vers', 'sw_vers', 'uname']); +}); + +test('a toolchain host that never returns reports the stalled probe after one retry', async () => { const clock = { nowMs: 0 }; const timeouts: number[] = []; - const host = fakeToolchainHost((command, _args, options) => { + const host = fakeToolchainHost(async (_command, _args, options) => { timeouts.push(options.timeoutMs ?? 0); - throw blockForWholeTimeout(clock, command, options); + throw await blockForWholeTimeout(clock, options); }); await assert.rejects( readSnapshotSourceToolchain(host, 'iOS 26.2', fakeClockDeadline(120_000, clock)), - (error: unknown) => - error instanceof AppError && error.message === 'xcodebuild timed out after 30000ms', + (error: unknown) => { + assertToolchainProbeStall(error, 'xcodebuild', [30_000, 30_000]); + return true; + }, ); // Exactly one retry, not an unbounded loop, and the retry is charged the // remainder rather than a fresh ceiling. @@ -65,6 +88,29 @@ test('a toolchain host that never returns still fails at the deadline with the s assert.equal(clock.nowMs, 60_000); }); +// The stall names the probe that hit it, not just the first one in the sequence: a host whose +// `sw_vers` answers late must not read as an Xcode problem. +test('a later probe that stalls out names that probe and its own attempts', async () => { + const clock = { nowMs: 0 }; + let macosBuildCalls = 0; + const host = fakeToolchainHost(async (command, args, options) => { + if (command !== 'sw_vers' || !args.includes('-buildVersion')) { + return toolchainAnswer(command, args); + } + macosBuildCalls += 1; + throw await blockForWholeTimeout(clock, options); + }); + + await assert.rejects( + readSnapshotSourceToolchain(host, 'iOS 26.2', fakeClockDeadline(120_000, clock)), + (error: unknown) => { + assertToolchainProbeStall(error, 'sw_vers', [30_000, 30_000]); + return true; + }, + ); + assert.equal(macosBuildCalls, 2); +}); + test('a probe that failed on its own and merely says "timed out" in its message is not retried', async () => { const clock = { nowMs: 0 }; let calls = 0; @@ -101,7 +147,7 @@ type ToolchainProbeCancellationCase = { aborts: 'never' | 'before-the-deadline' | 'while-it-blocks' | 'as-it-unwinds'; /** The phase deadline. 30 s is spent in full by one stalled probe, leaving no retry. */ deadlineMs: number; - expected: 'cancelled' | 'exec-timeout' | 'command-failure'; + expected: 'cancelled' | 'probe-stall' | 'command-failure'; execs: number; clockMs: number; }; @@ -146,12 +192,12 @@ const CANCELLATION_CASES: ToolchainProbeCancellationCase[] = [ clockMs: 30_000, }, { - // Nothing left to retry on, so the original timeout propagates unchanged. + // Nothing left to retry on, so a single stalled attempt already names the probe. label: 'never aborted, the first probe spends the whole deadline', firstProbe: 'exec-timeout', aborts: 'never', deadlineMs: 30_000, - expected: 'exec-timeout', + expected: 'probe-stall', execs: 1, clockMs: 30_000, }, @@ -162,13 +208,13 @@ test.each(CANCELLATION_CASES)('cancellation matrix: $label', async (testCase) => const request = new AbortController(); if (testCase.aborts === 'before-the-deadline') request.abort(); let execs = 0; - const host = fakeToolchainHost((command, args, options) => { + const host = fakeToolchainHost(async (command, args, options) => { execs += 1; if (execs > 1 || !testCase.firstProbe) return toolchainAnswer(command, args); if (testCase.aborts === 'while-it-blocks') request.abort(); const failure = testCase.firstProbe === 'exec-timeout' - ? blockForWholeTimeout(clock, command, options) + ? await blockForWholeTimeout(clock, options) : // No `timeoutMs` detail: the tool failed on its own, so nothing retries it. new AppError('COMMAND_FAILED', `${command}: unexpected error`, { cmd: command }); if (testCase.aborts === 'as-it-unwinds') request.abort(); @@ -204,13 +250,33 @@ function assertExpectedToolchainFailure( assert.equal(error.details?.reason, 'request_canceled', testCase.label); return; } + if (testCase.expected === 'probe-stall') { + assertToolchainProbeStall(error, 'xcodebuild', [30_000]); + return; + } assert.ok(error instanceof AppError, `${testCase.label}: expected the probe's own failure`); - assert.equal( - error.message, - testCase.expected === 'exec-timeout' - ? 'xcodebuild timed out after 30000ms' - : 'xcodebuild: unexpected error', - testCase.label, + assert.equal(error.message, 'xcodebuild: unexpected error', testCase.label); +} + +/** + * A probe that stalled out names itself, says what each attempt was armed with, and keeps the exec + * layer's own kill as its cause -- so a job can tell "this tool never answered" from a device + * failure without reading a stack frame (#2712). + */ +function assertToolchainProbeStall( + error: unknown, + command: string, + attemptTimeoutsMs: number[], +): void { + assert.ok(error instanceof SnapshotSourceError, `expected a toolchain probe stall, got ${error}`); + assert.equal(error.failureKind, 'timeout'); + assert.equal(error.failureCode, 'toolchain-probe-stalled'); + assert.equal(error.details?.command, command); + assert.deepEqual(error.details?.attemptTimeoutsMs, attemptTimeoutsMs); + assert.match(String(error.details?.hint), new RegExp(`run \`${command}\` by hand`)); + assert.ok( + isCommandTimeoutError(error.cause), + 'the exec layer kill the probe hit stays the cause', ); } @@ -219,33 +285,35 @@ function fakeClockDeadline(timeoutMs: number, clock: { nowMs: number }): Snapsho return createSnapshotSourceDeadline(timeoutMs, undefined, () => clock.nowMs); } -/** A probe that blocked for its whole timeout and was then killed, as the exec layer reports it. */ -function blockForWholeTimeout( +/** + * A probe that blocked for its whole timeout and was then killed. The fake clock advances by the + * budget the probe was handed, and the failure is the one `exec.ts` really raises for that kill. + */ +async function blockForWholeTimeout( clock: { nowMs: number }, - command: string, options: ExecOptions, -): AppError { - const timeoutMs = options.timeoutMs ?? 0; - clock.nowMs += timeoutMs; - return new AppError('COMMAND_FAILED', `${command} timed out after ${timeoutMs}ms`, { timeoutMs }); +): Promise { + clock.nowMs += options.timeoutMs ?? 0; + return await execKillTimeoutError(); } +/** + * What the host answers each probe the identity read is allowed to run. A command outside this list + * is a probe the identity read must not open at all (#2712). + */ function toolchainAnswer(command: string, args: string[]): ExecResult { - const stdout = - command === 'xcodebuild' - ? 'Xcode 26.2\nBuild version 17C52' - : command === 'sw_vers' - ? args.includes('-buildVersion') - ? '24G90' - : '15.6' - : command === 'uname' - ? 'arm64' - : '26.2'; - return { stdout, stderr: '', exitCode: 0 }; + if (command === 'xcodebuild') { + return { stdout: 'Xcode 26.2\nBuild version 17C52', stderr: '', exitCode: 0 }; + } + if (command === 'sw_vers') { + return { stdout: args.includes('-buildVersion') ? '24G90' : '15.6', stderr: '', exitCode: 0 }; + } + if (command === 'uname') return { stdout: 'arm64', stderr: '', exitCode: 0 }; + throw new Error(`the identity read execed ${command} ${args.join(' ')}`); } function fakeToolchainHost( - run: (command: string, args: string[], options: ExecOptions) => ExecResult, + run: (command: string, args: string[], options: ExecOptions) => ExecResult | Promise, ): SnapshotSourceHost { const real = createSnapshotSourceHost(); return { diff --git a/packages/platform-apple/src/snapshot-source/cache-identity.ts b/packages/platform-apple/src/snapshot-source/cache-identity.ts index 785b8708ba..5bf7e42a1e 100644 --- a/packages/platform-apple/src/snapshot-source/cache-identity.ts +++ b/packages/platform-apple/src/snapshot-source/cache-identity.ts @@ -2,16 +2,22 @@ import { createHash } from 'node:crypto'; import path from 'node:path'; import { isCommandTimeoutError, type ExecResult } from '@agent-device/host-kit/command'; import { COLD_TOOLCHAIN_PROBE_TIMEOUT_MS } from '../runner/apple-runner-platform.ts'; -import { snapshotSourceError } from './errors.ts'; +import { snapshotSourceError, type SnapshotSourceError } from './errors.ts'; import { remainingSnapshotSourceMs, type SnapshotSourceDeadline } from './deadline.ts'; import type { SnapshotSourceHost } from './types.ts'; +/** + * The half of the bridge cache key the host answers for. `xcode` carries the version and the build, + * which is what pins the Simulator SDK the bridge compiles against: that SDK ships inside the + * selected `Xcode.app`, so it cannot move while `xcodebuild -version` reports the same build. The + * identity therefore execs one Xcode-owned binary rather than two, because a toolchain probe that + * cannot answer fails the whole job with nothing but a cache key at stake (#2712). + */ export type SnapshotSourceToolchainIdentity = Readonly<{ xcode: string; macosProductVersion: string; macosBuild: string; architecture: 'arm64' | 'x86_64'; - simulatorSdk: string; simulatorRuntime: string; }>; @@ -53,16 +59,11 @@ export async function readSnapshotSourceToolchain( simulatorRuntime: string, deadline: SnapshotSourceDeadline, ): Promise { + // The one Xcode-owned binary this read execs: SnapshotSourceToolchainIdentity says why (#2712). const xcode = await toolOutput(host, 'xcodebuild', ['-version'], deadline); const macosProductVersion = await toolOutput(host, 'sw_vers', ['-productVersion'], deadline); const macosBuild = await toolOutput(host, 'sw_vers', ['-buildVersion'], deadline); const architecture = await toolOutput(host, 'uname', ['-m'], deadline); - const simulatorSdk = await toolOutput( - host, - 'xcrun', - ['--sdk', 'iphonesimulator', '--show-sdk-version'], - deadline, - ); const runtime = simulatorRuntime.trim(); if (!runtime) throw snapshotSourceError('unsupported', 'simulator-runtime-missing'); if (architecture !== 'arm64' && architecture !== 'x86_64') { @@ -75,7 +76,6 @@ export async function readSnapshotSourceToolchain( macosProductVersion, macosBuild, architecture, - simulatorSdk, simulatorRuntime: runtime, }; } @@ -99,10 +99,16 @@ async function toolOutput( return output; } +/** One exec, plus the single retry a first-exec stall can actually earn. */ +const TOOLCHAIN_PROBE_ATTEMPTS = 2; + /** - * Retries exactly once, and only the exec layer's structured timeout: the stall - * {@link COLD_TOOLCHAIN_PROBE_TIMEOUT_MS} names clears on the next exec of the same tool. - * Both attempts read one deadline, so the retry gets what the stall left. + * Retries exactly once, and only the exec layer's structured timeout: a stall that finished while the + * probe was being killed leaves an instant next exec of the same tool behind it, which is what + * {@link COLD_TOOLCHAIN_PROBE_TIMEOUT_MS} was sized for (#2422). A stall that had not finished does + * not clear that way, so the second timeout is reported as the host condition it is instead of + * pretending the probe was worth running twice. Both attempts read one deadline, so the retry only + * gets what the first stall left. */ async function runToolchainProbe( host: SnapshotSourceHost, @@ -110,28 +116,68 @@ async function runToolchainProbe( args: string[], deadline: SnapshotSourceDeadline, ): Promise { - try { - return await execToolchainProbe(host, command, args, deadline); - } catch (error) { - if (!isCommandTimeoutError(error)) throw error; - if (deadline.signal?.aborted) throw snapshotSourceError('cancelled', 'abort-signal'); - if (deadline.clock.remainingMs(deadline.now()) <= 0) throw error; - return await execToolchainProbe(host, command, args, deadline); + const attemptTimeoutsMs: number[] = []; + let stalledBy: SnapshotSourceError | undefined; + for (let attempt = 1; ; attempt += 1) { + let timeoutMs: number; + try { + timeoutMs = Math.min( + COLD_TOOLCHAIN_PROBE_TIMEOUT_MS, + remainingSnapshotSourceMs(deadline, 'toolchain-probe-deadline'), + ); + } catch (budgetError) { + // A budget that closes between an attempt and this line is the stall already observed, and it + // still names the probe that stalled rather than the arithmetic that noticed. + throw stalledBy ?? budgetError; + } + attemptTimeoutsMs.push(timeoutMs); + try { + return await execToolchainProbe(host, command, args, deadline, timeoutMs); + } catch (error) { + if (!isCommandTimeoutError(error)) throw error; + if (deadline.signal?.aborted) throw snapshotSourceError('cancelled', 'abort-signal'); + stalledBy = toolchainProbeStallError(command, attemptTimeoutsMs, error); + if (attempt >= TOOLCHAIN_PROBE_ATTEMPTS) throw stalledBy; + } } } +/** + * Says which probe could not answer, how many execs it took to learn that, and what each was armed + * with. The exec layer's ` timed out after Nms` alone reaches a job as an unattributed command + * failure, which reads exactly like a device failure (#2712). + */ +function toolchainProbeStallError( + command: string, + attemptTimeoutsMs: readonly number[], + cause: unknown, +): SnapshotSourceError { + const attempts = attemptTimeoutsMs.length; + return snapshotSourceError( + 'timeout', + 'toolchain-probe-stalled', + { + command, + attemptTimeoutsMs: [...attemptTimeoutsMs], + hint: + `${command} did not answer within ${attemptTimeoutsMs[attempts - 1]}ms on ${attempts} ` + + `attempt(s). A toolchain probe that cannot answer is a host condition rather than a ` + + `toolchain defect: run \`${command}\` by hand until it answers, then retry.`, + }, + cause, + ); +} + function execToolchainProbe( host: SnapshotSourceHost, command: string, args: string[], deadline: SnapshotSourceDeadline, + timeoutMs: number, ): Promise { return host.run(command, args, { allowFailure: true, signal: deadline.signal, - timeoutMs: Math.min( - COLD_TOOLCHAIN_PROBE_TIMEOUT_MS, - remainingSnapshotSourceMs(deadline, 'toolchain-probe-deadline'), - ), + timeoutMs, }); } diff --git a/packages/platform-apple/src/snapshot-source/cache.test.ts b/packages/platform-apple/src/snapshot-source/cache.test.ts index d381c0ef02..bfbb6da496 100644 --- a/packages/platform-apple/src/snapshot-source/cache.test.ts +++ b/packages/platform-apple/src/snapshot-source/cache.test.ts @@ -3,12 +3,14 @@ import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises'; import path from 'node:path'; import { test } from 'vitest'; +import { isCommandTimeoutError } from '@agent-device/host-kit/command'; import { createSnapshotSourceHost } from './host.ts'; import { ensureSnapshotBridgeBinary } from './cache.ts'; import { SnapshotSourceError } from './errors.ts'; import { createSnapshotSourceDeadline } from './deadline.ts'; import { DEFAULT_SNAPSHOT_SOURCE_LIMITS } from './limits.ts'; import type { SnapshotSourceHost } from './types.ts'; +import { execKillTimeoutError } from './__tests__/exec-timeout-fixture.ts'; import { mkdtempForTest } from '../__tests__/tmp-dir.ts'; test('snapshot bridge preparation is cold-once, atomic, and invalidates corrupt or stale entries', async () => { @@ -265,6 +267,54 @@ test('a bridge build that failed is reported over a cache lock that could not be } }); +test('a compile exec killed at its budget reports the bridge build, not the exec layer', async () => { + const root = await mkdtempForTest('agent-device-snapshot-source-build-stall-'); + const sourceRoot = path.join(root, 'source'); + const cacheRoot = path.join(root, 'cache'); + await (await import('@agent-device/host-kit/host-file')).ensureHostDirectory(sourceRoot); + await writeFile(path.join(sourceRoot, 'SnapshotBridge.m'), 'native source'); + await writeFile(path.join(sourceRoot, 'SnapshotBridgeRuntime.m'), 'native runtime'); + await writeFile(path.join(sourceRoot, 'SnapshotBridgeRuntime.h'), 'native header'); + await writeFile(path.join(sourceRoot, 'SnapshotBridgeCapture.h'), 'native header'); + await writeFile(path.join(sourceRoot, 'SnapshotBridgeCapture.m'), 'native header'); + const buildHost = createFakeBuildHost('binary'); + let compileTimeoutMs = 0; + const host: SnapshotSourceHost = { + ...buildHost, + run: async (command, args, options) => { + if (command !== 'xcrun' || !args.includes('clang')) { + return await buildHost.run(command, args, options); + } + compileTimeoutMs = options?.timeoutMs ?? 0; + throw await execKillTimeoutError(); + }, + }; + + try { + await assert.rejects( + ensureSnapshotBridgeBinary({ + host, + runtime: 'iOS 26.2', + limits: DEFAULT_SNAPSHOT_SOURCE_LIMITS, + deadline: testDeadline(120_000), + sourceRoot, + cacheRoot, + }), + (error: unknown) => { + assert.ok(error instanceof SnapshotSourceError); + assert.equal(error.failureKind, 'timeout'); + assert.equal(error.failureCode, 'native-build-stalled'); + assert.ok(compileTimeoutMs > 0, 'the compile exec ran and was killed at its budget'); + assert.equal(error.details?.timeoutMs, compileTimeoutMs, 'says the budget the compile hit'); + assert.ok(isCommandTimeoutError(error.cause), 'keeps the exec layer kill as the cause'); + return true; + }, + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + async function expectRejectedCancellation(value: Promise): Promise { await assert.rejects(value, (error: unknown) => { return ( diff --git a/packages/platform-apple/src/snapshot-source/cache.ts b/packages/platform-apple/src/snapshot-source/cache.ts index 73efeed2c4..4d5f4bf088 100644 --- a/packages/platform-apple/src/snapshot-source/cache.ts +++ b/packages/platform-apple/src/snapshot-source/cache.ts @@ -1,5 +1,6 @@ import { createHash } from 'node:crypto'; import path from 'node:path'; +import { isCommandTimeoutError, type ExecResult } from '@agent-device/host-kit/command'; import { withProcessLock } from '@agent-device/host-kit/file'; import { SnapshotSourceError, snapshotSourceError } from './errors.ts'; import { remainingSnapshotSourceMs, type SnapshotSourceDeadline } from './deadline.ts'; @@ -88,37 +89,12 @@ export async function ensureSnapshotBridgeBinary( remainingSnapshotSourceMs(deadline, 'native-build-deadline'); await input.host.ensureDirectory(temporaryPath); const outputPath = path.join(temporaryPath, BRIDGE_FILENAME); - const result = await input.host.run( - 'xcrun', - [ - '--sdk', - 'iphonesimulator', - 'clang', - '-arch', - toolchain.architecture, - '-mios-simulator-version-min=15.0', - '-fobjc-arc', - '-Werror', - '-Wall', - '-Wextra', - '-framework', - 'Foundation', - '-framework', - 'CoreGraphics', - ...SNAPSHOT_BRIDGE_COMPILE_FILENAMES.map((sourceFile) => - path.join(sourceRoot, sourceFile), - ), - '-o', - outputPath, - ], - { - signal: deadline.signal, - timeoutMs: Math.min( - BUILD_TIMEOUT_MS, - remainingSnapshotSourceMs(deadline, 'native-build-deadline'), - ), - allowFailure: true, - }, + const result = await compileSnapshotBridge( + input.host, + deadline, + toolchain.architecture, + sourceRoot, + outputPath, ); if (result.exitCode !== 0 || !input.host.exists(outputPath)) { throw snapshotSourceError('unsupported', 'native-build-failed', { @@ -159,6 +135,64 @@ export async function ensureSnapshotBridgeBinary( }); } +/** + * One clang invocation for the bridge sources. A compile exec this module asked to be killed is + * reported with the budget it hit: after the identity read stopped opening `xcrun` of its own + * (#2712), this is the process's first `xcrun` exec, and the exec layer's bare + * `xcrun timed out after Nms` would land on a job as an unattributed command failure again. + */ +async function compileSnapshotBridge( + host: SnapshotSourceHost, + deadline: SnapshotSourceDeadline, + architecture: SnapshotSourceToolchainIdentity['architecture'], + sourceRoot: string, + outputPath: string, +): Promise { + const timeoutMs = Math.min( + BUILD_TIMEOUT_MS, + remainingSnapshotSourceMs(deadline, 'native-build-deadline'), + ); + try { + return await host.run( + 'xcrun', + [ + '--sdk', + 'iphonesimulator', + 'clang', + '-arch', + architecture, + '-mios-simulator-version-min=15.0', + '-fobjc-arc', + '-Werror', + '-Wall', + '-Wextra', + '-framework', + 'Foundation', + '-framework', + 'CoreGraphics', + ...SNAPSHOT_BRIDGE_COMPILE_FILENAMES.map((sourceFile) => path.join(sourceRoot, sourceFile)), + '-o', + outputPath, + ], + { signal: deadline.signal, timeoutMs, allowFailure: true }, + ); + } catch (error) { + if (!isCommandTimeoutError(error)) throw error; + throw snapshotSourceError( + 'timeout', + 'native-build-stalled', + { + timeoutMs, + hint: + `The Simulator SDK toolchain did not answer within ${timeoutMs}ms, which stopped the bridge ` + + `build before clang reported anything. Run \`xcrun --sdk iphonesimulator clang --version\` ` + + `by hand until it answers, then retry.`, + }, + error, + ); + } +} + function resolveSnapshotBridgeSourceRoot(host: SnapshotSourceHost): string { const projectRoot = host.projectRoot(); const checkoutRoot = path.join(projectRoot, 'apple', 'snapshot-bridge');