diff --git a/packages/webui/src/runtime/browser/browser-service.ts b/packages/webui/src/runtime/browser/browser-service.ts index ce2da17a..fa006415 100644 --- a/packages/webui/src/runtime/browser/browser-service.ts +++ b/packages/webui/src/runtime/browser/browser-service.ts @@ -312,6 +312,8 @@ export class WebuiBrowserService { WebuiBrowserSurface, Map void> >(); + /** An old screencast must stop before a replacement starts on the same provider and target. */ + readonly #frameStops = new WeakMap>>(); /** How many frames each surface dropped, per surface. */ readonly #droppedFrames = new Map(); /** @@ -916,6 +918,7 @@ export class WebuiBrowserService { this.#providers.delete(surface); this.#selected.delete(surface); this.#frameSubscribers.delete(surface); + this.#frameHandlers.delete(surface); this.#webTabs.delete(surface); this.#webTabSubscribers.delete(surface); for (const target of this.#registry.list(surface)) { @@ -1135,6 +1138,21 @@ export class WebuiBrowserService { true, ); } + const targetKey = target.targetId; + const stops = this.#frameStops.get(provider) ?? new Map>(); + this.#frameStops.set(provider, stops); + for (let pending = stops.get(targetKey); pending; pending = stops.get(targetKey)) { + await waitForFrameStop(pending, signal); + if (signal?.aborted) return; + } + if (signal?.aborted) return; + if (this.#closed || this.#providers.get(surface) !== provider) { + throw browserOperationError( + "browser_provider_unavailable", + `Browser surface "${surface}" has no connected browser.`, + true, + ); + } // The target has to be a page this service knows. A stream for a target // that does not exist would deliver nothing at all, and a panel waiting // for a picture that cannot arrive looks like a panel that is broken. @@ -1187,7 +1205,6 @@ export class WebuiBrowserService { // subscribers: the provider holds one frame handler per page, so two // subscriptions for the same page would otherwise take each other's // frames away. - const targetKey = target.targetId; const surfaceSubscribers = this.#frameSubscribers.get(surface) ?? new Map void>>(); @@ -1206,8 +1223,8 @@ export class WebuiBrowserService { surfaceHandlers.get(targetKey) ?? ((frame: WebuiBrowserImageFrame) => { const current = this.#frameSubscribers.get(surface)?.get(targetKey); - if (!current) return; - for (const subscriber of [...current]) subscriber(frame); + if (current !== subscribers || this.#providers.get(surface) !== provider) return; + for (const subscriber of [...subscribers]) subscriber(frame); }); surfaceHandlers.set(targetKey, handler); this.#frameHandlers.set(surface, surfaceHandlers); @@ -1280,12 +1297,24 @@ export class WebuiBrowserService { } finally { if (firstFrameDeadline) clearTimeout(firstFrameDeadline); subscribers.delete(deliver); - if (subscribers.size === 0) { + if ( + subscribers.size === 0 && + this.#providers.get(surface) === provider && + this.#frameSubscribers.get(surface)?.get(targetKey) === subscribers && + this.#frameHandlers.get(surface)?.get(targetKey) === handler + ) { // Cancelling the stream releases the provider-side resources it held: - // the screencast is stopped, not merely ignored. - await provider.stopImageStream(targetKey).catch(() => undefined); + // detach the old maps before yielding, then serialize a replacement's + // start after the old provider stop. A late stop must not erase a new + // subscriber or turn off the stream it just started. surfaceSubscribers.delete(targetKey); surfaceHandlers.delete(targetKey); + const stopping = Promise.resolve() + .then(() => provider.stopImageStream(targetKey)) + .catch(() => undefined) + .finally(() => { stops.delete(targetKey); }); + stops.set(targetKey, stopping); + await stopping; } } } @@ -1934,6 +1963,22 @@ function sameTab(left: WebuiBrowserTargetId, right: WebuiBrowserTargetId): boole return left.surface === right.surface && left.browserId === right.browserId && left.targetId === right.targetId; } +/** Waits for the old stream to stop without holding a canceled replacement open. */ +async function waitForFrameStop(stop: Promise, signal?: AbortSignal): Promise { + if (!signal) return stop; + if (signal.aborted) return; + let onAbort: () => void = () => undefined; + const aborted = new Promise((resolve) => { + onAbort = () => resolve(); + signal.addEventListener("abort", onAbort, { once: true }); + }); + try { + await Promise.race([stop, aborted]); + } finally { + signal.removeEventListener("abort", onAbort); + } +} + /** * Awaits one command, or refuses it when the caller's signal aborts. * diff --git a/packages/webui/test/integration/agent-transcript.test.ts b/packages/webui/test/integration/agent-transcript.test.ts index f102bd56..71c6d308 100644 --- a/packages/webui/test/integration/agent-transcript.test.ts +++ b/packages/webui/test/integration/agent-transcript.test.ts @@ -10,6 +10,12 @@ import path from "node:path"; import { chromium, expect as browserExpect, type Browser, type Locator, type Page } from "@playwright/test"; import { afterAll, expect, it } from "vitest"; import type { WebuiBrowserAdapterBinding } from "../../src/runtime/harness/browser.js"; +import { WEBUI_BROWSER_ERROR_CODES } from "../../src/shared/contracts/browser.js"; +import { + OPEN_BROWSER_WEB_TAB_OPERATION_NAME, + WATCH_BROWSER_FRAMES_OPERATION_NAME, + WATCH_BROWSER_WEB_TABS_OPERATION_NAME, +} from "../../src/shared/operation-names.js"; const MODEL_ID = "stub-model"; const PROVIDER_NAME = "stub"; @@ -288,6 +294,61 @@ async function waitForBrowserFrameOrFailure( } } +type WireObservation = { at: number; direction: string; frame: unknown }; + +/** Correlate startup requests with wire outcomes without serializing any payload or identifier. */ +function browserStartupWireEvidence(wire: readonly WireObservation[], from: number, startedAt: number) { + const operations: ReadonlySet = new Set([ + OPEN_BROWSER_WEB_TAB_OPERATION_NAME, + WATCH_BROWSER_FRAMES_OPERATION_NAME, + WATCH_BROWSER_WEB_TABS_OPERATION_NAME, + ]); + const knownCodes: ReadonlySet = new Set(WEBUI_BROWSER_ERROR_CODES); + const probes = new Map(); + for (const [index, { at, direction, frame }] of wire.entries()) { + if (!frame || typeof frame !== "object") continue; + const { requestId, kind, operation, code, body } = frame as Record; + if (typeof requestId !== "string") continue; + const atMs = at - startedAt; + if (direction === "sent" && kind === "request") { + if (typeof operation === "string" && operations.has(operation) && + (index >= from || operation === WATCH_BROWSER_WEB_TABS_OPERATION_NAME)) { + probes.set(requestId, { operation, sentAtMs: atMs, events: 0, imageFrames: 0 }); + } + continue; + } + if (direction !== "received" || index < from) continue; + const probe = probes.get(requestId); + if (!probe) continue; + if (kind === "response") probe.responseAtMs ??= atMs; + if (kind === "error") { + probe.error ??= { atMs, code: typeof code === "string" && knownCodes.has(code) ? code : "other" }; + } + if (kind === "event") { + probe.events++; + probe.firstEventAtMs ??= atMs; + if (probe.operation === WATCH_BROWSER_FRAMES_OPERATION_NAME && body && typeof body === "object") { + const { format, data, sequence } = body as Record; + if ((format === "jpeg" || format === "png") && + typeof data === "string" && data.length > 0 && Number.isSafeInteger(sequence)) { + probe.imageFrames++; + probe.firstImageAtMs ??= atMs; + } + } + } + } + return { requestCount: probes.size, requests: [...probes.values()].slice(-12) }; +} + it.each(["transcript", "cross-client", "external-active-mutation", "external-active-mutation-gap"] as const)("real agent rendering and recovery: %s", async (scenario) => { let releaseAnswer: () => void = () => undefined; const answerGate = new Promise((resolve) => { releaseAnswer = resolve; }); @@ -503,6 +564,7 @@ it.each(["transcript", "cross-client", "external-active-mutation", "external-act const dataDir = await mkdtemp(path.join(tmpdir(), "webui-agent-transcript-e2e-")); const previousDataDir = process.env.MINIMAX_DATA_DIR; + const previousBrowserOutputDir = process.env.WEBUI_BROWSER_OUTPUT_DIR; const workspaceDir = path.join(dataDir, "workspace"); await mkdir(workspaceDir, { recursive: true }); await writeFile(path.join(workspaceDir, "reference-target.md"), "Synthetic workspace reference body.\n"); @@ -551,6 +613,10 @@ it.each(["transcript", "cross-client", "external-active-mutation", "external-act let runtimeEventAbort: AbortController | undefined; try { process.env.MINIMAX_DATA_DIR = dataDir; + if (previousBrowserOutputDir?.trim()) { + // Other tests in this job otherwise overwrite the same diagnostic filenames. + process.env.WEBUI_BROWSER_OUTPUT_DIR = path.join(previousBrowserOutputDir, "agent-transcript", scenario); + } (await import("@mavis/config")).resetConfig(); const { createWebuiRuntimeHost } = await import("../../src/runtime/assembly.js"); const { createHarnessPortFromHost } = await import("../../src/runtime/harness/adapter.js"); @@ -648,7 +714,7 @@ it.each(["transcript", "cross-client", "external-active-mutation", "external-act body?: Record; }[] = []; const received: { requestId?: string; kind?: string; body?: unknown }[] = []; - const wireTimeline: { at: number; direction: string; frame: unknown }[] = []; + const wireTimeline: WireObservation[] = []; page.on("websocket", (socket) => { socket.on("framesent", ({ payload }) => { try { @@ -1196,6 +1262,8 @@ it.each(["transcript", "cross-client", "external-active-mutation", "external-act await browserExpect(browserLauncher).toBeVisible(); const pageUrl = `http://127.0.0.1:${pageAddress.port}/`; await page.getByTestId("workspace-launcher-url").fill(pageUrl); + const browserStartupIndex = wireTimeline.length; + const browserStartupAt = Date.now(); await page.getByTestId("workspace-launcher-url-submit").click(); const browserFrame = page.getByTestId("browser-panel-frame"); // The panel can only render a frame after the isolated Chromium has @@ -1224,7 +1292,10 @@ it.each(["transcript", "cross-client", "external-active-mutation", "external-act `notice: ${notice?.text ?? "(none)"}`, `toast: ${toast?.text ?? "(none)"}`, `error code: ${toast?.errorCode ?? "(none)"}`, - JSON.stringify({ panelState: browserPanel.nodes }), + JSON.stringify({ + panelState: browserPanel.nodes, + wire: browserStartupWireEvidence(wireTimeline, browserStartupIndex, browserStartupAt), + }), ].join("\n")); } if (browserPanel.outcome === "timeout") { @@ -1236,6 +1307,7 @@ it.each(["transcript", "cross-client", "external-active-mutation", "external-act .filter((node) => node.id === "browser-panel-unknown" || node.id === "browser-panel-unauthorized") .map((node) => node.text), panelState: browserPanel.nodes, + wire: browserStartupWireEvidence(wireTimeline, browserStartupIndex, browserStartupAt), }), ].join("\n")); } @@ -2065,22 +2137,27 @@ it.each(["transcript", "cross-client", "external-active-mutation", "external-act } await observerContext.close(); } finally { - runtimeEventAbort?.abort(); - releaseAnswer(); - releaseTodoAnswer(); - releaseStoppedAnswer(); - releaseCrossClientHold(); - releaseImmediateAnswer(); - releaseGoalFollowUp(); - await browser?.close(); - await service?.close().catch(() => undefined); - await browserAdapter?.close().catch(() => undefined); - await assembled?.host.apiHost.close().catch(() => undefined); - await new Promise((resolve) => modelServer.close(resolve)); - if (pageServer) await new Promise((resolve) => pageServer!.close(() => resolve())); - await rm(dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); - if (previousDataDir === undefined) delete process.env.MINIMAX_DATA_DIR; - else process.env.MINIMAX_DATA_DIR = previousDataDir; - (await import("@mavis/config")).resetConfig(); + try { + runtimeEventAbort?.abort(); + releaseAnswer(); + releaseTodoAnswer(); + releaseStoppedAnswer(); + releaseCrossClientHold(); + releaseImmediateAnswer(); + releaseGoalFollowUp(); + await browser?.close(); + await service?.close().catch(() => undefined); + await browserAdapter?.close().catch(() => undefined); + await assembled?.host.apiHost.close().catch(() => undefined); + await new Promise((resolve) => modelServer.close(resolve)); + if (pageServer) await new Promise((resolve) => pageServer!.close(() => resolve())); + await rm(dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + if (previousDataDir === undefined) delete process.env.MINIMAX_DATA_DIR; + else process.env.MINIMAX_DATA_DIR = previousDataDir; + (await import("@mavis/config")).resetConfig(); + } finally { + if (previousBrowserOutputDir === undefined) delete process.env.WEBUI_BROWSER_OUTPUT_DIR; + else process.env.WEBUI_BROWSER_OUTPUT_DIR = previousBrowserOutputDir; + } } }, 120_000); diff --git a/packages/webui/test/unit/browser-service.test.ts b/packages/webui/test/unit/browser-service.test.ts index 50a749e9..91273f17 100644 --- a/packages/webui/test/unit/browser-service.test.ts +++ b/packages/webui/test/unit/browser-service.test.ts @@ -1431,6 +1431,199 @@ describe("the surface a run is working on", () => { }); +describe("browser frame subscription handoff", () => { + it("waits for a pending start before stopping its cancelled last watcher", async () => { + const { service, provider } = createService({ ...ALL_OFF, isolatedChromium: true }); + await service.listTargets("isolated-chromium"); + const target = tab("tab-1"); + const controller = new AbortController(); + const watch = service.watchFrames("isolated-chromium", target, controller.signal)[Symbol.asyncIterator](); + const startEntered = Promise.withResolvers(); + const releaseStart = Promise.withResolvers(); + const startImageStream = provider.startImageStream.bind(provider); + vi.spyOn(provider, "startImageStream").mockImplementationOnce(async (targetId, handler) => { + startEntered.resolve(); + await releaseStart.promise; + await startImageStream(targetId, handler); + }); + try { + const firstFrame = watch.next(); + await startEntered.promise; + controller.abort(); + expect(provider.stoppedStreams).toEqual([]); + releaseStart.resolve(); + await expect(firstFrame).resolves.toMatchObject({ done: true }); + expect(provider.startedStreams).toEqual(["tab-1"]); + expect(provider.stoppedStreams).toEqual(["tab-1"]); + } finally { + releaseStart.resolve(); + controller.abort(); + await watch.return?.(); + await service.close(); + } + }); + + it.each(["resolves", "rejects"] as const)("delivers frames to a replacement watcher after an in-progress stop that %s", async (outcome) => { + const { service, provider } = createService({ ...ALL_OFF, isolatedChromium: true }); + await service.listTargets("isolated-chromium"); + const target = tab("tab-1"); + const previous = service.watchFrames("isolated-chromium", target)[Symbol.asyncIterator](); + const replacementAbort = new AbortController(); + const replacement = service.watchFrames("isolated-chromium", target, replacementAbort.signal)[Symbol.asyncIterator](); + const stopEntered = Promise.withResolvers(); + const releaseStop = Promise.withResolvers(); + const startImageStream = vi.spyOn(provider, "startImageStream"); + const stopImageStream = provider.stopImageStream.bind(provider); + vi.spyOn(provider, "stopImageStream").mockImplementationOnce(async (targetId) => { + stopEntered.resolve(); + await releaseStop.promise; + await stopImageStream(targetId); + if (outcome === "rejects") throw new Error("Synthetic stop failure"); + }); + try { + const first = previous.next(); + await waitUntil(() => provider.startedStreams.length === 1); + provider.pushFrame(1); + const firstResult = await first; + expect(firstResult.done).toBe(false); + const previousClosed = previous.return!(); + await stopEntered.promise; + const replacementFrame = replacement.next(); + // Let the replacement reach the service while the first stop is gated. + await new Promise((resolve) => setImmediate(resolve)); + expect(provider.startedStreams).toEqual(["tab-1"]); + releaseStop.resolve(); + await previousClosed; + await waitUntil(() => provider.startedStreams.length === 2); + let receivedFrame = false; + void replacementFrame.then((result) => { receivedFrame = !result.done; }, () => undefined); + // A provider callback retained by the old stream cannot feed the new watcher. + startImageStream.mock.calls[0]![1](firstResult.value); + await new Promise((resolve) => setImmediate(resolve)); + expect(receivedFrame).toBe(false); + provider.pushFrame(1); + await waitUntil(() => receivedFrame); + await expect(replacementFrame).resolves.toMatchObject({ done: false }); + expect(provider.stoppedStreams).toEqual(["tab-1"]); + await replacement.return!(); + expect(provider.stoppedStreams).toEqual(["tab-1", "tab-1"]); + } finally { + releaseStop.resolve(); + replacementAbort.abort(); + await Promise.allSettled([previous.return?.(), replacement.return?.()]); + await service.close(); + } + }); + + it("cancels a replacement watcher without waiting for the old provider stop", async () => { + const { service, provider } = createService({ ...ALL_OFF, isolatedChromium: true }); + await service.listTargets("isolated-chromium"); + const target = tab("tab-1"); + const controller = new AbortController(); + const previous = service.watchFrames("isolated-chromium", target)[Symbol.asyncIterator](); + const replacement = service.watchFrames("isolated-chromium", target, controller.signal)[Symbol.asyncIterator](); + const stopEntered = Promise.withResolvers(); + const releaseStop = Promise.withResolvers(); + const stopImageStream = provider.stopImageStream.bind(provider); + vi.spyOn(provider, "stopImageStream").mockImplementationOnce(async (targetId) => { + stopEntered.resolve(); + await releaseStop.promise; + await stopImageStream(targetId); + }); + try { + const first = previous.next(); + await waitUntil(() => provider.startedStreams.length === 1); + provider.pushFrame(1); + await expect(first).resolves.toMatchObject({ done: false }); + const previousClosed = previous.return!(); + await stopEntered.promise; + const next = replacement.next(); + await new Promise((resolve) => setImmediate(resolve)); + expect(provider.startedStreams).toEqual(["tab-1"]); + controller.abort(); + let cancelled = false; + void next.then((result) => { cancelled = Boolean(result.done); }, () => undefined); + await waitUntil(() => cancelled); + await expect(next).resolves.toMatchObject({ done: true }); + expect(provider.stoppedStreams).toEqual([]); + releaseStop.resolve(); + await previousClosed; + expect(provider.startedStreams).toEqual(["tab-1"]); + expect(provider.stoppedStreams).toEqual(["tab-1"]); + } finally { + releaseStop.resolve(); + controller.abort(); + await Promise.allSettled([previous.return?.(), replacement.return?.()]); + await service.close(); + } + }); + + it("does not stop a provider stream while a second watcher remains", async () => { + const { service, provider } = createService({ ...ALL_OFF, isolatedChromium: true }); + await service.listTargets("isolated-chromium"); + const target = tab("tab-1"); + const controller = new AbortController(); + const a = service.watchFrames("isolated-chromium", target, controller.signal)[Symbol.asyncIterator](); + const b = service.watchFrames("isolated-chromium", target, controller.signal)[Symbol.asyncIterator](); + try { + const first = a.next(); + await waitUntil(() => provider.startedStreams.length === 1); + const second = b.next(); + await new Promise((resolve) => setImmediate(resolve)); + provider.pushFrame(1); + await expect(Promise.all([first, second])).resolves.toMatchObject([{ done: false }, { done: false }]); + await a.return!(); + expect(provider.startedStreams).toEqual(["tab-1"]); + expect(provider.stoppedStreams).toEqual([]); + const next = b.next(); + provider.pushFrame(1); + await expect(next).resolves.toMatchObject({ done: false }); + await b.return!(); + expect(provider.stoppedStreams).toEqual(["tab-1"]); + } finally { + controller.abort(); + await Promise.allSettled([a.return?.(), b.return?.()]); + await service.close(); + } + }); + + it("does not restart a provider closed while a replacement watcher is waiting", async () => { + const { service, provider } = createService({ ...ALL_OFF, isolatedChromium: true }); + await service.listTargets("isolated-chromium"); + const target = tab("tab-1"); + const controller = new AbortController(); + const previous = service.watchFrames("isolated-chromium", target, controller.signal)[Symbol.asyncIterator](); + const replacement = service.watchFrames("isolated-chromium", target, controller.signal)[Symbol.asyncIterator](); + const stopEntered = Promise.withResolvers(); + const releaseStop = Promise.withResolvers(); + vi.spyOn(provider, "stopImageStream").mockImplementationOnce(async () => { + stopEntered.resolve(); + await releaseStop.promise; + }); + try { + const first = previous.next(); + await waitUntil(() => provider.startedStreams.length === 1); + provider.pushFrame(1); + await expect(first).resolves.toMatchObject({ done: false }); + const previousClosed = previous.return!(); + await stopEntered.promise; + const failure = expect(replacement.next()).rejects.toMatchObject({ code: "browser_provider_unavailable" }); + await new Promise((resolve) => setImmediate(resolve)); + await service.close(); + releaseStop.resolve(); + await previousClosed; + await failure; + expect(provider.closed).toBe(true); + expect(provider.startedStreams).toEqual(["tab-1"]); + } finally { + releaseStop.resolve(); + controller.abort(); + await Promise.allSettled([previous.return?.(), replacement.return?.()]); + await service.close(); + } + }); +}); + /** One open stream, plus what it has delivered so far. */ interface OpenStream { readonly controller: AbortController;