diff --git a/packages/webui/src/runtime/browser/browser-service.ts b/packages/webui/src/runtime/browser/browser-service.ts index fa006415..224ef311 100644 --- a/packages/webui/src/runtime/browser/browser-service.ts +++ b/packages/webui/src/runtime/browser/browser-service.ts @@ -1176,6 +1176,7 @@ export class WebuiBrowserService { // second. A frame that is already superseded when it is read is not worth // delivering, so the queue holds the latest and drops the rest. const queue: WebuiBrowserImageFrame[] = []; + let receivedFrame = false; let notify: (() => void) | undefined; const deliver = (frame: WebuiBrowserImageFrame) => { // The fence. A frame belongs to the page behind the target the @@ -1190,6 +1191,7 @@ export class WebuiBrowserService { this.#scheduleDiagnostics(); return; } + receivedFrame = true; const index = queue.findIndex( (queued) => queued.target.targetId === frame.target.targetId && @@ -1258,6 +1260,16 @@ export class WebuiBrowserService { resolve(); }, firstFrameBudgetMs); }); + // A shared cast need not paint again when another panel joins a static + // page. Ask only for this subscriber's first picture, at its live size. + // Live images still win and keep flowing if the capture fails. + let watching = true; + const needsInitialFrame = () => watching && !receivedFrame && !signal?.aborted; + if (!first && needsInitialFrame() && provider.captureStreamInitialFrame) { + void provider.captureStreamInitialFrame(target, viewport, needsInitialFrame).then((frame) => { + if (frame && needsInitialFrame()) deliver(frame); + }).catch(() => undefined); + } try { while (!signal?.aborted) { while (queue.length > 0) { @@ -1295,6 +1307,7 @@ export class WebuiBrowserService { notify = undefined; } } finally { + watching = false; if (firstFrameDeadline) clearTimeout(firstFrameDeadline); subscribers.delete(deliver); if ( diff --git a/packages/webui/src/runtime/browser/isolated-chromium/provider.ts b/packages/webui/src/runtime/browser/isolated-chromium/provider.ts index 030de645..af195034 100644 --- a/packages/webui/src/runtime/browser/isolated-chromium/provider.ts +++ b/packages/webui/src/runtime/browser/isolated-chromium/provider.ts @@ -176,6 +176,12 @@ export interface WebuiBrowserSurfaceProvider { targetId: string, onFrame: (frame: WebuiBrowserImageFrame) => void, ): Promise; + /** One fresh frame for a late subscriber to a live but visually idle stream. */ + captureStreamInitialFrame?( + target: WebuiBrowserTargetId, + viewport?: WebuiBrowserViewport, + stillNeeded?: () => boolean, + ): Promise; /** Stops the live images and releases what the stream held. */ stopImageStream(targetId: string): Promise; /** Delivers one validated input command to the target. */ @@ -231,6 +237,16 @@ interface WebuiIsolatedChromiumPage { frameSequence: number; /** Whether a subscriber is currently asking for this page's pictures. */ streaming: boolean; + /** The requested geometry and whether its CDP change completed. */ + viewport?: { width: number; height: number; ready: boolean }; + pendingViewportChanges?: number; + /** At most one native initial-frame capture can be in flight per page. */ + initialCapture?: { + generation: number; + viewport: WebuiIsolatedChromiumPage["viewport"]; + handler: WebuiIsolatedChromiumPage["frameHandler"]; + result: Promise; + }; /** The subscriber that asked for this page's pictures. */ frameHandler?: (frame: WebuiBrowserImageFrame) => void; /** @@ -757,18 +773,33 @@ export class WebuiIsolatedChromiumProvider implements WebuiBrowserSurfaceProvide } const connection = this.#connection; if (!connection) return; - await this.#withAttachmentRetry(() => - connection.send( - "Emulation.setDeviceMetricsOverride", - { - width: Math.max(1, Math.round(viewport.width)), - height: Math.max(1, Math.round(viewport.height)), - deviceScaleFactor: 1, - mobile: false, - }, - page.sessionId, - ), - ); + const dimensions = { + width: Math.max(1, Math.round(viewport.width)), + height: Math.max(1, Math.round(viewport.height)), + ready: false, + }; + const hadPendingChange = (page.pendingViewportChanges ?? 0) > 0; + page.pendingViewportChanges = (page.pendingViewportChanges ?? 0) + 1; + page.viewport = dimensions; + try { + await this.#withAttachmentRetry(() => + connection.send( + "Emulation.setDeviceMetricsOverride", + { + width: dimensions.width, + height: dimensions.height, + deviceScaleFactor: 1, + mobile: false, + }, + page.sessionId, + ), + ); + // Attachment retries can reorder overlapping writes. The last request + // does not prove its geometry won if another reply was still pending. + dimensions.ready = !hadPendingChange; + } finally { + page.pendingViewportChanges -= 1; + } } async startImageStream( @@ -808,6 +839,51 @@ export class WebuiIsolatedChromiumProvider implements WebuiBrowserSurfaceProvide ); } + async captureStreamInitialFrame( + target: WebuiBrowserTargetId, + viewport?: WebuiBrowserViewport, + stillNeeded: () => boolean = () => true, + ): Promise { + const page = this.#pageOf(target.targetId); + const connection = this.#connection; + if (!page?.streaming || !connection || target.surface !== this.surface || + target.browserId !== this.browserId || page.generation !== target.generation) return; + const geometry = page.viewport; + if (geometry && !geometry.ready) return; + if (viewport && (!geometry || + geometry.width !== Math.max(1, Math.round(viewport.width)) || + geometry.height !== Math.max(1, Math.round(viewport.height)))) return; + const handler = page.frameHandler; + const current = () => this.#connection === connection && this.#pageOf(target.targetId) === page && + page.streaming && page.frameHandler === handler && page.viewport === geometry && + page.generation === target.generation; + + while (page.initialCapture) { + if (!current() || !stillNeeded()) return; + const pending = page.initialCapture; + const frame = await pending.result; + if (pending.generation === target.generation && pending.viewport === geometry && + pending.handler === handler) return current() && stillNeeded() ? frame : undefined; + } + if (!current() || !stillNeeded()) return; + const result = (async (): Promise => { + try { + const screenshot = await connection.send( + "Page.captureScreenshot", + { format: "jpeg", quality: 70, captureBeyondViewport: false }, + page.sessionId, + ) as { data?: unknown } | undefined; + if (!current() || typeof screenshot?.data !== "string" || !screenshot.data) return; + return { target: { ...target }, data: screenshot.data, format: "jpeg", sequence: ++page.frameSequence }; + } catch { + // A one-shot image is opportunistic; the real screencast can still deliver. + return undefined; + } + })().finally(() => { page.initialCapture = undefined; }); + page.initialCapture = { generation: target.generation, viewport: geometry, handler, result }; + return result; + } + async stopImageStream(targetId: string): Promise { const page = this.#pageOf(targetId); if (!page) return; diff --git a/packages/webui/test/integration/browser-panel-live-navigation.test.ts b/packages/webui/test/integration/browser-panel-live-navigation.test.ts index 1fa82af3..5e60386c 100644 --- a/packages/webui/test/integration/browser-panel-live-navigation.test.ts +++ b/packages/webui/test/integration/browser-panel-live-navigation.test.ts @@ -9,10 +9,11 @@ import { createServer, type Server } from "node:http"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { WebuiBrowserService } from "../../src/runtime/browser/browser-service.js"; import { WebuiBrowserOperationError } from "../../src/runtime/browser/errors.js"; +import { WebuiIsolatedChromiumProvider } from "../../src/runtime/browser/isolated-chromium/provider.js"; import type { WebuiBrowserAuthorization, WebuiBrowserImageFrame, @@ -121,6 +122,58 @@ describe.skipIf(!chromeAvailable)("the browser panel's real Chromium navigation } }); + it.each(["unchanged", "resized"] as const)("gives a late watcher its first frame on a static Chromium tab (%s)", async (mode) => { + const target = await openHome(); + const controller = new AbortController(); + const oldViewport = { width: 600, height: 500 }; + const requested = mode === "resized" ? { width: 700, height: 550 } : oldViewport; + const a = service.watchFrames("isolated-chromium", target, controller.signal, oldViewport)[Symbol.asyncIterator](); + const b = service.watchFrames("isolated-chromium", target, controller.signal, requested)[Symbol.asyncIterator](); + const startImageStream = WebuiIsolatedChromiumProvider.prototype.startImageStream; + let relayLiveFrames = true; + // Gate delivery after the real provider handles and ACKs CDP live frames. + // A platform may otherwise paint on resize and exercise only live delivery. + const liveCast = vi.spyOn(WebuiIsolatedChromiumProvider.prototype, "startImageStream") + .mockImplementation(function (this: WebuiIsolatedChromiumProvider, targetId, receive) { + return startImageStream.call(this, targetId, (frame) => { + if (relayLiveFrames) receive(frame); + }); + }); + const capture = vi.spyOn(WebuiIsolatedChromiumProvider.prototype, "captureStreamInitialFrame"); + try { + const first = await a.next(); + expect(first.done).toBe(false); + relayLiveFrames = false; + const next = b.next(); + let replacement: IteratorResult | undefined; + void next.then((frame) => { replacement = frame; }, () => undefined); + await waitFor(async () => replacement !== undefined, "replacement watch's first image"); + expect(replacement).toMatchObject({ done: false, value: { target, format: "jpeg" } }); + expect(replacement?.value.sequence).toBeGreaterThan(first.value.sequence); + const viewport = await evaluate(target, "JSON.stringify([innerWidth, innerHeight])"); + expect(viewport).toBe(JSON.stringify([requested.width, requested.height])); + if (!replacement || replacement.done) throw new Error("the replacement watch closed without an image"); + expect(capture).toHaveBeenCalledTimes(1); + // Verify the particular image is the capture, not an earlier live frame. + expect(replacement.value).toBe(await capture.mock.results[0]?.value); + const image = await evaluate(target, `(async () => { + const image = new Image(); + image.src = ${JSON.stringify("data:image/jpeg;base64," + replacement.value.data)}; + await image.decode(); + return JSON.stringify([image.naturalWidth, image.naturalHeight]); + })()`); + expect(image).toBe(viewport); + } finally { + controller.abort(); + try { + await Promise.allSettled([a.return?.(), b.return?.()]); + } finally { + liveCast.mockRestore(); + capture.mockRestore(); + } + } + }, 30_000); + async function openHome(): Promise { const opened = await service.openWebTab(undefined, { url: `${homeOrigin}/` }, undefined, "person"); await waitFor(async () => (await readTitle(opened.target)) === "Browser panel start", "home page load"); diff --git a/packages/webui/test/unit/browser-service.test.ts b/packages/webui/test/unit/browser-service.test.ts index 91273f17..eab66fc9 100644 --- a/packages/webui/test/unit/browser-service.test.ts +++ b/packages/webui/test/unit/browser-service.test.ts @@ -169,6 +169,10 @@ class RecordingProvider implements WebuiBrowserSurfaceProvider { }); } + async captureStreamInitialFrame(): Promise { + return undefined; + } + async stopImageStream(targetId: string): Promise { this.stoppedStreams.push(targetId); this.frameHandlers.delete(targetId); @@ -1558,6 +1562,75 @@ describe("browser frame subscription handoff", () => { } }); + it("shows an already-streaming static tab to an overlapping watcher without a new paint", async () => { + const { service, provider } = createService({ ...ALL_OFF, isolatedChromium: true }); + await service.listTargets("isolated-chromium"); + const target = tab("tab-1"); + vi.spyOn(provider, "captureStreamInitialFrame").mockResolvedValue({ + target, data: "AAAA", format: "jpeg", sequence: 2, + }); + const controller = new AbortController(); + const a = service.watchFrames("isolated-chromium", target, controller.signal)[Symbol.asyncIterator](); + const b = service.watchFrames("isolated-chromium", target, controller.signal)[Symbol.asyncIterator](); + try { + const first = a.next(); + await waitUntil(() => provider.startedStreams.length === 1); + provider.pushFrame(1); + await expect(first).resolves.toMatchObject({ done: false, value: { sequence: 1 } }); + const successor = b.next(); + let received = false; + void successor.then((result) => { received = !result.done; }, () => undefined); + await waitUntil(() => received); + await expect(successor).resolves.toMatchObject({ done: false, value: { sequence: 2 } }); + await a.return!(); + expect(provider.stoppedStreams).toEqual([]); + expect(provider.startedStreams).toEqual(["tab-1"]); + } finally { + controller.abort(); + await Promise.allSettled([a.return?.(), b.return?.()]); + await service.close(); + } + }); + + it.each(["rejects", "wrong-generation", "late"] as const)( + "keeps live delivery when an initial frame capture is %s", + async (mode) => { + const { service, provider } = createService({ ...ALL_OFF, isolatedChromium: true }); + await service.listTargets("isolated-chromium"); + const target = tab("tab-1"); + const captured = Promise.withResolvers(); + const capture = vi.spyOn(provider, "captureStreamInitialFrame").mockReturnValue(captured.promise); + const controller = new AbortController(); + const a = service.watchFrames("isolated-chromium", target, controller.signal)[Symbol.asyncIterator](); + const b = service.watchFrames("isolated-chromium", target, controller.signal)[Symbol.asyncIterator](); + try { + const first = a.next(); + await waitUntil(() => provider.startedStreams.length === 1); + provider.pushFrame(1); + await expect(first).resolves.toMatchObject({ done: false }); + const successor = b.next(); + let received = false; + void successor.then((frame) => { received = !frame.done; }, () => undefined); + await waitUntil(() => capture.mock.calls.length === 1); + const snapshot = { target, data: "BBBB", format: "jpeg", sequence: 2 } as const; + if (mode === "rejects") captured.reject(new Error("Synthetic capture failure")); + if (mode === "wrong-generation") { + captured.resolve({ ...snapshot, target: { ...target, generation: target.generation + 1 } }); + } + await new Promise((resolve) => setImmediate(resolve)); + expect(received).toBe(false); + provider.pushFrame(1); + if (mode === "late") captured.resolve(snapshot); + await expect(successor).resolves.toMatchObject({ done: false, value: { data: "AAAA", target } }); + } finally { + captured.resolve(undefined); + controller.abort(); + await Promise.allSettled([a.return?.(), b.return?.()]); + await service.close(); + } + }, + ); + it("does not stop a provider stream while a second watcher remains", async () => { const { service, provider } = createService({ ...ALL_OFF, isolatedChromium: true }); await service.listTargets("isolated-chromium"); diff --git a/packages/webui/test/unit/design-runtime.test.ts b/packages/webui/test/unit/design-runtime.test.ts index ecc1a554..da9d9ddd 100644 --- a/packages/webui/test/unit/design-runtime.test.ts +++ b/packages/webui/test/unit/design-runtime.test.ts @@ -182,13 +182,21 @@ describe("native Design persistence and execution", () => { state = await service.apply({ documentId: id, action: { type: "add_node", nodeType: "text", prompt: "downstream" } }); const second = state.document!.nodes[1]; state = await service.apply({ documentId: id, action: { type: "connect", from: first.id, to: second.id } }); state = await service.apply({ documentId: id, action: { type: "update_node", nodeId: first.id, patch: { metadata: { output: { status: "idle", text: "previous product", media: [], error: "" } } } } }); - await service.apply({ documentId: id, action: { type: "run", nodeId: first.id } }); - await eventually(async () => expect((await service.getState({ documentId: id })).document!.nodes[0].metadata.output.status).toBe("error")); - state = await service.getState({ documentId: id }); expect(state.document!.nodes[0].metadata.output.text).toBe("previous product"); expect(calls).toHaveLength(1); - expect(typeof state.tasks[0].startedAt).toBe("number"); expect(typeof state.tasks[0].finishedAt).toBe("number"); - const finishedAt = state.tasks[0].finishedAt; + const run = await service.apply({ documentId: id, action: { type: "run", nodeId: first.id } }); + const taskId = (run.result as { id: string }).id; + await eventually(async () => { + const settled = await service.getState({ documentId: id }); + expect(settled.document!.nodes[0].metadata.output.status).toBe("error"); + // The node product is saved before the task journal records settlement. + expect(settled.tasks.find((task) => task.id === taskId)).toMatchObject({ + status: "error", startedAt: expect.any(Number), finishedAt: expect.any(Number), + }); + state = settled; + }); + expect(state.document!.nodes[0].metadata.output.text).toBe("previous product"); expect(calls).toHaveLength(1); + const finishedAt = state.tasks.find((task) => task.id === taskId)!.finishedAt; const revision = state.document!.updatedAt; await new Promise((resolve) => setTimeout(resolve, 600)); - const after = await service.getState({ documentId: id }); expect(after.document!.updatedAt).toBe(revision); expect(after.tasks[0].finishedAt).toBe(finishedAt); + const after = await service.getState({ documentId: id }); expect(after.document!.updatedAt).toBe(revision); expect(after.tasks.find((task) => task.id === taskId)?.finishedAt).toBe(finishedAt); }); it("executes upstream chains in order and preserves populated boards when applying templates", async () => {