Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions packages/webui/src/runtime/browser/browser-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1176,6 +1176,7 @@ export class WebuiBrowserService {
// second. A frame that is already superseded when it is read is not worth
// delivering, so the queue holds the latest and drops the rest.
const queue: WebuiBrowserImageFrame[] = [];
let receivedFrame = false;
let notify: (() => void) | undefined;
const deliver = (frame: WebuiBrowserImageFrame) => {
// The fence. A frame belongs to the page behind the target the
Expand All @@ -1190,6 +1191,7 @@ export class WebuiBrowserService {
this.#scheduleDiagnostics();
return;
}
receivedFrame = true;
const index = queue.findIndex(
(queued) =>
queued.target.targetId === frame.target.targetId &&
Expand Down Expand Up @@ -1258,6 +1260,16 @@ export class WebuiBrowserService {
resolve();
}, firstFrameBudgetMs);
});
// A shared cast need not paint again when another panel joins a static
// page. Ask only for this subscriber's first picture, at its live size.
// Live images still win and keep flowing if the capture fails.
let watching = true;
const needsInitialFrame = () => watching && !receivedFrame && !signal?.aborted;
if (!first && needsInitialFrame() && provider.captureStreamInitialFrame) {
void provider.captureStreamInitialFrame(target, viewport, needsInitialFrame).then((frame) => {
if (frame && needsInitialFrame()) deliver(frame);
}).catch(() => undefined);
}
try {
while (!signal?.aborted) {
while (queue.length > 0) {
Expand Down Expand Up @@ -1295,6 +1307,7 @@ export class WebuiBrowserService {
notify = undefined;
}
} finally {
watching = false;
if (firstFrameDeadline) clearTimeout(firstFrameDeadline);
subscribers.delete(deliver);
if (
Expand Down
100 changes: 88 additions & 12 deletions packages/webui/src/runtime/browser/isolated-chromium/provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,12 @@ export interface WebuiBrowserSurfaceProvider {
targetId: string,
onFrame: (frame: WebuiBrowserImageFrame) => void,
): Promise<void>;
/** One fresh frame for a late subscriber to a live but visually idle stream. */
captureStreamInitialFrame?(
target: WebuiBrowserTargetId,
viewport?: WebuiBrowserViewport,
stillNeeded?: () => boolean,
): Promise<WebuiBrowserImageFrame | undefined>;
/** Stops the live images and releases what the stream held. */
stopImageStream(targetId: string): Promise<void>;
/** Delivers one validated input command to the target. */
Expand Down Expand Up @@ -231,6 +237,16 @@ interface WebuiIsolatedChromiumPage {
frameSequence: number;
/** Whether a subscriber is currently asking for this page's pictures. */
streaming: boolean;
/** The requested geometry and whether its CDP change completed. */
viewport?: { width: number; height: number; ready: boolean };
pendingViewportChanges?: number;
/** At most one native initial-frame capture can be in flight per page. */
initialCapture?: {
generation: number;
viewport: WebuiIsolatedChromiumPage["viewport"];
handler: WebuiIsolatedChromiumPage["frameHandler"];
result: Promise<WebuiBrowserImageFrame | undefined>;
};
/** The subscriber that asked for this page's pictures. */
frameHandler?: (frame: WebuiBrowserImageFrame) => void;
/**
Expand Down Expand Up @@ -757,18 +773,33 @@ export class WebuiIsolatedChromiumProvider implements WebuiBrowserSurfaceProvide
}
const connection = this.#connection;
if (!connection) return;
await this.#withAttachmentRetry(() =>
connection.send(
"Emulation.setDeviceMetricsOverride",
{
width: Math.max(1, Math.round(viewport.width)),
height: Math.max(1, Math.round(viewport.height)),
deviceScaleFactor: 1,
mobile: false,
},
page.sessionId,
),
);
const dimensions = {
width: Math.max(1, Math.round(viewport.width)),
height: Math.max(1, Math.round(viewport.height)),
ready: false,
};
const hadPendingChange = (page.pendingViewportChanges ?? 0) > 0;
page.pendingViewportChanges = (page.pendingViewportChanges ?? 0) + 1;
page.viewport = dimensions;
try {
await this.#withAttachmentRetry(() =>
connection.send(
"Emulation.setDeviceMetricsOverride",
{
width: dimensions.width,
height: dimensions.height,
deviceScaleFactor: 1,
mobile: false,
},
page.sessionId,
),
);
// Attachment retries can reorder overlapping writes. The last request
// does not prove its geometry won if another reply was still pending.
dimensions.ready = !hadPendingChange;
} finally {
page.pendingViewportChanges -= 1;
}
}

async startImageStream(
Expand Down Expand Up @@ -808,6 +839,51 @@ export class WebuiIsolatedChromiumProvider implements WebuiBrowserSurfaceProvide
);
}

async captureStreamInitialFrame(
target: WebuiBrowserTargetId,
viewport?: WebuiBrowserViewport,
stillNeeded: () => boolean = () => true,
): Promise<WebuiBrowserImageFrame | undefined> {
const page = this.#pageOf(target.targetId);
const connection = this.#connection;
if (!page?.streaming || !connection || target.surface !== this.surface ||
target.browserId !== this.browserId || page.generation !== target.generation) return;
const geometry = page.viewport;
if (geometry && !geometry.ready) return;
if (viewport && (!geometry ||
geometry.width !== Math.max(1, Math.round(viewport.width)) ||
geometry.height !== Math.max(1, Math.round(viewport.height)))) return;
const handler = page.frameHandler;
const current = () => this.#connection === connection && this.#pageOf(target.targetId) === page &&
page.streaming && page.frameHandler === handler && page.viewport === geometry &&
page.generation === target.generation;

while (page.initialCapture) {
if (!current() || !stillNeeded()) return;
const pending = page.initialCapture;
const frame = await pending.result;
if (pending.generation === target.generation && pending.viewport === geometry &&
pending.handler === handler) return current() && stillNeeded() ? frame : undefined;
}
if (!current() || !stillNeeded()) return;
const result = (async (): Promise<WebuiBrowserImageFrame | undefined> => {
try {
const screenshot = await connection.send(
"Page.captureScreenshot",
{ format: "jpeg", quality: 70, captureBeyondViewport: false },
page.sessionId,
) as { data?: unknown } | undefined;
if (!current() || typeof screenshot?.data !== "string" || !screenshot.data) return;
return { target: { ...target }, data: screenshot.data, format: "jpeg", sequence: ++page.frameSequence };
} catch {
// A one-shot image is opportunistic; the real screencast can still deliver.
return undefined;
}
})().finally(() => { page.initialCapture = undefined; });
page.initialCapture = { generation: target.generation, viewport: geometry, handler, result };
return result;
}

async stopImageStream(targetId: string): Promise<void> {
const page = this.#pageOf(targetId);
if (!page) return;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,11 @@ import { createServer, type Server } from "node:http";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";

import { WebuiBrowserService } from "../../src/runtime/browser/browser-service.js";
import { WebuiBrowserOperationError } from "../../src/runtime/browser/errors.js";
import { WebuiIsolatedChromiumProvider } from "../../src/runtime/browser/isolated-chromium/provider.js";
import type {
WebuiBrowserAuthorization,
WebuiBrowserImageFrame,
Expand Down Expand Up @@ -121,6 +122,58 @@ describe.skipIf(!chromeAvailable)("the browser panel's real Chromium navigation
}
});

it.each(["unchanged", "resized"] as const)("gives a late watcher its first frame on a static Chromium tab (%s)", async (mode) => {
const target = await openHome();
const controller = new AbortController();
const oldViewport = { width: 600, height: 500 };
const requested = mode === "resized" ? { width: 700, height: 550 } : oldViewport;
const a = service.watchFrames("isolated-chromium", target, controller.signal, oldViewport)[Symbol.asyncIterator]();
const b = service.watchFrames("isolated-chromium", target, controller.signal, requested)[Symbol.asyncIterator]();
const startImageStream = WebuiIsolatedChromiumProvider.prototype.startImageStream;
let relayLiveFrames = true;
// Gate delivery after the real provider handles and ACKs CDP live frames.
// A platform may otherwise paint on resize and exercise only live delivery.
const liveCast = vi.spyOn(WebuiIsolatedChromiumProvider.prototype, "startImageStream")
.mockImplementation(function (this: WebuiIsolatedChromiumProvider, targetId, receive) {
return startImageStream.call(this, targetId, (frame) => {
if (relayLiveFrames) receive(frame);
});
});
const capture = vi.spyOn(WebuiIsolatedChromiumProvider.prototype, "captureStreamInitialFrame");
try {
const first = await a.next();
expect(first.done).toBe(false);
relayLiveFrames = false;
const next = b.next();
let replacement: IteratorResult<WebuiBrowserImageFrame> | undefined;
void next.then((frame) => { replacement = frame; }, () => undefined);
await waitFor(async () => replacement !== undefined, "replacement watch's first image");
expect(replacement).toMatchObject({ done: false, value: { target, format: "jpeg" } });
expect(replacement?.value.sequence).toBeGreaterThan(first.value.sequence);
const viewport = await evaluate(target, "JSON.stringify([innerWidth, innerHeight])");
expect(viewport).toBe(JSON.stringify([requested.width, requested.height]));
if (!replacement || replacement.done) throw new Error("the replacement watch closed without an image");
expect(capture).toHaveBeenCalledTimes(1);
// Verify the particular image is the capture, not an earlier live frame.
expect(replacement.value).toBe(await capture.mock.results[0]?.value);
const image = await evaluate(target, `(async () => {
const image = new Image();
image.src = ${JSON.stringify("data:image/jpeg;base64," + replacement.value.data)};
await image.decode();
return JSON.stringify([image.naturalWidth, image.naturalHeight]);
})()`);
expect(image).toBe(viewport);
} finally {
controller.abort();
try {
await Promise.allSettled([a.return?.(), b.return?.()]);
} finally {
liveCast.mockRestore();
capture.mockRestore();
}
}
}, 30_000);

async function openHome(): Promise<WebuiBrowserTargetId> {
const opened = await service.openWebTab(undefined, { url: `${homeOrigin}/` }, undefined, "person");
await waitFor(async () => (await readTitle(opened.target)) === "Browser panel start", "home page load");
Expand Down
73 changes: 73 additions & 0 deletions packages/webui/test/unit/browser-service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,10 @@ class RecordingProvider implements WebuiBrowserSurfaceProvider {
});
}

async captureStreamInitialFrame(): Promise<WebuiBrowserImageFrame | undefined> {
return undefined;
}

async stopImageStream(targetId: string): Promise<void> {
this.stoppedStreams.push(targetId);
this.frameHandlers.delete(targetId);
Expand Down Expand Up @@ -1558,6 +1562,75 @@ describe("browser frame subscription handoff", () => {
}
});

it("shows an already-streaming static tab to an overlapping watcher without a new paint", async () => {
const { service, provider } = createService({ ...ALL_OFF, isolatedChromium: true });
await service.listTargets("isolated-chromium");
const target = tab("tab-1");
vi.spyOn(provider, "captureStreamInitialFrame").mockResolvedValue({
target, data: "AAAA", format: "jpeg", sequence: 2,
});
const controller = new AbortController();
const a = service.watchFrames("isolated-chromium", target, controller.signal)[Symbol.asyncIterator]();
const b = service.watchFrames("isolated-chromium", target, controller.signal)[Symbol.asyncIterator]();
try {
const first = a.next();
await waitUntil(() => provider.startedStreams.length === 1);
provider.pushFrame(1);
await expect(first).resolves.toMatchObject({ done: false, value: { sequence: 1 } });
const successor = b.next();
let received = false;
void successor.then((result) => { received = !result.done; }, () => undefined);
await waitUntil(() => received);
await expect(successor).resolves.toMatchObject({ done: false, value: { sequence: 2 } });
await a.return!();
expect(provider.stoppedStreams).toEqual([]);
expect(provider.startedStreams).toEqual(["tab-1"]);
} finally {
controller.abort();
await Promise.allSettled([a.return?.(), b.return?.()]);
await service.close();
}
});

it.each(["rejects", "wrong-generation", "late"] as const)(
"keeps live delivery when an initial frame capture is %s",
async (mode) => {
const { service, provider } = createService({ ...ALL_OFF, isolatedChromium: true });
await service.listTargets("isolated-chromium");
const target = tab("tab-1");
const captured = Promise.withResolvers<WebuiBrowserImageFrame | undefined>();
const capture = vi.spyOn(provider, "captureStreamInitialFrame").mockReturnValue(captured.promise);
const controller = new AbortController();
const a = service.watchFrames("isolated-chromium", target, controller.signal)[Symbol.asyncIterator]();
const b = service.watchFrames("isolated-chromium", target, controller.signal)[Symbol.asyncIterator]();
try {
const first = a.next();
await waitUntil(() => provider.startedStreams.length === 1);
provider.pushFrame(1);
await expect(first).resolves.toMatchObject({ done: false });
const successor = b.next();
let received = false;
void successor.then((frame) => { received = !frame.done; }, () => undefined);
await waitUntil(() => capture.mock.calls.length === 1);
const snapshot = { target, data: "BBBB", format: "jpeg", sequence: 2 } as const;
if (mode === "rejects") captured.reject(new Error("Synthetic capture failure"));
if (mode === "wrong-generation") {
captured.resolve({ ...snapshot, target: { ...target, generation: target.generation + 1 } });
}
await new Promise<void>((resolve) => setImmediate(resolve));
expect(received).toBe(false);
provider.pushFrame(1);
if (mode === "late") captured.resolve(snapshot);
await expect(successor).resolves.toMatchObject({ done: false, value: { data: "AAAA", target } });
} finally {
captured.resolve(undefined);
controller.abort();
await Promise.allSettled([a.return?.(), b.return?.()]);
await service.close();
}
},
);

it("does not stop a provider stream while a second watcher remains", async () => {
const { service, provider } = createService({ ...ALL_OFF, isolatedChromium: true });
await service.listTargets("isolated-chromium");
Expand Down
20 changes: 14 additions & 6 deletions packages/webui/test/unit/design-runtime.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -182,13 +182,21 @@ describe("native Design persistence and execution", () => {
state = await service.apply({ documentId: id, action: { type: "add_node", nodeType: "text", prompt: "downstream" } }); const second = state.document!.nodes[1];
state = await service.apply({ documentId: id, action: { type: "connect", from: first.id, to: second.id } });
state = await service.apply({ documentId: id, action: { type: "update_node", nodeId: first.id, patch: { metadata: { output: { status: "idle", text: "previous product", media: [], error: "" } } } } });
await service.apply({ documentId: id, action: { type: "run", nodeId: first.id } });
await eventually(async () => expect((await service.getState({ documentId: id })).document!.nodes[0].metadata.output.status).toBe("error"));
state = await service.getState({ documentId: id }); expect(state.document!.nodes[0].metadata.output.text).toBe("previous product"); expect(calls).toHaveLength(1);
expect(typeof state.tasks[0].startedAt).toBe("number"); expect(typeof state.tasks[0].finishedAt).toBe("number");
const finishedAt = state.tasks[0].finishedAt;
const run = await service.apply({ documentId: id, action: { type: "run", nodeId: first.id } });
const taskId = (run.result as { id: string }).id;
await eventually(async () => {
const settled = await service.getState({ documentId: id });
expect(settled.document!.nodes[0].metadata.output.status).toBe("error");
// The node product is saved before the task journal records settlement.
expect(settled.tasks.find((task) => task.id === taskId)).toMatchObject({
status: "error", startedAt: expect.any(Number), finishedAt: expect.any(Number),
});
state = settled;
});
expect(state.document!.nodes[0].metadata.output.text).toBe("previous product"); expect(calls).toHaveLength(1);
const finishedAt = state.tasks.find((task) => task.id === taskId)!.finishedAt;
const revision = state.document!.updatedAt; await new Promise((resolve) => setTimeout(resolve, 600));
const after = await service.getState({ documentId: id }); expect(after.document!.updatedAt).toBe(revision); expect(after.tasks[0].finishedAt).toBe(finishedAt);
const after = await service.getState({ documentId: id }); expect(after.document!.updatedAt).toBe(revision); expect(after.tasks.find((task) => task.id === taskId)?.finishedAt).toBe(finishedAt);
});

it("executes upstream chains in order and preserves populated boards when applying templates", async () => {
Expand Down