Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 51 additions & 6 deletions packages/webui/src/runtime/browser/browser-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,8 @@ export class WebuiBrowserService {
WebuiBrowserSurface,
Map<string, (frame: WebuiBrowserImageFrame) => void>
>();
/** An old screencast must stop before a replacement starts on the same provider and target. */
readonly #frameStops = new WeakMap<WebuiBrowserSurfaceProvider, Map<string, Promise<void>>>();
/** How many frames each surface dropped, per surface. */
readonly #droppedFrames = new Map<WebuiBrowserSurface, number>();
/**
Expand Down Expand Up @@ -916,6 +918,7 @@ export class WebuiBrowserService {
this.#providers.delete(surface);
this.#selected.delete(surface);
this.#frameSubscribers.delete(surface);
this.#frameHandlers.delete(surface);
this.#webTabs.delete(surface);
this.#webTabSubscribers.delete(surface);
for (const target of this.#registry.list(surface)) {
Expand Down Expand Up @@ -1135,6 +1138,21 @@ export class WebuiBrowserService {
true,
);
}
const targetKey = target.targetId;
const stops = this.#frameStops.get(provider) ?? new Map<string, Promise<void>>();
this.#frameStops.set(provider, stops);
for (let pending = stops.get(targetKey); pending; pending = stops.get(targetKey)) {
await waitForFrameStop(pending, signal);
if (signal?.aborted) return;
}
if (signal?.aborted) return;
if (this.#closed || this.#providers.get(surface) !== provider) {
throw browserOperationError(
"browser_provider_unavailable",
`Browser surface "${surface}" has no connected browser.`,
true,
);
}
// The target has to be a page this service knows. A stream for a target
// that does not exist would deliver nothing at all, and a panel waiting
// for a picture that cannot arrive looks like a panel that is broken.
Expand Down Expand Up @@ -1187,7 +1205,6 @@ export class WebuiBrowserService {
// subscribers: the provider holds one frame handler per page, so two
// subscriptions for the same page would otherwise take each other's
// frames away.
const targetKey = target.targetId;
const surfaceSubscribers =
this.#frameSubscribers.get(surface) ??
new Map<string, Set<(frame: WebuiBrowserImageFrame) => void>>();
Expand All @@ -1206,8 +1223,8 @@ export class WebuiBrowserService {
surfaceHandlers.get(targetKey) ??
((frame: WebuiBrowserImageFrame) => {
const current = this.#frameSubscribers.get(surface)?.get(targetKey);
if (!current) return;
for (const subscriber of [...current]) subscriber(frame);
if (current !== subscribers || this.#providers.get(surface) !== provider) return;
for (const subscriber of [...subscribers]) subscriber(frame);
});
surfaceHandlers.set(targetKey, handler);
this.#frameHandlers.set(surface, surfaceHandlers);
Expand Down Expand Up @@ -1280,12 +1297,24 @@ export class WebuiBrowserService {
} finally {
if (firstFrameDeadline) clearTimeout(firstFrameDeadline);
subscribers.delete(deliver);
if (subscribers.size === 0) {
if (
subscribers.size === 0 &&
this.#providers.get(surface) === provider &&
this.#frameSubscribers.get(surface)?.get(targetKey) === subscribers &&
this.#frameHandlers.get(surface)?.get(targetKey) === handler
) {
// Cancelling the stream releases the provider-side resources it held:
// the screencast is stopped, not merely ignored.
await provider.stopImageStream(targetKey).catch(() => undefined);
// detach the old maps before yielding, then serialize a replacement's
// start after the old provider stop. A late stop must not erase a new
// subscriber or turn off the stream it just started.
surfaceSubscribers.delete(targetKey);
surfaceHandlers.delete(targetKey);
const stopping = Promise.resolve()
.then(() => provider.stopImageStream(targetKey))
.catch(() => undefined)
.finally(() => { stops.delete(targetKey); });
stops.set(targetKey, stopping);
await stopping;
}
}
}
Expand Down Expand Up @@ -1934,6 +1963,22 @@ function sameTab(left: WebuiBrowserTargetId, right: WebuiBrowserTargetId): boole
return left.surface === right.surface && left.browserId === right.browserId && left.targetId === right.targetId;
}

/** Waits for the old stream to stop without holding a canceled replacement open. */
async function waitForFrameStop(stop: Promise<void>, signal?: AbortSignal): Promise<void> {
if (!signal) return stop;
if (signal.aborted) return;
let onAbort: () => void = () => undefined;
const aborted = new Promise<void>((resolve) => {
onAbort = () => resolve();
signal.addEventListener("abort", onAbort, { once: true });
});
try {
await Promise.race([stop, aborted]);
} finally {
signal.removeEventListener("abort", onAbort);
}
}

/**
* Awaits one command, or refuses it when the caller's signal aborts.
*
Expand Down
115 changes: 96 additions & 19 deletions packages/webui/test/integration/agent-transcript.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,12 @@ import path from "node:path";
import { chromium, expect as browserExpect, type Browser, type Locator, type Page } from "@playwright/test";
import { afterAll, expect, it } from "vitest";
import type { WebuiBrowserAdapterBinding } from "../../src/runtime/harness/browser.js";
import { WEBUI_BROWSER_ERROR_CODES } from "../../src/shared/contracts/browser.js";
import {
OPEN_BROWSER_WEB_TAB_OPERATION_NAME,
WATCH_BROWSER_FRAMES_OPERATION_NAME,
WATCH_BROWSER_WEB_TABS_OPERATION_NAME,
} from "../../src/shared/operation-names.js";

const MODEL_ID = "stub-model";
const PROVIDER_NAME = "stub";
Expand Down Expand Up @@ -288,6 +294,61 @@ async function waitForBrowserFrameOrFailure(
}
}

type WireObservation = { at: number; direction: string; frame: unknown };

/** Correlate startup requests with wire outcomes without serializing any payload or identifier. */
function browserStartupWireEvidence(wire: readonly WireObservation[], from: number, startedAt: number) {
const operations: ReadonlySet<string> = new Set([
OPEN_BROWSER_WEB_TAB_OPERATION_NAME,
WATCH_BROWSER_FRAMES_OPERATION_NAME,
WATCH_BROWSER_WEB_TABS_OPERATION_NAME,
]);
const knownCodes: ReadonlySet<string> = new Set(WEBUI_BROWSER_ERROR_CODES);
const probes = new Map<string, {
operation: string;
sentAtMs: number;
responseAtMs?: number;
error?: { atMs: number; code: string };
events: number;
imageFrames: number;
firstEventAtMs?: number;
firstImageAtMs?: number;
}>();
for (const [index, { at, direction, frame }] of wire.entries()) {
if (!frame || typeof frame !== "object") continue;
const { requestId, kind, operation, code, body } = frame as Record<string, unknown>;
if (typeof requestId !== "string") continue;
const atMs = at - startedAt;
if (direction === "sent" && kind === "request") {
if (typeof operation === "string" && operations.has(operation) &&
(index >= from || operation === WATCH_BROWSER_WEB_TABS_OPERATION_NAME)) {
probes.set(requestId, { operation, sentAtMs: atMs, events: 0, imageFrames: 0 });
}
continue;
}
if (direction !== "received" || index < from) continue;
const probe = probes.get(requestId);
if (!probe) continue;
if (kind === "response") probe.responseAtMs ??= atMs;
if (kind === "error") {
probe.error ??= { atMs, code: typeof code === "string" && knownCodes.has(code) ? code : "other" };
}
if (kind === "event") {
probe.events++;
probe.firstEventAtMs ??= atMs;
if (probe.operation === WATCH_BROWSER_FRAMES_OPERATION_NAME && body && typeof body === "object") {
const { format, data, sequence } = body as Record<string, unknown>;
if ((format === "jpeg" || format === "png") &&
typeof data === "string" && data.length > 0 && Number.isSafeInteger(sequence)) {
probe.imageFrames++;
probe.firstImageAtMs ??= atMs;
}
}
}
}
return { requestCount: probes.size, requests: [...probes.values()].slice(-12) };
}

it.each(["transcript", "cross-client", "external-active-mutation", "external-active-mutation-gap"] as const)("real agent rendering and recovery: %s", async (scenario) => {
let releaseAnswer: () => void = () => undefined;
const answerGate = new Promise<void>((resolve) => { releaseAnswer = resolve; });
Expand Down Expand Up @@ -503,6 +564,7 @@ it.each(["transcript", "cross-client", "external-active-mutation", "external-act

const dataDir = await mkdtemp(path.join(tmpdir(), "webui-agent-transcript-e2e-"));
const previousDataDir = process.env.MINIMAX_DATA_DIR;
const previousBrowserOutputDir = process.env.WEBUI_BROWSER_OUTPUT_DIR;
const workspaceDir = path.join(dataDir, "workspace");
await mkdir(workspaceDir, { recursive: true });
await writeFile(path.join(workspaceDir, "reference-target.md"), "Synthetic workspace reference body.\n");
Expand Down Expand Up @@ -551,6 +613,10 @@ it.each(["transcript", "cross-client", "external-active-mutation", "external-act
let runtimeEventAbort: AbortController | undefined;
try {
process.env.MINIMAX_DATA_DIR = dataDir;
if (previousBrowserOutputDir?.trim()) {
// Other tests in this job otherwise overwrite the same diagnostic filenames.
process.env.WEBUI_BROWSER_OUTPUT_DIR = path.join(previousBrowserOutputDir, "agent-transcript", scenario);
}
(await import("@mavis/config")).resetConfig();
const { createWebuiRuntimeHost } = await import("../../src/runtime/assembly.js");
const { createHarnessPortFromHost } = await import("../../src/runtime/harness/adapter.js");
Expand Down Expand Up @@ -648,7 +714,7 @@ it.each(["transcript", "cross-client", "external-active-mutation", "external-act
body?: Record<string, unknown>;
}[] = [];
const received: { requestId?: string; kind?: string; body?: unknown }[] = [];
const wireTimeline: { at: number; direction: string; frame: unknown }[] = [];
const wireTimeline: WireObservation[] = [];
page.on("websocket", (socket) => {
socket.on("framesent", ({ payload }) => {
try {
Expand Down Expand Up @@ -1196,6 +1262,8 @@ it.each(["transcript", "cross-client", "external-active-mutation", "external-act
await browserExpect(browserLauncher).toBeVisible();
const pageUrl = `http://127.0.0.1:${pageAddress.port}/`;
await page.getByTestId("workspace-launcher-url").fill(pageUrl);
const browserStartupIndex = wireTimeline.length;
const browserStartupAt = Date.now();
await page.getByTestId("workspace-launcher-url-submit").click();
const browserFrame = page.getByTestId("browser-panel-frame");
// The panel can only render a frame after the isolated Chromium has
Expand Down Expand Up @@ -1224,7 +1292,10 @@ it.each(["transcript", "cross-client", "external-active-mutation", "external-act
`notice: ${notice?.text ?? "(none)"}`,
`toast: ${toast?.text ?? "(none)"}`,
`error code: ${toast?.errorCode ?? "(none)"}`,
JSON.stringify({ panelState: browserPanel.nodes }),
JSON.stringify({
panelState: browserPanel.nodes,
wire: browserStartupWireEvidence(wireTimeline, browserStartupIndex, browserStartupAt),
}),
].join("\n"));
}
if (browserPanel.outcome === "timeout") {
Expand All @@ -1236,6 +1307,7 @@ it.each(["transcript", "cross-client", "external-active-mutation", "external-act
.filter((node) => node.id === "browser-panel-unknown" || node.id === "browser-panel-unauthorized")
.map((node) => node.text),
panelState: browserPanel.nodes,
wire: browserStartupWireEvidence(wireTimeline, browserStartupIndex, browserStartupAt),
}),
].join("\n"));
}
Expand Down Expand Up @@ -2065,22 +2137,27 @@ it.each(["transcript", "cross-client", "external-active-mutation", "external-act
}
await observerContext.close();
} finally {
runtimeEventAbort?.abort();
releaseAnswer();
releaseTodoAnswer();
releaseStoppedAnswer();
releaseCrossClientHold();
releaseImmediateAnswer();
releaseGoalFollowUp();
await browser?.close();
await service?.close().catch(() => undefined);
await browserAdapter?.close().catch(() => undefined);
await assembled?.host.apiHost.close().catch(() => undefined);
await new Promise((resolve) => modelServer.close(resolve));
if (pageServer) await new Promise<void>((resolve) => pageServer!.close(() => resolve()));
await rm(dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
if (previousDataDir === undefined) delete process.env.MINIMAX_DATA_DIR;
else process.env.MINIMAX_DATA_DIR = previousDataDir;
(await import("@mavis/config")).resetConfig();
try {
runtimeEventAbort?.abort();
releaseAnswer();
releaseTodoAnswer();
releaseStoppedAnswer();
releaseCrossClientHold();
releaseImmediateAnswer();
releaseGoalFollowUp();
await browser?.close();
await service?.close().catch(() => undefined);
await browserAdapter?.close().catch(() => undefined);
await assembled?.host.apiHost.close().catch(() => undefined);
await new Promise((resolve) => modelServer.close(resolve));
if (pageServer) await new Promise<void>((resolve) => pageServer!.close(() => resolve()));
await rm(dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
if (previousDataDir === undefined) delete process.env.MINIMAX_DATA_DIR;
else process.env.MINIMAX_DATA_DIR = previousDataDir;
(await import("@mavis/config")).resetConfig();
} finally {
if (previousBrowserOutputDir === undefined) delete process.env.WEBUI_BROWSER_OUTPUT_DIR;
else process.env.WEBUI_BROWSER_OUTPUT_DIR = previousBrowserOutputDir;
}
}
}, 120_000);
Loading
Loading