From d6e3fc247e8c7cb05f9ce0f1e7a48d5c2b4bbfa0 Mon Sep 17 00:00:00 2001 From: Holger Benl Date: Fri, 25 Sep 2026 16:03:55 +0200 Subject: [PATCH] chore(bidi): scale screencast frames in the browser --- .../playwright-core/src/server/bidi/bidiPage.ts | 12 ++++-------- .../src/server/bidi/third_party/bidiProtocolCore.ts | 13 +++++++++++++ packages/playwright-core/src/server/page.ts | 2 -- .../playwright-core/src/server/videoRecorder.ts | 13 ++++++------- .../expectations/moz-firefox-nightly-library.txt | 6 +++++- .../bidi/expectations/moz-firefox-nightly-page.txt | 1 + 6 files changed, 29 insertions(+), 18 deletions(-) diff --git a/packages/playwright-core/src/server/bidi/bidiPage.ts b/packages/playwright-core/src/server/bidi/bidiPage.ts index 4245b07a587e8..25dbd077224fd 100644 --- a/packages/playwright-core/src/server/bidi/bidiPage.ts +++ b/packages/playwright-core/src/server/bidi/bidiPage.ts @@ -613,6 +613,10 @@ export class BidiPage implements PageDelegate { format: { type: 'image/jpeg', quality: options.quality / 100 + }, + imageSize: { + maxWidth: options.width, + maxHeight: options.height, } }); if (payload) { @@ -640,14 +644,6 @@ export class BidiPage implements PageDelegate { } } - getFFmpegVideoFilterArgs({ width, height }: { width: number, height: number }) { - // We use "scale" and "pad" video filters (-vf option) to resize incoming frames - // that might be of a different size to the desired video size. - // https://ffmpeg.org/ffmpeg-filters.html#scale - // https://ffmpeg.org/ffmpeg-filters.html#pad-1 - return `scale=w='min(iw,${width})':h='min(ih,${height})':force_original_aspect_ratio=decrease:eval=frame,pad=${width}:${height}:0:0:gray`; - } - rafCountForStablePosition(): number { return 1; } diff --git a/packages/playwright-core/src/server/bidi/third_party/bidiProtocolCore.ts b/packages/playwright-core/src/server/bidi/third_party/bidiProtocolCore.ts index 69413589d6e4c..2646a446aa42f 100644 --- a/packages/playwright-core/src/server/bidi/third_party/bidiProtocolCore.ts +++ b/packages/playwright-core/src/server/bidi/third_party/bidiProtocolCore.ts @@ -635,6 +635,7 @@ export namespace BrowsingContext { origin?: 'viewport' | 'document'; format?: BrowsingContext.ImageFormat; clip?: BrowsingContext.ClipRectangle; + imageSize?: BrowsingContext.ImageSize; }; } export namespace BrowsingContext { @@ -646,6 +647,18 @@ export namespace BrowsingContext { quality?: number; }; } +export namespace BrowsingContext { + export type ImageSize = { + /** + * Must be greater than or equal to `1`. + */ + maxWidth?: JsUint; + /** + * Must be greater than or equal to `1`. + */ + maxHeight?: JsUint; + }; +} export namespace BrowsingContext { export type ClipRectangle = | BrowsingContext.BoxClipRectangle diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts index 0c5d10398148f..f91acde0f2d4c 100644 --- a/packages/playwright-core/src/server/page.ts +++ b/packages/playwright-core/src/server/page.ts @@ -107,8 +107,6 @@ export interface PageDelegate { // WebKit hack. shouldToggleStyleSheetToSyncAnimations(): boolean; setDockTile(image: Buffer): Promise; - // Allow Bidi to set different ffmpeg video filter args. - getFFmpegVideoFilterArgs?: (options: { width: number, height: number }) => string; } type EmulatedSize = { screen: types.Size, viewport: types.Size }; diff --git a/packages/playwright-core/src/server/videoRecorder.ts b/packages/playwright-core/src/server/videoRecorder.ts index ffddeb56956cc..98e6cdbd042fc 100644 --- a/packages/playwright-core/src/server/videoRecorder.ts +++ b/packages/playwright-core/src/server/videoRecorder.ts @@ -30,7 +30,7 @@ import { registry } from './registry'; import type * as types from './types'; import type { ChildProcess } from 'child_process'; import type { Screencast, ScreencastClient } from './screencast'; -import type { Page, PageDelegate } from './page'; +import type { Page } from './page'; const kDefaultFps = 25; @@ -60,7 +60,7 @@ export class VideoRecorder { const { size } = this._screencast.addClient(this._client); // For video files only, prioritize encoding into the given size, regardless of the actual pixel data. const videoSize = options.size ?? size; - this._videoRecorder = new FfmpegVideoRecorder(ffmpegPath, videoSize, options.fps ?? kDefaultFps, outputFile, this._screencast.page.delegate); + this._videoRecorder = new FfmpegVideoRecorder(ffmpegPath, videoSize, options.fps ?? kDefaultFps, outputFile); this._artifact = new Artifact(this._screencast.page.browserContext, outputFile); return this._artifact; } @@ -109,7 +109,7 @@ class FfmpegVideoRecorder { private _launchPromise: Promise; private _outputFile: string; - constructor(ffmpegPath: string, size: types.Size, fps: number, outputFile: string, page: PageDelegate) { + constructor(ffmpegPath: string, size: types.Size, fps: number, outputFile: string) { if (!outputFile.endsWith('.webm')) throw new Error('File must have .webm extension'); this._outputFile = outputFile; @@ -117,10 +117,10 @@ class FfmpegVideoRecorder { this._size = size; this._fps = fps; this._creationTimeMs = Date.now(); - this._launchPromise = this._launch(page).catch(e => e); + this._launchPromise = this._launch().catch(e => e); } - private async _launch(page: PageDelegate) { + private async _launch() { await mkdirIfNeeded(this._outputFile); // How to tune the codec: // 1. Read vp8 documentation to figure out the options. @@ -170,7 +170,6 @@ class FfmpegVideoRecorder { const w = this._size.width; const h = this._size.height; - const videoFilterArgs = page.getFFmpegVideoFilterArgs?.({ width: w, height: h }) ?? `pad=${w}:${h}:0:0:gray,crop=${w}:${h}:0:0`; // Bitrate and encoder threads scale with the pixel rate relative to 800x450 at 25fps: // 800x450 at 25fps -> scale 1, bitrate 1000k, 1 thread // 800x800 at 25fps -> scale 1.78, bitrate 1778k, 1 thread @@ -179,7 +178,7 @@ class FfmpegVideoRecorder { const pixelRateScale = Math.max(1, w * h * this._fps / (800 * 450 * kDefaultFps)); const bitrate = Math.round(pixelRateScale * 1000); const threads = Math.min(8, Math.ceil(pixelRateScale / 4)); - const args = `-loglevel error -f matroska -fpsprobesize 0 -probesize 32 -analyzeduration 0 -i pipe:0 -y -an -r ${this._fps} -c:v vp8 -qmin 0 -qmax 50 -crf 8 -deadline realtime -speed 8 -b:v ${bitrate}k -threads ${threads} -vf ${videoFilterArgs}`.split(' '); + const args = `-loglevel error -f matroska -fpsprobesize 0 -probesize 32 -analyzeduration 0 -i pipe:0 -y -an -r ${this._fps} -c:v vp8 -qmin 0 -qmax 50 -crf 8 -deadline realtime -speed 8 -b:v ${bitrate}k -threads ${threads} -vf pad=${w}:${h}:0:0:gray,crop=${w}:${h}:0:0`.split(' '); args.push('-metadata', `creation_time=${new Date(this._creationTimeMs).toISOString()}`); args.push(this._outputFile); diff --git a/tests/bidi/expectations/moz-firefox-nightly-library.txt b/tests/bidi/expectations/moz-firefox-nightly-library.txt index 54d883f479692..af096803a20ec 100644 --- a/tests/bidi/expectations/moz-firefox-nightly-library.txt +++ b/tests/bidi/expectations/moz-firefox-nightly-library.txt @@ -44,6 +44,7 @@ library/browsercontext-reuse.spec.ts › reuse launch › should update viewport library/browsercontext-service-worker-policy.spec.ts › block › should not throw error on about:blank [fail] library/browsercontext-storage-state.spec.ts › should roundtrip local storage in third-party context [fail] library/browsercontext-storage-state.spec.ts › should set local storage in third-party context [fail] +library/browsercontext-storage-state.spec.ts › should work when service worker is intefering and the origin is not open [fail] library/browsercontext-user-agent.spec.ts › should override navigator.platform to match custom user agent [fail] library/browsercontext-viewport-mobile.spec.ts › mobile viewport › default mobile viewports to 980 width [fail] library/browsercontext-viewport-mobile.spec.ts › mobile viewport › should be detectable [fail] @@ -53,6 +54,7 @@ library/browsercontext-viewport-mobile.spec.ts › mobile viewport › should pr library/browsercontext-viewport-mobile.spec.ts › mobile viewport › should support touch emulation [fail] library/browsercontext-viewport-mobile.spec.ts › mobile viewport › should support window.orientation emulation [fail] library/browsercontext-viewport.spec.ts › should drag with high dpi [fail] +library/browsercontext-viewport.spec.ts › should keep touch emulation after screenshot beyond viewport [fail] library/browsercontext-viewport.spec.ts › should support touch with null viewport [fail] library/browsertype-connect.spec.ts › launchServer › should be able to connect 20 times to a single server without warnings [timeout] library/browsertype-connect.spec.ts › launchServer › should save download [fail] @@ -86,6 +88,8 @@ library/chromium/launcher.spec.ts › should report console messages from conten library/chromium/launcher.spec.ts › should return background pages [timeout] library/chromium/launcher.spec.ts › should return background pages when recording video [timeout] library/chromium/launcher.spec.ts › should support request/response events when using backgroundPage() [timeout] +library/client-certificates.spec.ts › browser › persistentContext › should intercept TLS and send no client certificate when noCertificate is set for the visited origin [fail] +library/client-certificates.spec.ts › browser › should intercept TLS and send no client certificate when noCertificate is set for the visited origin [fail] library/client-certificates.spec.ts › browser › should not intercept TLS for origins without a client certificate [fail] library/debug-controller.spec.ts › should allow setting recorder mode only for specific browser [fail] library/debug-controller.spec.ts › should record [flaky] @@ -171,7 +175,7 @@ library/role-utils.spec.ts › axe-core accessible-text [timeout] library/role-utils.spec.ts › wpt accname #2 [timeout] library/role-utils.spec.ts › wpt accname #3 [timeout] library/route-web-socket.spec.ts › should emit close upon frame detach [timeout] -library/screencast.spec.ts › screencast.start delivers frames via onFrame callback [fail] +library/screencast.spec.ts › onFrame receives viewport size [fail] library/screenshot.spec.ts › page screenshot › should work with device scale factor and scale:css [fail] library/screenshot.spec.ts › page screenshot › should work with device scale factor, clip and scale:css [fail] library/selector-generator.spec.ts › selector generator › should work in dynamic iframes without navigation [fail] diff --git a/tests/bidi/expectations/moz-firefox-nightly-page.txt b/tests/bidi/expectations/moz-firefox-nightly-page.txt index 5fde30e517f8d..72d12de13b0d4 100644 --- a/tests/bidi/expectations/moz-firefox-nightly-page.txt +++ b/tests/bidi/expectations/moz-firefox-nightly-page.txt @@ -68,6 +68,7 @@ page/page-click.spec.ts › should click in a transformed iframe [timeout] page/page-click.spec.ts › should click in a transformed iframe with force [fail] page/page-click.spec.ts › should click the button with fixed position inside an iframe [fail] page/page-click.spec.ts › should double click the button [fail] +page/page-click.spec.ts › should not retain removed iframe after clicking inside it [fail] page/page-click.spec.ts › should select the text by triple clicking [fail] page/page-drag.spec.ts › Drag and drop › should cancel on escape [fail] page/page-drag.spec.ts › Drag and drop › should respect the drop effect [fail]