chore(bidi): scale screencast frames in the browser

packages/playwright-core/src/server/bidi/bidiPage.ts

@@ -613,6 +613,10 @@ export class BidiPage implements PageDelegate {
         format: {
           type: 'image/jpeg',
           quality: options.quality / 100
+        },
+        imageSize: {
+          maxWidth: options.width,
+          maxHeight: options.height,
         }
       });
       if (payload) {
@@ -640,14 +644,6 @@ export class BidiPage implements PageDelegate {
     }
   }
 
-  getFFmpegVideoFilterArgs({ width, height }: { width: number, height: number }) {
-    // We use "scale" and "pad" video filters (-vf option) to resize incoming frames
-    // that might be of a different size to the desired video size.
-    //   https://ffmpeg.org/ffmpeg-filters.html#scale
-    //   https://ffmpeg.org/ffmpeg-filters.html#pad-1
-    return `scale=w='min(iw,${width})':h='min(ih,${height})':force_original_aspect_ratio=decrease:eval=frame,pad=${width}:${height}:0:0:gray`;
-  }
-
   rafCountForStablePosition(): number {
     return 1;
   }

packages/playwright-core/src/server/bidi/third_party/bidiProtocolCore.ts

@@ -635,6 +635,7 @@ export namespace BrowsingContext {
     origin?: 'viewport' | 'document';
     format?: BrowsingContext.ImageFormat;
     clip?: BrowsingContext.ClipRectangle;
+    imageSize?: BrowsingContext.ImageSize;
   };
 }
 export namespace BrowsingContext {
@@ -646,6 +647,18 @@ export namespace BrowsingContext {
     quality?: number;
   };
 }
+export namespace BrowsingContext {
+  export type ImageSize = {
+    /**
+     * Must be greater than or equal to `1`.
+     */
+    maxWidth?: JsUint;
+    /**
+     * Must be greater than or equal to `1`.
+     */
+    maxHeight?: JsUint;
+  };
+}
 export namespace BrowsingContext {
   export type ClipRectangle =
     | BrowsingContext.BoxClipRectangle

packages/playwright-core/src/server/page.ts

@@ -107,8 +107,6 @@ export interface PageDelegate {
   // WebKit hack.
   shouldToggleStyleSheetToSyncAnimations(): boolean;
   setDockTile(image: Buffer): Promise<void>;
-  // Allow Bidi to set different ffmpeg video filter args.
-  getFFmpegVideoFilterArgs?: (options: { width: number, height: number }) => string;
 }
 
 type EmulatedSize = { screen: types.Size, viewport: types.Size };

packages/playwright-core/src/server/videoRecorder.ts

@@ -30,7 +30,7 @@ import { registry } from './registry';
 import type * as types from './types';
 import type { ChildProcess } from 'child_process';
 import type { Screencast, ScreencastClient } from './screencast';
-import type { Page, PageDelegate } from './page';
+import type { Page } from './page';
 
 const kDefaultFps = 25;
 
@@ -60,7 +60,7 @@ export class VideoRecorder {
     const { size } = this._screencast.addClient(this._client);
     // For video files only, prioritize encoding into the given size, regardless of the actual pixel data.
     const videoSize = options.size ?? size;
-    this._videoRecorder = new FfmpegVideoRecorder(ffmpegPath, videoSize, options.fps ?? kDefaultFps, outputFile, this._screencast.page.delegate);
+    this._videoRecorder = new FfmpegVideoRecorder(ffmpegPath, videoSize, options.fps ?? kDefaultFps, outputFile);
     this._artifact = new Artifact(this._screencast.page.browserContext, outputFile);
     return this._artifact;
   }
@@ -109,18 +109,18 @@ class FfmpegVideoRecorder {
   private _launchPromise: Promise<Error | null>;
   private _outputFile: string;
 
-  constructor(ffmpegPath: string, size: types.Size, fps: number, outputFile: string, page: PageDelegate) {
+  constructor(ffmpegPath: string, size: types.Size, fps: number, outputFile: string) {
     if (!outputFile.endsWith('.webm'))
       throw new Error('File must have .webm extension');
     this._outputFile = outputFile;
     this._ffmpegPath = ffmpegPath;
     this._size = size;
     this._fps = fps;
     this._creationTimeMs = Date.now();
-    this._launchPromise = this._launch(page).catch(e => e);
+    this._launchPromise = this._launch().catch(e => e);
   }
 
-  private async _launch(page: PageDelegate) {
+  private async _launch() {
     await mkdirIfNeeded(this._outputFile);
     // How to tune the codec:
     // 1. Read vp8 documentation to figure out the options.
@@ -170,7 +170,6 @@ class FfmpegVideoRecorder {
 
     const w = this._size.width;
     const h = this._size.height;
-    const videoFilterArgs = page.getFFmpegVideoFilterArgs?.({ width: w, height: h }) ?? `pad=${w}:${h}:0:0:gray,crop=${w}:${h}:0:0`;
     // Bitrate and encoder threads scale with the pixel rate relative to 800x450 at 25fps:
     //   800x450 at 25fps   -> scale 1,    bitrate 1000k,  1 thread
     //   800x800 at 25fps   -> scale 1.78, bitrate 1778k,  1 thread
@@ -179,7 +178,7 @@ class FfmpegVideoRecorder {
     const pixelRateScale = Math.max(1, w * h * this._fps / (800 * 450 * kDefaultFps));
     const bitrate = Math.round(pixelRateScale * 1000);
     const threads = Math.min(8, Math.ceil(pixelRateScale / 4));
-    const args = `-loglevel error -f matroska -fpsprobesize 0 -probesize 32 -analyzeduration 0 -i pipe:0 -y -an -r ${this._fps} -c:v vp8 -qmin 0 -qmax 50 -crf 8 -deadline realtime -speed 8 -b:v ${bitrate}k -threads ${threads} -vf ${videoFilterArgs}`.split(' ');
+    const args = `-loglevel error -f matroska -fpsprobesize 0 -probesize 32 -analyzeduration 0 -i pipe:0 -y -an -r ${this._fps} -c:v vp8 -qmin 0 -qmax 50 -crf 8 -deadline realtime -speed 8 -b:v ${bitrate}k -threads ${threads} -vf pad=${w}:${h}:0:0:gray,crop=${w}:${h}:0:0`.split(' ');
     args.push('-metadata', `creation_time=${new Date(this._creationTimeMs).toISOString()}`);
     args.push(this._outputFile);
 

tests/bidi/expectations/moz-firefox-nightly-library.txt

@@ -44,6 +44,7 @@ library/browsercontext-reuse.spec.ts › reuse launch › should update viewport
 library/browsercontext-service-worker-policy.spec.ts › block › should not throw error on about:blank [fail]
 library/browsercontext-storage-state.spec.ts › should roundtrip local storage in third-party context [fail]
 library/browsercontext-storage-state.spec.ts › should set local storage in third-party context [fail]
+library/browsercontext-storage-state.spec.ts › should work when service worker is intefering and the origin is not open [fail]
 library/browsercontext-user-agent.spec.ts › should override navigator.platform to match custom user agent [fail]
 library/browsercontext-viewport-mobile.spec.ts › mobile viewport › default mobile viewports to 980 width [fail]
 library/browsercontext-viewport-mobile.spec.ts › mobile viewport › should be detectable [fail]
@@ -53,6 +54,7 @@ library/browsercontext-viewport-mobile.spec.ts › mobile viewport › should pr
 library/browsercontext-viewport-mobile.spec.ts › mobile viewport › should support touch emulation [fail]
 library/browsercontext-viewport-mobile.spec.ts › mobile viewport › should support window.orientation emulation [fail]
 library/browsercontext-viewport.spec.ts › should drag with high dpi [fail]
+library/browsercontext-viewport.spec.ts › should keep touch emulation after screenshot beyond viewport [fail]
 library/browsercontext-viewport.spec.ts › should support touch with null viewport [fail]
 library/browsertype-connect.spec.ts › launchServer › should be able to connect 20 times to a single server without warnings [timeout]
 library/browsertype-connect.spec.ts › launchServer › should save download [fail]
@@ -86,6 +88,8 @@ library/chromium/launcher.spec.ts › should report console messages from conten
 library/chromium/launcher.spec.ts › should return background pages [timeout]
 library/chromium/launcher.spec.ts › should return background pages when recording video [timeout]
 library/chromium/launcher.spec.ts › should support request/response events when using backgroundPage() [timeout]
+library/client-certificates.spec.ts › browser › persistentContext › should intercept TLS and send no client certificate when noCertificate is set for the visited origin [fail]
+library/client-certificates.spec.ts › browser › should intercept TLS and send no client certificate when noCertificate is set for the visited origin [fail]
 library/client-certificates.spec.ts › browser › should not intercept TLS for origins without a client certificate [fail]
 library/debug-controller.spec.ts › should allow setting recorder mode only for specific browser [fail]
 library/debug-controller.spec.ts › should record [flaky]
@@ -171,7 +175,7 @@ library/role-utils.spec.ts › axe-core accessible-text [timeout]
 library/role-utils.spec.ts › wpt accname #2 [timeout]
 library/role-utils.spec.ts › wpt accname #3 [timeout]
 library/route-web-socket.spec.ts › should emit close upon frame detach [timeout]
-library/screencast.spec.ts › screencast.start delivers frames via onFrame callback [fail]
+library/screencast.spec.ts › onFrame receives viewport size [fail]
 library/screenshot.spec.ts › page screenshot › should work with device scale factor and scale:css [fail]
 library/screenshot.spec.ts › page screenshot › should work with device scale factor, clip and scale:css [fail]
 library/selector-generator.spec.ts › selector generator › should work in dynamic iframes without navigation [fail]

tests/bidi/expectations/moz-firefox-nightly-page.txt

@@ -68,6 +68,7 @@ page/page-click.spec.ts › should click in a transformed iframe [timeout]
 page/page-click.spec.ts › should click in a transformed iframe with force [fail]
 page/page-click.spec.ts › should click the button with fixed position inside an iframe [fail]
 page/page-click.spec.ts › should double click the button [fail]
+page/page-click.spec.ts › should not retain removed iframe after clicking inside it [fail]
 page/page-click.spec.ts › should select the text by triple clicking [fail]
 page/page-drag.spec.ts › Drag and drop › should cancel on escape [fail]
 page/page-drag.spec.ts › Drag and drop › should respect the drop effect [fail]