From 4b1f4249ecc987bb8112d896ed95cf21148938e0 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Thu, 10 Sep 2026 17:23:30 -0700 Subject: [PATCH 01/21] feat(coverage): experimental istanbul coverage collection via tracing tracing.start({ coverage: true }) enables collection of istanbul coverage from instrumented application code. A server-side CoverageRecorder installs a binding and init script into the browser context, accumulates counters relayed right before navigations, sweeps all frames when a trace chunk is stopped and stores the result as coverage.json inside the trace file. tracing.flushCoverage() syncs pages that are about to be closed by in-page script; pages closed via the API are swept automatically. The test runner enables collection with trace: { coverage: true } in the config. --- docs/src/api/class-tracing.md | 15 +++ docs/src/test-api/class-testoptions.md | 1 + packages/injected/src/coverageScript.ts | 87 ++++++++++++ packages/isomorphic/index.ts | 1 + packages/isomorphic/istanbulCoverage.ts | 76 +++++++++++ packages/isomorphic/protocolMetainfo.ts | 1 + packages/playwright-client/types/types.d.ts | 24 ++++ .../playwright-core/src/client/channels.d.ts | 6 + .../playwright-core/src/client/tracing.ts | 9 +- .../playwright-core/src/server/channels.d.ts | 6 + .../src/server/coverageRecorder.ts | 97 +++++++++++++ .../server/dispatchers/tracingDispatcher.ts | 4 + packages/playwright-core/src/server/page.ts | 1 + .../src/server/trace/recorder/tracing.ts | 43 +++++- packages/playwright-core/types/types.d.ts | 24 ++++ packages/playwright/src/worker/testTracing.ts | 4 +- packages/playwright/types/test.d.ts | 2 +- packages/protocol/spec/tracing.yml | 5 + packages/protocol/src/validator.ts | 3 + tests/library/browsertype-connect.spec.ts | 20 +++ tests/library/coverage.spec.ts | 92 +++++++++++++ tests/playwright-test/coverage.spec.ts | 127 ++++++++++++++++++ utils/generate_injected.js | 5 + utils/generate_types/overrides-test.d.ts | 2 +- 24 files changed, 648 insertions(+), 7 deletions(-) create mode 100644 packages/injected/src/coverageScript.ts create mode 100644 packages/isomorphic/istanbulCoverage.ts create mode 100644 packages/playwright-core/src/server/coverageRecorder.ts create mode 100644 tests/library/coverage.spec.ts create mode 100644 tests/playwright-test/coverage.spec.ts diff --git a/docs/src/api/class-tracing.md b/docs/src/api/class-tracing.md index 3d46033821be1..ee4e653e48f55 100644 --- a/docs/src/api/class-tracing.md +++ b/docs/src/api/class-tracing.md @@ -146,6 +146,13 @@ To specify the final trace zip file name, you need to pass `path` option to Whether to capture screenshots during tracing. Screenshots are used to build a timeline preview. +### option: Tracing.start.coverage +* since: v1.64 +* langs: js +- `coverage` <[boolean]> + +**Experimental.** Whether to collect code coverage from istanbul-instrumented application code. Build the application with an istanbul instrumentation plugin, for example [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right before navigations, and stores them as `coverage.json` in istanbul format inside the trace file. Use [`method: Tracing.flushCoverage`] to collect counters from a page that is about to be closed by in-page script. + ### option: Tracing.start.snapshots * since: v1.12 * langs: js @@ -479,6 +486,14 @@ Specifies a custom location for the group to be shown in the trace viewer. Defau Closes the last group created by [`method: Tracing.group`]. +## async method: Tracing.flushCoverage +* since: v1.64 +* langs: js + +**Experimental.** Collects istanbul coverage counters accumulated by all pages of the browser context into the coverage recorded by the trace. Requires tracing with the [`option: Tracing.start.coverage`] option to be started first. + +Coverage is collected automatically when the trace chunk is stopped and when a page is closed with [`method: Page.close`]. Call this method before a page is closed by in-page script, for example a popup closing itself, so that its counters are not lost. + ## async method: Tracing.stop * since: v1.12 diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.md index 5777731f99a99..82cc51f6089ef 100644 --- a/docs/src/test-api/class-testoptions.md +++ b/docs/src/test-api/class-testoptions.md @@ -673,6 +673,7 @@ export default defineConfig({ - `dom` ?<[boolean]> Capture DOM snapshot on every action and record network activity. Optional. - `aria` ?<[boolean]> Capture aria snapshot of the page on every action. Optional. - `screen` ?<[boolean]> Capture a screenshot of the page on every action. Optional. + - `coverage` ?<[boolean]> **Experimental.** Whether to collect coverage from istanbul-instrumented application code into the trace. Defaults to false. Optional. - `sources` ?<[boolean]> Whether to include source files for trace actions. Defaults to true. Optional. Whether to record trace for each test. Defaults to `'off'`. The initial run of a test is the "first run"; subsequent runs caused by [retries](../test-retries.md) are "retries". diff --git a/packages/injected/src/coverageScript.ts b/packages/injected/src/coverageScript.ts new file mode 100644 index 0000000000000..3bcd3578071b3 --- /dev/null +++ b/packages/injected/src/coverageScript.ts @@ -0,0 +1,87 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { IstanbulCoverage } from '@isomorphic/istanbulCoverage'; + +const kBacklogKey = '__pwCoverageBacklog'; + +// Harvests istanbul counters accumulated by instrumented application code in +// `__coverage__`. Every flush serializes the counters and resets them, so +// each flush reports only the delta since the previous one. +export class CoverageScript { + private _global: typeof globalThis; + + constructor(global: typeof globalThis, bindingName: string, collectName: string) { + this._global = global; + (global as any)[collectName] = () => this.collect(); + // Counters die with the document, and binding calls made during unload + // are not delivered, so park the delta in sessionStorage for the next + // same-origin document or the end-of-test sweep to pick up. + global.addEventListener('pagehide', () => this._parkCurrent()); + // Relay deltas parked by previous documents, now that delivery is safe. + for (const json of this._takeBacklog()) + (global as any)[bindingName](json).catch(() => {}); + } + + collect(): string[] { + const chunks = this._takeBacklog(); + const current = this._takeCurrent(); + if (current) + chunks.push(current); + return chunks; + } + + private _takeCurrent(): string | undefined { + const coverage: IstanbulCoverage | undefined = (this._global as any).__coverage__; + if (!coverage) + return undefined; + const json = JSON.stringify(coverage); + for (const file of Object.values(coverage)) { + for (const key of Object.keys(file.s)) + file.s[key] = 0; + for (const key of Object.keys(file.f)) + file.f[key] = 0; + for (const key of Object.keys(file.b)) + file.b[key] = file.b[key].map(() => 0); + } + return json; + } + + private _takeBacklog(): string[] { + try { + const backlog = this._global.sessionStorage.getItem(kBacklogKey); + if (backlog) { + this._global.sessionStorage.removeItem(kBacklogKey); + return JSON.parse(backlog); + } + } catch { + } + return []; + } + + private _parkCurrent() { + const json = this._takeCurrent(); + if (!json) + return; + try { + const existing = this._global.sessionStorage.getItem(kBacklogKey); + const backlog = existing ? JSON.parse(existing) : []; + backlog.push(json); + this._global.sessionStorage.setItem(kBacklogKey, JSON.stringify(backlog)); + } catch { + } + } +} diff --git a/packages/isomorphic/index.ts b/packages/isomorphic/index.ts index bcd969c0ff4a1..146759e178416 100644 --- a/packages/isomorphic/index.ts +++ b/packages/isomorphic/index.ts @@ -21,6 +21,7 @@ export * from './base64'; export * from './colors'; export * from './headers'; export * from './imageUtils'; +export * from './istanbulCoverage'; export * from './jsonSchema'; export * from './locatorGenerators'; export * from './manualPromise'; diff --git a/packages/isomorphic/istanbulCoverage.ts b/packages/isomorphic/istanbulCoverage.ts new file mode 100644 index 0000000000000..1b146fb7c48e9 --- /dev/null +++ b/packages/isomorphic/istanbulCoverage.ts @@ -0,0 +1,76 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// The istanbul coverage format, as produced by istanbul-instrumented code in +// the `__coverage__` global and consumed by istanbul report tooling. +// Mirrors the shapes from istanbul-lib-coverage. + +export type IstanbulLocation = { + line: number; + column: number; +}; + +export type IstanbulRange = { + start: IstanbulLocation; + end: IstanbulLocation; +}; + +export type IstanbulFunctionMapping = { + name: string; + decl: IstanbulRange; + loc: IstanbulRange; + line?: number; +}; + +export type IstanbulBranchMapping = { + type: string; + loc: IstanbulRange; + locations: IstanbulRange[]; + line?: number; +}; + +export type IstanbulFileCoverage = { + path: string; + statementMap: { [key: string]: IstanbulRange }; + fnMap: { [key: string]: IstanbulFunctionMapping }; + branchMap: { [key: string]: IstanbulBranchMapping }; + // Hit counts, keyed by the corresponding map entries above. + s: { [key: string]: number }; + f: { [key: string]: number }; + b: { [key: string]: number[] }; +}; + +export type IstanbulCoverage = { [file: string]: IstanbulFileCoverage }; + +// Merges istanbul hit counts: counters add up, map entries are taken from the +// first occurrence of each file. Assumes all data comes from the same build. +export function mergeIstanbulCoverage(into: Map, data: IstanbulCoverage) { + for (const [file, fileCov] of Object.entries(data)) { + const existing = into.get(file); + if (!existing) { + into.set(file, fileCov); + continue; + } + for (const key of Object.keys(fileCov.s)) + existing.s[key] = (existing.s[key] || 0) + fileCov.s[key]; + for (const key of Object.keys(fileCov.f)) + existing.f[key] = (existing.f[key] || 0) + fileCov.f[key]; + for (const key of Object.keys(fileCov.b)) { + const branches = existing.b[key] || (existing.b[key] = []); + fileCov.b[key].forEach((count, i) => branches[i] = (branches[i] || 0) + count); + } + } +} diff --git a/packages/isomorphic/protocolMetainfo.ts b/packages/isomorphic/protocolMetainfo.ts index 102877079ee7d..37211d55e7d6b 100644 --- a/packages/isomorphic/protocolMetainfo.ts +++ b/packages/isomorphic/protocolMetainfo.ts @@ -338,6 +338,7 @@ export const methodMetainfo = new Map([ ['Tracing.tracingStartChunk', { title: 'Start tracing', group: 'configuration', }], ['Tracing.tracingGroup', { title: 'Trace "{name}"', renderParams: ['name'], }], ['Tracing.tracingGroupEnd', { title: 'Group end', }], + ['Tracing.tracingFlushCoverage', { title: 'Flush coverage', group: 'configuration', }], ['Tracing.tracingStopChunk', { title: 'Stop tracing', group: 'configuration', }], ['Tracing.tracingStop', { title: 'Stop tracing', group: 'configuration', }], ['Tracing.harStart', { internal: true, }], diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts index be0790ba1a826..fcfe492ef689f 100644 --- a/packages/playwright-client/types/types.d.ts +++ b/packages/playwright-client/types/types.d.ts @@ -23204,6 +23204,18 @@ export interface Touchscreen { * */ export interface Tracing { + /** + * **Experimental.** Collects istanbul coverage counters accumulated by all pages of the browser context into the + * coverage recorded by the trace. Requires tracing with the + * [`coverage`](https://playwright.dev/docs/api/class-tracing#tracing-start-option-coverage) option to be started + * first. + * + * Coverage is collected automatically when the trace chunk is stopped and when a page is closed with + * [page.close([options])](https://playwright.dev/docs/api/class-page#page-close). Call this method before a page is + * closed by in-page script, for example a popup closing itself, so that its counters are not lost. + */ + flushCoverage(): Promise; + /** * **NOTE** Use `test.step` instead when available. * @@ -23268,6 +23280,18 @@ export interface Tracing { * @param options */ start(options?: { + /** + * **Experimental.** Whether to collect code coverage from istanbul-instrumented application code. Build the + * application with an istanbul instrumentation plugin, for example + * [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or + * [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the + * `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right + * before navigations, and stores them as `coverage.json` in istanbul format inside the trace file. Use + * [tracing.flushCoverage()](https://playwright.dev/docs/api/class-tracing#tracing-flush-coverage) to collect counters + * from a page that is about to be closed by in-page script. + */ + coverage?: boolean; + /** * When enabled, the trace is written to an unarchived file that is updated in real time as actions occur, instead of * caching changes and archiving them into a zip file at the end. This is useful for live trace viewing during test diff --git a/packages/playwright-core/src/client/channels.d.ts b/packages/playwright-core/src/client/channels.d.ts index a645cd61a81e3..40da002b32d2c 100644 --- a/packages/playwright-core/src/client/channels.d.ts +++ b/packages/playwright-core/src/client/channels.d.ts @@ -5064,6 +5064,7 @@ export interface TracingChannel extends TracingEventTarget, Channel { tracingStartChunk(params: TracingTracingStartChunkParams, options: TimeoutOptions): Promise; tracingGroup(params: TracingTracingGroupParams, options: TimeoutOptions): Promise; tracingGroupEnd(params: TracingTracingGroupEndParams, options: TimeoutOptions): Promise; + tracingFlushCoverage(params: TracingTracingFlushCoverageParams, options: TimeoutOptions): Promise; tracingStopChunk(params: TracingTracingStopChunkParams, options: TimeoutOptions): Promise; tracingStop(params: TracingTracingStopParams, options: TimeoutOptions): Promise; harStart(params: TracingHarStartParams, options: TimeoutOptions): Promise; @@ -5075,6 +5076,7 @@ export type TracingTracingStartParams = { snapshotAria?: boolean, snapshotScreen?: boolean, screencast?: boolean, + coverage?: boolean, live?: boolean, }; export type TracingTracingStartOptions = { @@ -5083,6 +5085,7 @@ export type TracingTracingStartOptions = { snapshotAria?: boolean, snapshotScreen?: boolean, screencast?: boolean, + coverage?: boolean, live?: boolean, }; export type TracingTracingStartResult = void; @@ -5116,6 +5119,9 @@ export type TracingTracingGroupResult = void; export type TracingTracingGroupEndParams = {}; export type TracingTracingGroupEndOptions = {}; export type TracingTracingGroupEndResult = void; +export type TracingTracingFlushCoverageParams = {}; +export type TracingTracingFlushCoverageOptions = {}; +export type TracingTracingFlushCoverageResult = void; export type TracingTracingStopChunkParams = { mode: 'archive' | 'discard' | 'entries', }; diff --git a/packages/playwright-core/src/client/tracing.ts b/packages/playwright-core/src/client/tracing.ts index 4f403f5679a1c..679be72ac027f 100644 --- a/packages/playwright-core/src/client/tracing.ts +++ b/packages/playwright-core/src/client/tracing.ts @@ -42,7 +42,7 @@ export class Tracing extends ChannelOwner implements ap super(parent, type, guid, initializer); } - async start(options: { name?: string, title?: string, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, sources?: boolean, live?: boolean } = {}) { + async start(options: { name?: string, title?: string, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, coverage?: boolean, sources?: boolean, live?: boolean } = {}) { await this._wrapApiCall(async () => { this._includeSources = !!options.sources; this._isLive = !!options.live; @@ -53,6 +53,7 @@ export class Tracing extends ChannelOwner implements ap snapshotAria: snapshots.aria, snapshotScreen: snapshots.screen, screencast: options.screenshots, + coverage: options.coverage, live: options.live, }, kNoTimeout); const { traceName } = await this._channel.tracingStartChunk({ name: options.name, title: options.title }, kNoTimeout); @@ -78,6 +79,12 @@ export class Tracing extends ChannelOwner implements ap await this._channel.tracingGroupEnd({}, kNoTimeout); } + async flushCoverage() { + await this._wrapApiCall(async () => { + await this._channel.tracingFlushCoverage({}, kNoTimeout); + }); + } + private async _startCollectingStacks(traceName: string, live: boolean) { if (!this._isTracing) { this._isTracing = true; diff --git a/packages/playwright-core/src/server/channels.d.ts b/packages/playwright-core/src/server/channels.d.ts index f2fc853c42368..7655960f56a63 100644 --- a/packages/playwright-core/src/server/channels.d.ts +++ b/packages/playwright-core/src/server/channels.d.ts @@ -5065,6 +5065,7 @@ export interface TracingChannel extends TracingEventTarget, Channel { tracingStartChunk(params: TracingTracingStartChunkParams, progress: Progress): Promise; tracingGroup(params: TracingTracingGroupParams, progress: Progress): Promise; tracingGroupEnd(params: TracingTracingGroupEndParams, progress: Progress): Promise; + tracingFlushCoverage(params: TracingTracingFlushCoverageParams, progress: Progress): Promise; tracingStopChunk(params: TracingTracingStopChunkParams, progress: Progress): Promise; tracingStop(params: TracingTracingStopParams, progress: Progress): Promise; harStart(params: TracingHarStartParams, progress: Progress): Promise; @@ -5076,6 +5077,7 @@ export type TracingTracingStartParams = { snapshotAria?: boolean, snapshotScreen?: boolean, screencast?: boolean, + coverage?: boolean, live?: boolean, }; export type TracingTracingStartOptions = { @@ -5084,6 +5086,7 @@ export type TracingTracingStartOptions = { snapshotAria?: boolean, snapshotScreen?: boolean, screencast?: boolean, + coverage?: boolean, live?: boolean, }; export type TracingTracingStartResult = void; @@ -5117,6 +5120,9 @@ export type TracingTracingGroupResult = void; export type TracingTracingGroupEndParams = {}; export type TracingTracingGroupEndOptions = {}; export type TracingTracingGroupEndResult = void; +export type TracingTracingFlushCoverageParams = {}; +export type TracingTracingFlushCoverageOptions = {}; +export type TracingTracingFlushCoverageResult = void; export type TracingTracingStopChunkParams = { mode: 'archive' | 'discard' | 'entries', }; diff --git a/packages/playwright-core/src/server/coverageRecorder.ts b/packages/playwright-core/src/server/coverageRecorder.ts new file mode 100644 index 0000000000000..80b61d3611abb --- /dev/null +++ b/packages/playwright-core/src/server/coverageRecorder.ts @@ -0,0 +1,97 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { mergeIstanbulCoverage } from '@isomorphic/istanbulCoverage'; +import * as rawCoverageSource from '../generated/coverageScriptSource'; + +import type { BrowserContext } from './browserContext'; +import type { Page } from './page'; +import type { Progress } from './progress'; +import type { IstanbulFileCoverage } from '@isomorphic/istanbulCoverage'; + +const kCoverageBindingName = '__pwCoverageSink'; +const kCoverageCollectName = '__pwCoverageCollect'; + +const coverageBootstrapSource = `(() => { + const module = {}; + ${rawCoverageSource.source} + new (module.exports.CoverageScript())(window, ${JSON.stringify(kCoverageBindingName)}, ${JSON.stringify(kCoverageCollectName)}); +})()`; + +const coverageCollectExpression = `window[${JSON.stringify(kCoverageCollectName)}] ? window[${JSON.stringify(kCoverageCollectName)}]() : []`; + +export class CoverageRecorder { + private _context: BrowserContext; + private _coverage = new Map(); + private _installed = false; + private _active = false; + + constructor(context: BrowserContext) { + this._context = context; + } + + activate() { + this._active = true; + } + + deactivate() { + this._active = false; + } + + active() { + return this._active; + } + + async install(progress: Progress) { + if (this._installed) + return; + this._installed = true; + await this._context.exposeBinding(progress, kCoverageBindingName, (source, json: string) => this._append(json)); + await this._context.addInitScript(progress, coverageBootstrapSource); + // Init scripts only affect future documents, bootstrap the existing ones. + await progress.race(this._context.safeNonStallingEvaluateInAllFrames(coverageBootstrapSource, 'main')); + } + + async flush() { + for (const page of this._context.pages()) + await this.collectFromPage(page); + } + + // Collects pending coverage from all pages and returns it serialized, resetting the accumulator. + async take(): Promise { + if (!this._active) + return; + await this.flush(); + const sorted = Object.fromEntries([...this._coverage.entries()].sort(([a], [b]) => a.localeCompare(b))); + this._coverage.clear(); + return JSON.stringify(sorted); + } + + async collectFromPage(page: Page) { + for (const frame of page.frames()) { + const chunks: string[] = await frame.nonStallingEvaluateInExistingContext(coverageCollectExpression, 'main').catch(() => []); + for (const json of chunks || []) + this._append(json); + } + } + + private _append(json: string) { + try { + mergeIstanbulCoverage(this._coverage, JSON.parse(json)); + } catch { + } + } +} diff --git a/packages/playwright-core/src/server/dispatchers/tracingDispatcher.ts b/packages/playwright-core/src/server/dispatchers/tracingDispatcher.ts index 950ab8401bb89..fc3c0d990d88c 100644 --- a/packages/playwright-core/src/server/dispatchers/tracingDispatcher.ts +++ b/packages/playwright-core/src/server/dispatchers/tracingDispatcher.ts @@ -56,6 +56,10 @@ export class TracingDispatcher extends Dispatcher { + await this._object.flushCoverage(progress); + } + async tracingStopChunk(params: channels.TracingTracingStopChunkParams, progress: Progress): Promise { const { artifact, entries } = await this._object.stopChunk(progress, params); return { artifact: artifact ? ArtifactDispatcher.from(this, artifact) : undefined, entries }; diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts index 00ee7449d2fd0..bebb3aa54bb44 100644 --- a/packages/playwright-core/src/server/page.ts +++ b/packages/playwright-core/src/server/page.ts @@ -821,6 +821,7 @@ export class Page extends SdkObject { } async close(progress: Progress, options: { reason?: string } = {}) { + await progress.race(this.browserContext.tracing.flushCoverageBeforePageClose(this).catch(() => {})); await progress.race(this._close(options)); } diff --git a/packages/playwright-core/src/server/trace/recorder/tracing.ts b/packages/playwright-core/src/server/trace/recorder/tracing.ts index 3ba35b8348979..d4dcc453a7050 100644 --- a/packages/playwright-core/src/server/trace/recorder/tracing.ts +++ b/packages/playwright-core/src/server/trace/recorder/tracing.ts @@ -30,6 +30,7 @@ import { getPlaywrightVersion } from '../../userAgent'; import { Snapshotter } from './snapshotter'; import { Artifact } from '../../artifact'; import { BrowserContext } from '../../browserContext'; +import { CoverageRecorder } from '../../coverageRecorder'; import { Dispatcher } from '../../dispatchers/dispatcher'; import { serializeError } from '../../errors'; import { HarRecorder } from '../../har/harRecorder'; @@ -63,6 +64,7 @@ export type TracerOptions = { snapshotAria?: boolean; snapshotScreen?: boolean; screencast?: boolean; + coverage?: boolean; live?: boolean; }; @@ -100,6 +102,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps private _contextCreatedEvent: trace.ContextCreatedTraceEvent; private _pendingHarEntries = new Set(); private _started = false; + private _coverageRecorder: CoverageRecorder | undefined; readonly harRecorders = new Map(); constructor(context: BrowserContext | APIRequestContext, tracesDir: string | undefined) { @@ -184,6 +187,10 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps // Tracing is 10x bigger if we include scripts in every trace. if (options.snapshotDom) this._harTracer.start({ omitScripts: !options.live }); + if (options.coverage && this._context instanceof BrowserContext) { + this._coverageRecorder ??= new CoverageRecorder(this._context); + this._coverageRecorder.activate(); + } this._started = true; } @@ -235,6 +242,8 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps this._harTracer.setOmitWebSocketFrames(!!process.env.PLAYWRIGHT_TRACING_NO_WEBSOCKET_FRAMES); if (this._state.options.snapshotDom) await this._snapshotter?.start(progress); + if (this._state.options.coverage) + await this._coverageRecorder?.install(progress); return { traceName: this._state.traceName }; } @@ -339,6 +348,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps this._closeAllGroups(); this._harTracer.stop(); this.flushHarEntries(); + this._coverageRecorder?.deactivate(); await this._fs.sync().finally(() => { this._state = undefined; }); @@ -387,12 +397,39 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps this._groupEnd(); } + async flushCoverage(progress: Progress) { + if (!this._coverageRecorder?.active()) + throw new Error(`Coverage collection has not been started. Pass the "coverage" option to tracing.start().`); + await progress.race(this._coverageRecorder.flush()); + } + + // Best-effort collection before the page is gone, no-op unless collecting. + async flushCoverageBeforePageClose(page: Page) { + if (!this._coverageRecorder?.active()) + return; + await this._coverageRecorder.collectFromPage(page); + } + + // Collects pending coverage into a file that will become the "coverage.json" + // entry of the chunk, while the pages can still be evaluated in. + private async _takeCoverage(progress: Progress, mode: TracingTracingStopChunkParams['mode']): Promise { + if (!this._state?.recording || !this._state.options.coverage || !this._coverageRecorder) + return; + const coverageJson = await progress.race(this._coverageRecorder.take()); + if (mode === 'discard' || coverageJson === undefined) + return; + const coverageFile = path.join(this._state.tracesDir, `${this._state.traceName}-coverage-${this._state.chunkOrdinal}.json`); + this._fs.writeFile(coverageFile, coverageJson); + return coverageFile; + } + async stopChunk(progress: Progress, params: TracingTracingStopChunkParams): Promise<{ artifact?: Artifact, entries?: NameValue[] }> { if (this._isStopping) throw new Error(`Tracing is already stopping`); this._isStopping = true; try { - const result = this._stopChunk(params); + const coverageFile = await this._takeCoverage(progress, params.mode); + const result = this._stopChunk(params, coverageFile); if (!result) return {}; @@ -421,7 +458,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps } } - private _stopChunk(params: TracingTracingStopChunkParams): { entries: NameValue[], zipFileName: string } | undefined { + private _stopChunk(params: TracingTracingStopChunkParams, coverageFile: string | undefined): { entries: NameValue[], zipFileName: string } | undefined { if (!this._state || !this._state.recording) { if (params.mode !== 'discard') throw new Error(`Must start tracing before stopping`); @@ -453,6 +490,8 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps const entries: NameValue[] = []; entries.push({ name: 'trace.trace', value: this._state.traceFile }); entries.push({ name: 'trace.network', value: newNetworkFile }); + if (coverageFile) + entries.push({ name: 'coverage.json', value: coverageFile }); for (const file of new Set([...this._state.chunkFiles, ...this._state.crossChunkFiles])) entries.push({ name: file, value: path.join(this._state.tracesDir, file) }); diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index be0790ba1a826..fcfe492ef689f 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -23204,6 +23204,18 @@ export interface Touchscreen { * */ export interface Tracing { + /** + * **Experimental.** Collects istanbul coverage counters accumulated by all pages of the browser context into the + * coverage recorded by the trace. Requires tracing with the + * [`coverage`](https://playwright.dev/docs/api/class-tracing#tracing-start-option-coverage) option to be started + * first. + * + * Coverage is collected automatically when the trace chunk is stopped and when a page is closed with + * [page.close([options])](https://playwright.dev/docs/api/class-page#page-close). Call this method before a page is + * closed by in-page script, for example a popup closing itself, so that its counters are not lost. + */ + flushCoverage(): Promise; + /** * **NOTE** Use `test.step` instead when available. * @@ -23268,6 +23280,18 @@ export interface Tracing { * @param options */ start(options?: { + /** + * **Experimental.** Whether to collect code coverage from istanbul-instrumented application code. Build the + * application with an istanbul instrumentation plugin, for example + * [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or + * [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the + * `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right + * before navigations, and stores them as `coverage.json` in istanbul format inside the trace file. Use + * [tracing.flushCoverage()](https://playwright.dev/docs/api/class-tracing#tracing-flush-coverage) to collect counters + * from a page that is about to be closed by in-page script. + */ + coverage?: boolean; + /** * When enabled, the trace is written to an unarchived file that is updated in real time as actions occur, instead of * caching changes and archiving them into a zip file at the end. This is useful for live trace viewing during test diff --git a/packages/playwright/src/worker/testTracing.ts b/packages/playwright/src/worker/testTracing.ts index fed713a572507..8b8637544eece 100644 --- a/packages/playwright/src/worker/testTracing.ts +++ b/packages/playwright/src/worker/testTracing.ts @@ -38,7 +38,7 @@ const version: trace.VERSION = 10; let traceOrdinal = 0; type TraceFixtureValue = PlaywrightWorkerOptions['trace'] | undefined; -type TraceOptions = { screenshots: boolean, snapshots: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, sources: boolean, attachments: boolean, live: boolean, mode: TraceMode }; +type TraceOptions = { screenshots: boolean, snapshots: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, coverage: boolean, sources: boolean, attachments: boolean, live: boolean, mode: TraceMode }; export class TestTracing { private _testInfo: TestInfoImpl; @@ -93,7 +93,7 @@ export class TestTracing { } async startIfNeeded(value: TraceFixtureValue) { - const defaultTraceOptions: TraceOptions = { screenshots: true, snapshots: true, sources: true, attachments: true, live: false, mode: 'off' }; + const defaultTraceOptions: TraceOptions = { screenshots: true, snapshots: true, coverage: false, sources: true, attachments: true, live: false, mode: 'off' }; if (!value) { this._options = defaultTraceOptions; diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 8d0eef73cc056..c05d1bdf72185 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -7202,7 +7202,7 @@ export interface PlaywrightWorkerOptions { * * Learn more about [recording trace](https://playwright.dev/docs/test-use-options#recording-options). */ - trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, sources?: boolean, attachments?: boolean }; + trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, coverage?: boolean, sources?: boolean, attachments?: boolean }; /** * Whether to record video for each test. Defaults to `'off'`. The initial run of a test is the "first run"; * subsequent runs caused by [retries](https://playwright.dev/docs/test-retries) are "retries". diff --git a/packages/protocol/spec/tracing.yml b/packages/protocol/spec/tracing.yml index de9a644d9d287..ed9bafb812174 100644 --- a/packages/protocol/spec/tracing.yml +++ b/packages/protocol/spec/tracing.yml @@ -26,6 +26,7 @@ Tracing: snapshotAria: boolean? snapshotScreen: boolean? screencast: boolean? + coverage: boolean? live: boolean? tracingStartChunk: @@ -53,6 +54,10 @@ Tracing: tracingGroupEnd: title: Group end + tracingFlushCoverage: + title: Flush coverage + group: configuration + tracingStopChunk: title: Stop tracing group: configuration diff --git a/packages/protocol/src/validator.ts b/packages/protocol/src/validator.ts index 21e7cbe6036e3..0cf8a41e9bd03 100644 --- a/packages/protocol/src/validator.ts +++ b/packages/protocol/src/validator.ts @@ -3065,6 +3065,7 @@ scheme.TracingTracingStartParams = tObject({ snapshotAria: tOptional(tBoolean), snapshotScreen: tOptional(tBoolean), screencast: tOptional(tBoolean), + coverage: tOptional(tBoolean), live: tOptional(tBoolean), }); scheme.TracingTracingStartResult = tOptional(tObject({})); @@ -3086,6 +3087,8 @@ scheme.TracingTracingGroupParams = tObject({ scheme.TracingTracingGroupResult = tOptional(tObject({})); scheme.TracingTracingGroupEndParams = tOptional(tObject({})); scheme.TracingTracingGroupEndResult = tOptional(tObject({})); +scheme.TracingTracingFlushCoverageParams = tOptional(tObject({})); +scheme.TracingTracingFlushCoverageResult = tOptional(tObject({})); scheme.TracingTracingStopChunkParams = tObject({ mode: tEnum(['archive', 'discard', 'entries']), }); diff --git a/tests/library/browsertype-connect.spec.ts b/tests/library/browsertype-connect.spec.ts index 269745b8d1b5a..4db584f5187ff 100644 --- a/tests/library/browsertype-connect.spec.ts +++ b/tests/library/browsertype-connect.spec.ts @@ -741,6 +741,26 @@ for (const kind of ['launchServer', 'run-server'] as const) { expect(sourceFile).toEqual(thisFile); }); + test('should record trace with coverage', async ({ connect, startRemoteServer, trace }, testInfo) => { + test.skip(trace === 'on'); + const remoteServer = await startRemoteServer(kind); + const browser = await connect(remoteServer.wsEndpoint()); + const context = await browser.newContext(); + const page = await context.newPage(); + + await context.tracing.start({ coverage: true }); + const fileCov = { path: 'a.js', statementMap: { '0': { start: { line: 1, column: 0 }, end: { line: 1, column: 20 } } }, fnMap: {}, branchMap: {}, s: { '0': 3 }, f: {}, b: {} }; + await page.setContent(``); + await context.tracing.stop({ path: testInfo.outputPath('trace1.zip') }); + + await context.close(); + await browser.close(); + + const { resources } = await parseTraceRaw(testInfo.outputPath('trace1.zip')); + const data = JSON.parse(resources.get('coverage.json')!.toString()); + expect(data['a.js'].s['0']).toBe(3); + }); + test('should fulfill with global fetch result', async ({ connect, startRemoteServer, playwright, server }) => { const remoteServer = await startRemoteServer(kind); const browser = await connect(remoteServer.wsEndpoint()); diff --git a/tests/library/coverage.spec.ts b/tests/library/coverage.spec.ts new file mode 100644 index 0000000000000..76a23b407efd8 --- /dev/null +++ b/tests/library/coverage.spec.ts @@ -0,0 +1,92 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { browserTest as it, expect } from '../config/browserTest'; +import { parseTraceRaw } from '../config/utils'; + +const coverageScript = (file: string, s0: number) => { + const fileCov = { + path: file, + statementMap: { '0': { start: { line: 1, column: 0 }, end: { line: 1, column: 20 } } }, + fnMap: {}, + branchMap: {}, + s: { '0': s0 }, + f: {}, + b: {}, + }; + return ``; +}; + +async function readCoverage(traceFile: string): Promise { + const { resources } = await parseTraceRaw(traceFile); + const entry = resources.get('coverage.json'); + return entry ? JSON.parse(entry.toString()) : undefined; +} + +it('should collect istanbul coverage into the trace', async ({ browser }, testInfo) => { + const context = await browser.newContext(); + await context.tracing.start({ coverage: true }); + + const page = await context.newPage(); + await page.setContent(coverageScript('a.js', 3)); + await context.tracing.flushCoverage(); + + // Coverage of a closed page is collected automatically. + const page2 = await context.newPage(); + await page2.setContent(coverageScript('b.js', 2)); + await page2.close(); + + const traceFile = testInfo.outputPath('trace.zip'); + await context.tracing.stop({ path: traceFile }); + await context.close(); + + const data = await readCoverage(traceFile); + expect(data['a.js'].s['0']).toBe(3); + expect(data['b.js'].s['0']).toBe(2); +}); + +it('should collect coverage per trace chunk', async ({ browser }, testInfo) => { + const context = await browser.newContext(); + await context.tracing.start({ coverage: true }); + const page = await context.newPage(); + + await page.setContent(coverageScript('a.js', 5)); + const traceFile1 = testInfo.outputPath('trace1.zip'); + await context.tracing.stopChunk({ path: traceFile1 }); + + await context.tracing.startChunk(); + await page.setContent(coverageScript('b.js', 7)); + const traceFile2 = testInfo.outputPath('trace2.zip'); + await context.tracing.stopChunk({ path: traceFile2 }); + + await context.tracing.stop(); + await context.close(); + + const data1 = await readCoverage(traceFile1); + expect(data1['a.js'].s['0']).toBe(5); + expect(data1['b.js']).toBe(undefined); + + const data2 = await readCoverage(traceFile2); + expect(data2['b.js'].s['0']).toBe(7); + expect(data2['a.js']).toBe(undefined); +}); + +it('should throw when flushing without coverage', async ({ browser }) => { + const context = await browser.newContext(); + await context.newPage(); + await expect(context.tracing.flushCoverage()).rejects.toThrow('Coverage collection has not been started'); + await context.close(); +}); diff --git a/tests/playwright-test/coverage.spec.ts b/tests/playwright-test/coverage.spec.ts new file mode 100644 index 0000000000000..8fd9c64c016fa --- /dev/null +++ b/tests/playwright-test/coverage.spec.ts @@ -0,0 +1,127 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect } from './playwright-test-fixtures'; +import { parseTraceRaw } from '../config/utils'; + +const coverageHelper = ` + const makeCoverage = (file, s0) => ({ + [file]: { + path: file, + statementMap: { + '0': { start: { line: 1, column: 0 }, end: { line: 1, column: 20 } }, + '1': { start: { line: 2, column: 0 }, end: { line: 2, column: 20 } }, + }, + fnMap: { + '0': { name: 'foo', decl: { start: { line: 1, column: 0 }, end: { line: 1, column: 3 } }, loc: { start: { line: 1, column: 0 }, end: { line: 3, column: 1 } }, line: 1 }, + }, + branchMap: { + '0': { loc: { start: { line: 2, column: 0 }, end: { line: 2, column: 20 } }, type: 'if', locations: [{ start: { line: 2, column: 0 }, end: { line: 2, column: 10 } }, { start: { line: 2, column: 10 }, end: { line: 2, column: 20 } }], line: 2 }, + }, + s: { '0': s0, '1': 0 }, + f: { '0': 1 }, + b: { '0': [1, 0] }, + }, + }); + const coverageScript = (file, s0) => ''; +`; + +async function readCoverage(tracePath: string): Promise { + const { resources } = await parseTraceRaw(tracePath); + const entry = resources.get('coverage.json'); + return entry ? JSON.parse(entry.toString()) : undefined; +} + +test('should collect istanbul coverage into the trace', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + use: { trace: { mode: 'on', coverage: true } }, + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + ${coverageHelper} + test('pass', async ({ page }) => { + const http = require('http'); + const server = http.createServer((req, res) => { + res.setHeader('Content-Type', 'text/html'); + res.end(coverageScript('src/app.js', req.url === '/page1' ? 5 : 2)); + }); + await new Promise(f => server.listen(0, '127.0.0.1', f)); + const origin = 'http://127.0.0.1:' + server.address().port; + await page.goto(origin + '/page1'); + // Navigating away parks the first document's counters for the next one. + await page.goto(origin + '/page2'); + server.close(); + }); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + + const data = await readCoverage(test.info().outputPath('test-results', 'a-pass', 'trace.zip')); + expect(data['src/app.js'].s['0']).toBe(7); + expect(data['src/app.js'].s['1']).toBe(0); + expect(data['src/app.js'].f['0']).toBe(2); + expect(data['src/app.js'].b['0']).toEqual([2, 0]); +}); + +test('should collect coverage from frames', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + use: { trace: { mode: 'on', coverage: true } }, + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + ${coverageHelper} + test('pass', async ({ page }) => { + await page.setContent(coverageScript('src/app.js', 1) + ''); + await expect(page.locator('iframe')).toBeVisible(); + }); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + + const data = await readCoverage(test.info().outputPath('test-results', 'a-pass', 'trace.zip')); + expect(data['src/app.js'].s['0']).toBe(1); + expect(data['src/frame.js'].s['0']).toBe(3); +}); + +test('should not collect coverage without the option', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + use: { trace: 'on' }, + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + ${coverageHelper} + test('pass', async ({ page }) => { + await page.setContent(coverageScript('src/app.js', 1)); + }); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + + const data = await readCoverage(test.info().outputPath('test-results', 'a-pass', 'trace.zip')); + expect(data).toBe(undefined); +}); diff --git a/utils/generate_injected.js b/utils/generate_injected.js index 4162e312d2e03..86fb7cb39d3a1 100644 --- a/utils/generate_injected.js +++ b/utils/generate_injected.js @@ -81,6 +81,11 @@ const injectedScripts = [ path.join(ROOT, 'packages', 'injected', 'lib'), path.join(ROOT, 'packages', 'playwright-core', 'src', 'generated'), ], + [ + path.join(ROOT, 'packages', 'injected', 'src', 'coverageScript.ts'), + path.join(ROOT, 'packages', 'injected', 'lib'), + path.join(ROOT, 'packages', 'playwright-core', 'src', 'generated'), + ], ]; const modulePrefix = ` diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 90a983fdcea29..bbdc358b903d9 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -272,7 +272,7 @@ export interface PlaywrightWorkerOptions { connectOptions: ConnectOptions | undefined; reuseContext: boolean; screenshot: ScreenshotMode | { mode: ScreenshotMode } & Pick; - trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, sources?: boolean, attachments?: boolean }; + trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, coverage?: boolean, sources?: boolean, attachments?: boolean }; video: VideoMode | /** deprecated */ 'retry-with-video' | { mode: VideoMode, size?: ViewportSize, show?: { actions?: { duration?: number, position?: 'top-left' | 'top' | 'top-right' | 'bottom-left' | 'bottom' | 'bottom-right', fontSize?: number, cursor?: 'none' | 'pointer' }, test?: { level?: 'file' | 'title' | 'step', position?: 'top-left' | 'top' | 'top-right' | 'bottom-left' | 'bottom' | 'bottom-right', fontSize?: number } } }; } From bd6d86957fc40219e12ff67ad7addc1896c828ce Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Thu, 10 Sep 2026 17:58:22 -0700 Subject: [PATCH 02/21] feat(coverage): aggregate coverage from traces with the coverage reporter The 'coverage' reporter collects the coverage.json entries from the per-test trace attachments, merges them and writes coverage-final.json, lcov.info and a console summary, plus the istanbul html report when the project has istanbul-lib-* installed. Aggregation covers the tests whose traces are kept, so a full aggregate requires trace mode 'on'. --- packages/playwright/src/common/config.ts | 2 +- packages/playwright/src/reporters/coverage.ts | 218 ++++++++++++++++++ packages/playwright/src/runner/reporters.ts | 2 + packages/playwright/types/test.d.ts | 2 +- tests/playwright-test/coverage.spec.ts | 62 +++++ utils/generate_types/overrides-test.d.ts | 2 +- 6 files changed, 285 insertions(+), 3 deletions(-) create mode 100644 packages/playwright/src/reporters/coverage.ts diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index e139194f08e5e..a237ccd041251 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -291,7 +291,7 @@ export function toReporters(reporters: BuiltInReporter | ReporterDescription[] | return reporters; } -export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html', 'blob', 'perfetto'] as const; +export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html', 'blob', 'perfetto', 'coverage'] as const; export type BuiltInReporter = typeof builtInReporters[number]; export type ContextReuseMode = 'none' | 'when-possible'; diff --git a/packages/playwright/src/reporters/coverage.ts b/packages/playwright/src/reporters/coverage.ts new file mode 100644 index 0000000000000..80e0d4e4a0843 --- /dev/null +++ b/packages/playwright/src/reporters/coverage.ts @@ -0,0 +1,218 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; +import path from 'path'; + +import { mergeIstanbulCoverage } from '@isomorphic/istanbulCoverage'; +import { ZipFile } from '@utils/zipFile'; +import { resolveReporterOutputPath } from '../util'; + +import type { IstanbulCoverage, IstanbulFileCoverage } from '@isomorphic/istanbulCoverage'; +import type { CommonReporterOptions } from './base'; +import type { ReporterV2 } from './reporterV2'; +import type { TestCase, TestResult } from '../../types/testReporter'; + +type CoverageReporterOptions = { + outputDir?: string, +}; + +class CoverageReporter implements ReporterV2 { + private _options: CoverageReporterOptions & CommonReporterOptions; + private _traceFiles = new Set(); + + constructor(options: CoverageReporterOptions & CommonReporterOptions) { + this._options = options; + } + + version(): 'v2' { + return 'v2'; + } + + printsToStdio() { + return false; + } + + onTestEnd(test: TestCase, result: TestResult) { + for (const attachment of result.attachments) { + if (attachment.name === 'trace' && attachment.path) + this._traceFiles.add(attachment.path); + } + } + + async onEnd() { + const coverage = new Map(); + for (const traceFile of this._traceFiles) { + const zipFile = new ZipFile(traceFile); + try { + const entries = await zipFile.entries(); + if (entries.includes('coverage.json')) + mergeIstanbulCoverage(coverage, JSON.parse((await zipFile.read('coverage.json')).toString('utf8'))); + } catch { + } finally { + zipFile.close(); + } + } + + if (!coverage.size) { + writeLine([ + `\nNo code coverage was collected.`, + `Make sure the application under test is built with istanbul instrumentation`, + `(e.g. vite-plugin-istanbul or babel-plugin-istanbul) so that pages expose window.__coverage__,`, + `and that tracing is configured with coverage, e.g. trace: { mode: 'on', coverage: true }.`, + `Coverage is aggregated from the tests whose traces are kept.`, + ].join('\n')); + return; + } + + const outputDir = resolveReporterOutputPath('coverage', this._options.configDir, this._options.outputDir); + await fs.promises.mkdir(outputDir, { recursive: true }); + const mergedJson: IstanbulCoverage = Object.fromEntries([...coverage.entries()].sort(([a], [b]) => a.localeCompare(b))); + await fs.promises.writeFile(path.join(outputDir, 'coverage-final.json'), JSON.stringify(mergedJson)); + await fs.promises.writeFile(path.join(outputDir, 'lcov.info'), lcovReport(mergedJson)); + const hasHtml = this._tryWriteHtmlReport(mergedJson, outputDir); + + const summary = computeSummary(mergedJson); + const lines = [ + ``, + `Code coverage (${coverage.size} files):`, + ` statements: ${formatMetric(summary.statements)}`, + ` branches: ${formatMetric(summary.branches)}`, + ` functions: ${formatMetric(summary.functions)}`, + ` lines: ${formatMetric(summary.lines)}`, + `Coverage report written to ${path.relative(process.cwd(), outputDir)}`, + ]; + if (!hasHtml) + lines.push(`For an HTML report, install istanbul-lib-coverage, istanbul-lib-report and istanbul-reports, or run 'npx nyc report --temp-dir=${path.relative(process.cwd(), outputDir)} --reporter=html'.`); + writeLine(lines.join('\n')); + } + + // Istanbul report libraries are not shipped with Playwright. Use them for the + // html report when the project has them installed. + private _tryWriteHtmlReport(mergedJson: IstanbulCoverage, outputDir: string): boolean { + try { + const resolveFrom = (name: string) => require.resolve(name, { paths: [this._options.configDir] }); + const libCoverage = require(resolveFrom('istanbul-lib-coverage')); + const libReport = require(resolveFrom('istanbul-lib-report')); + const reports = require(resolveFrom('istanbul-reports')); + const context = libReport.createContext({ + dir: outputDir, + coverageMap: libCoverage.createCoverageMap(mergedJson), + }); + reports.create('html').execute(context); + return true; + } catch { + return false; + } + } +} + +type Metric = { covered: number, total: number }; + +function computeSummary(mergedJson: IstanbulCoverage) { + const statements: Metric = { covered: 0, total: 0 }; + const branches: Metric = { covered: 0, total: 0 }; + const functions: Metric = { covered: 0, total: 0 }; + const lines: Metric = { covered: 0, total: 0 }; + for (const fileCov of Object.values(mergedJson)) { + for (const count of Object.values(fileCov.s)) { + ++statements.total; + if (count > 0) + ++statements.covered; + } + for (const count of Object.values(fileCov.f)) { + ++functions.total; + if (count > 0) + ++functions.covered; + } + for (const counts of Object.values(fileCov.b)) { + for (const count of counts) { + ++branches.total; + if (count > 0) + ++branches.covered; + } + } + for (const count of lineCoverage(fileCov).values()) { + ++lines.total; + if (count > 0) + ++lines.covered; + } + } + return { statements, branches, functions, lines }; +} + +function lineCoverage(fileCov: IstanbulFileCoverage): Map { + const lines = new Map(); + for (const [key, statement] of Object.entries(fileCov.statementMap)) { + const line = statement.start.line; + const count = fileCov.s[key] || 0; + lines.set(line, Math.max(lines.get(line) || 0, count)); + } + return lines; +} + +function formatMetric(metric: Metric) { + const percent = metric.total ? (metric.covered / metric.total * 100).toFixed(2) : '100.00'; + return `${percent}% (${metric.covered}/${metric.total})`; +} + +function lcovReport(mergedJson: IstanbulCoverage): string { + const out: string[] = []; + for (const [file, fileCov] of Object.entries(mergedJson)) { + out.push(`SF:${file}`); + let functionsCovered = 0; + for (const [key, fn] of Object.entries(fileCov.fnMap)) { + out.push(`FN:${fn.decl.start.line},${fn.name}`); + out.push(`FNDA:${fileCov.f[key] || 0},${fn.name}`); + if (fileCov.f[key] > 0) + ++functionsCovered; + } + out.push(`FNF:${Object.keys(fileCov.fnMap).length}`); + out.push(`FNH:${functionsCovered}`); + let branchesFound = 0; + let branchesCovered = 0; + for (const [key, branch] of Object.entries(fileCov.branchMap)) { + const counts: number[] = fileCov.b[key] || []; + branch.locations.forEach((location, i) => { + const count = counts[i] || 0; + out.push(`BRDA:${location.start.line ?? branch.loc.start.line},${key},${i},${count}`); + ++branchesFound; + if (count > 0) + ++branchesCovered; + }); + } + out.push(`BRF:${branchesFound}`); + out.push(`BRH:${branchesCovered}`); + const lines = lineCoverage(fileCov); + let linesCovered = 0; + for (const [line, count] of [...lines.entries()].sort((a, b) => a[0] - b[0])) { + out.push(`DA:${line},${count}`); + if (count > 0) + ++linesCovered; + } + out.push(`LF:${lines.size}`); + out.push(`LH:${linesCovered}`); + out.push(`end_of_record`); + } + return out.join('\n') + '\n'; +} + +function writeLine(line: string) { + // eslint-disable-next-line no-restricted-properties + process.stdout.write(line + '\n'); +} + +export default CoverageReporter; diff --git a/packages/playwright/src/runner/reporters.ts b/packages/playwright/src/runner/reporters.ts index 01966f8b3d993..34cff70b12c59 100644 --- a/packages/playwright/src/runner/reporters.ts +++ b/packages/playwright/src/runner/reporters.ts @@ -19,6 +19,7 @@ import { calculateSha1 } from '@utils/crypto'; import { loadReporter } from './loadUtils'; import { formatError } from '../reporters/base'; import { BlobReporter } from '../reporters/blob'; +import CoverageReporter from '../reporters/coverage'; import DotReporter from '../reporters/dot'; import EmptyReporter from '../reporters/empty'; import GitHubReporter from '../reporters/github'; @@ -50,6 +51,7 @@ export async function createReporters(config: FullConfigInternal, mode: 'list' | 'junit': JUnitReporter, 'null': EmptyReporter, 'html': HtmlReporter, + 'coverage': CoverageReporter, }; const reporters: ReporterV2[] = []; descriptions ??= config.config.reporter; diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index c05d1bdf72185..07effc78c1858 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -919,7 +919,7 @@ interface TestConfig { * ``` * */ - reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html'|'perfetto', string> | ReporterDescription[]; + reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html'|'perfetto'|'coverage', string> | ReporterDescription[]; /** * Global options for all tests, for example * [testOptions.browserName](https://playwright.dev/docs/api/class-testoptions#test-options-browser-name). Learn more diff --git a/tests/playwright-test/coverage.spec.ts b/tests/playwright-test/coverage.spec.ts index 8fd9c64c016fa..6156effe3c3a0 100644 --- a/tests/playwright-test/coverage.spec.ts +++ b/tests/playwright-test/coverage.spec.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import fs from 'fs'; import { test, expect } from './playwright-test-fixtures'; import { parseTraceRaw } from '../config/utils'; @@ -104,6 +105,67 @@ test('should collect coverage from frames', async ({ runInlineTest }) => { expect(data['src/frame.js'].s['0']).toBe(3); }); +test('should aggregate coverage with the coverage reporter', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + use: { trace: { mode: 'on', coverage: true } }, + reporter: [['dot'], ['coverage', { outputDir: 'coverage-report' }]], + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + ${coverageHelper} + test('first', async ({ page }) => { + await page.setContent(coverageScript('src/app.js', 5)); + }); + `, + 'b.test.ts': ` + import { test, expect } from '@playwright/test'; + ${coverageHelper} + test('second', async ({ page }) => { + await page.setContent(coverageScript('src/app.js', 2)); + }); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(2); + + const final = JSON.parse(fs.readFileSync(test.info().outputPath('coverage-report', 'coverage-final.json'), 'utf8')); + expect(final['src/app.js'].s['0']).toBe(7); + expect(final['src/app.js'].s['1']).toBe(0); + expect(final['src/app.js'].f['0']).toBe(2); + expect(final['src/app.js'].b['0']).toEqual([2, 0]); + + const lcov = fs.readFileSync(test.info().outputPath('coverage-report', 'lcov.info'), 'utf8'); + expect(lcov).toContain('SF:src/app.js'); + expect(lcov).toContain('DA:1,7'); + expect(lcov).toContain('FNDA:2,foo'); + + expect(result.output).toContain('Code coverage (1 files)'); + expect(result.output).toContain('statements: 50.00% (1/2)'); +}); + +test('should warn when no coverage was collected', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + use: { trace: { mode: 'on', coverage: true } }, + reporter: [['dot'], ['coverage', { outputDir: 'coverage-report' }]], + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('no instrumented code', async ({ page }) => { + await page.setContent('
hello
'); + }); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.output).toContain('No code coverage was collected'); + expect(fs.existsSync(test.info().outputPath('coverage-report', 'coverage-final.json'))).toBe(false); +}); + test('should not collect coverage without the option', async ({ runInlineTest }) => { const result = await runInlineTest({ 'playwright.config.ts': ` diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index bbdc358b903d9..4a829c80ff689 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -69,7 +69,7 @@ type LiteralUnion = T | (U & { zz_IGNORE_ME?: never }); interface TestConfig { projects?: Project[]; - reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html'|'perfetto', string> | ReporterDescription[]; + reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html'|'perfetto'|'coverage', string> | ReporterDescription[]; use?: UseOptions; webServer?: TestConfigWebServer | TestConfigWebServer[]; } From 1520719eae2053dcadd6d760b468802a1ba846dc Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Fri, 11 Sep 2026 13:05:23 -0700 Subject: [PATCH 03/21] feat(coverage): report coverage as counters-only deltas The statement, function and branch maps dominate the size of an istanbul report, so only send them with the first report of each file and report the hit counters alone afterwards. Files that were not hit since the previous report are skipped, files that were never hit are still reported once to keep them in the denominator. The merge adopts the maps from the first report that carries them and zero fills the counters that the reports omit. --- packages/injected/src/coverageScript.ts | 67 +++++++++++++++++++++---- packages/isomorphic/istanbulCoverage.ts | 29 +++++++++-- tests/library/coverage.spec.ts | 43 ++++++++++++++++ 3 files changed, 123 insertions(+), 16 deletions(-) diff --git a/packages/injected/src/coverageScript.ts b/packages/injected/src/coverageScript.ts index 3bcd3578071b3..79fc44dfa2728 100644 --- a/packages/injected/src/coverageScript.ts +++ b/packages/injected/src/coverageScript.ts @@ -14,15 +14,18 @@ * limitations under the License. */ -import type { IstanbulCoverage } from '@isomorphic/istanbulCoverage'; +import type { IstanbulCoverage, IstanbulCoverageDelta, IstanbulFileCoverageDelta } from '@isomorphic/istanbulCoverage'; const kBacklogKey = '__pwCoverageBacklog'; // Harvests istanbul counters accumulated by instrumented application code in // `__coverage__`. Every flush serializes the counters and resets them, so -// each flush reports only the delta since the previous one. +// each flush reports only the delta since the previous one. The statement, +// function and branch maps are large, so they are only reported with the +// first report of each file. export class CoverageScript { private _global: typeof globalThis; + private _reportedFiles = new Set(); constructor(global: typeof globalThis, bindingName: string, collectName: string) { this._global = global; @@ -48,16 +51,28 @@ export class CoverageScript { const coverage: IstanbulCoverage | undefined = (this._global as any).__coverage__; if (!coverage) return undefined; - const json = JSON.stringify(coverage); - for (const file of Object.values(coverage)) { - for (const key of Object.keys(file.s)) - file.s[key] = 0; - for (const key of Object.keys(file.f)) - file.f[key] = 0; - for (const key of Object.keys(file.b)) - file.b[key] = file.b[key].map(() => 0); + const delta: IstanbulCoverageDelta = {}; + let hasFiles = false; + for (const [file, fileCoverage] of Object.entries(coverage)) { + const s = takeCounters(fileCoverage.s); + const f = takeCounters(fileCoverage.f); + const b = takeBranchCounters(fileCoverage.b); + // Report every file once to account for the files that were never hit, + // afterwards only report the files that were hit since the last report. + const isFirstReport = !this._reportedFiles.has(file); + if (!isFirstReport && !s && !f && !b) + continue; + const entry: IstanbulFileCoverageDelta = { path: fileCoverage.path, s: s || {}, f: f || {}, b: b || {} }; + if (isFirstReport) { + entry.statementMap = fileCoverage.statementMap; + entry.fnMap = fileCoverage.fnMap; + entry.branchMap = fileCoverage.branchMap; + this._reportedFiles.add(file); + } + delta[file] = entry; + hasFiles = true; } - return json; + return hasFiles ? JSON.stringify(delta) : undefined; } private _takeBacklog(): string[] { @@ -85,3 +100,33 @@ export class CoverageScript { } } } + +// Returns the non-zero counters and resets them, or undefined when there are none. +function takeCounters(counters: { [key: string]: number }): { [key: string]: number } | undefined { + let result: { [key: string]: number } | undefined; + for (const key of Object.keys(counters)) { + const count = counters[key]; + if (!count) + continue; + if (!result) + result = {}; + result[key] = count; + counters[key] = 0; + } + return result; +} + +// Branch counters are positional, so hit branches are reported with the whole array. +function takeBranchCounters(counters: { [key: string]: number[] }): { [key: string]: number[] } | undefined { + let result: { [key: string]: number[] } | undefined; + for (const key of Object.keys(counters)) { + const counts = counters[key]; + if (!counts.some(Boolean)) + continue; + if (!result) + result = {}; + result[key] = counts.slice(); + counters[key] = counts.map(() => 0); + } + return result; +} diff --git a/packages/isomorphic/istanbulCoverage.ts b/packages/isomorphic/istanbulCoverage.ts index 1b146fb7c48e9..0d4565a3ecfd3 100644 --- a/packages/isomorphic/istanbulCoverage.ts +++ b/packages/isomorphic/istanbulCoverage.ts @@ -55,14 +55,33 @@ export type IstanbulFileCoverage = { export type IstanbulCoverage = { [file: string]: IstanbulFileCoverage }; +// Collected counters, reported as a delta since the previous report. The maps +// are only sent with the first report of each file, so they are optional here. +export type IstanbulFileCoverageDelta = Partial & Pick; + +export type IstanbulCoverageDelta = { [file: string]: IstanbulFileCoverageDelta }; + // Merges istanbul hit counts: counters add up, map entries are taken from the -// first occurrence of each file. Assumes all data comes from the same build. -export function mergeIstanbulCoverage(into: Map, data: IstanbulCoverage) { +// first report that carries them. Assumes all data comes from the same build. +export function mergeIstanbulCoverage(into: Map, data: IstanbulCoverageDelta) { for (const [file, fileCov] of Object.entries(data)) { - const existing = into.get(file); + let existing = into.get(file); if (!existing) { - into.set(file, fileCov); - continue; + existing = { path: fileCov.path, statementMap: {}, fnMap: {}, branchMap: {}, s: {}, f: {}, b: {} }; + into.set(file, existing); + } + // A counters-only report can arrive first if the report carrying the maps was lost. + if (fileCov.statementMap && !Object.keys(existing.statementMap).length) { + existing.statementMap = fileCov.statementMap; + existing.fnMap = fileCov.fnMap || {}; + existing.branchMap = fileCov.branchMap || {}; + // Reports only carry the counters that were hit, zero fill the rest. + for (const key of Object.keys(existing.statementMap)) + existing.s[key] = existing.s[key] || 0; + for (const key of Object.keys(existing.fnMap)) + existing.f[key] = existing.f[key] || 0; + for (const [key, branch] of Object.entries(existing.branchMap)) + existing.b[key] = existing.b[key] || branch.locations.map(() => 0); } for (const key of Object.keys(fileCov.s)) existing.s[key] = (existing.s[key] || 0) + fileCov.s[key]; diff --git a/tests/library/coverage.spec.ts b/tests/library/coverage.spec.ts index 76a23b407efd8..5109613099ee2 100644 --- a/tests/library/coverage.spec.ts +++ b/tests/library/coverage.spec.ts @@ -84,6 +84,49 @@ it('should collect coverage per trace chunk', async ({ browser }, testInfo) => { expect(data2['a.js']).toBe(undefined); }); +it('should report maps once and counters incrementally', async ({ browser }) => { + const context = await browser.newContext(); + await context.tracing.start({ coverage: true }); + const page = await context.newPage(); + await page.setContent(coverageScript('a.js', 3)); + + const collect = () => page.evaluate(() => (window as any).__pwCoverageCollect().map((json: string) => JSON.parse(json))); + + const first = await collect(); + expect(first[0]['a.js'].statementMap).toBeTruthy(); + expect(first[0]['a.js'].s).toEqual({ '0': 3 }); + + // Nothing was hit since the last report. + expect(await collect()).toEqual([]); + + await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'] += 2); + const third = await collect(); + expect(third[0]['a.js'].statementMap).toBe(undefined); + expect(third[0]['a.js'].s).toEqual({ '0': 2 }); + + await context.tracing.stop(); + await context.close(); +}); + +it('should accumulate counters across flushes and keep never hit files', async ({ browser }, testInfo) => { + const context = await browser.newContext(); + await context.tracing.start({ coverage: true }); + const page = await context.newPage(); + await page.setContent(coverageScript('a.js', 0)); + await context.tracing.flushCoverage(); + await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'] += 4); + await context.tracing.flushCoverage(); + await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'] += 3); + + const traceFile = testInfo.outputPath('trace.zip'); + await context.tracing.stop({ path: traceFile }); + await context.close(); + + const data = await readCoverage(traceFile); + expect(data['a.js'].s['0']).toBe(7); + expect(Object.keys(data['a.js'].statementMap)).toEqual(['0']); +}); + it('should throw when flushing without coverage', async ({ browser }) => { const context = await browser.newContext(); await context.newPage(); From c93a56649b0a146468e4a7d7550dfe5256da6ab1 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Fri, 11 Sep 2026 13:24:03 -0700 Subject: [PATCH 04/21] feat(coverage): stash coverage in localStorage to survive page close Counters parked on pagehide used to go to sessionStorage, which dies together with a page that closes itself. Stash them in localStorage instead, where any same origin document picks them up, and relay the stashes on every collect, not only when a new document boots. Stashes are scoped to the recorder session so that the ones left by a previous run in a persistent profile are discarded, and carry an id so that the recorder drops the copies when several documents pick up the same stash. --- packages/injected/src/coverageScript.ts | 69 ++++++++------ packages/isomorphic/istanbulCoverage.ts | 7 ++ .../src/server/coverageRecorder.ts | 34 +++++-- tests/library/coverage.spec.ts | 93 +++++++++++++++++-- 4 files changed, 158 insertions(+), 45 deletions(-) diff --git a/packages/injected/src/coverageScript.ts b/packages/injected/src/coverageScript.ts index 79fc44dfa2728..650277e633514 100644 --- a/packages/injected/src/coverageScript.ts +++ b/packages/injected/src/coverageScript.ts @@ -16,7 +16,7 @@ import type { IstanbulCoverage, IstanbulCoverageDelta, IstanbulFileCoverageDelta } from '@isomorphic/istanbulCoverage'; -const kBacklogKey = '__pwCoverageBacklog'; +export const kCoverageStashPrefix = '__pwCoverage.'; // Harvests istanbul counters accumulated by instrumented application code in // `__coverage__`. Every flush serializes the counters and resets them, so @@ -25,29 +25,33 @@ const kBacklogKey = '__pwCoverageBacklog'; // first report of each file. export class CoverageScript { private _global: typeof globalThis; + private _sessionId: string; private _reportedFiles = new Set(); + private _stashOrdinal = 0; - constructor(global: typeof globalThis, bindingName: string, collectName: string) { + constructor(global: typeof globalThis, bindingName: string, collectName: string, sessionId: string) { this._global = global; + this._sessionId = sessionId; (global as any)[collectName] = () => this.collect(); // Counters die with the document, and binding calls made during unload - // are not delivered, so park the delta in sessionStorage for the next - // same-origin document or the end-of-test sweep to pick up. - global.addEventListener('pagehide', () => this._parkCurrent()); - // Relay deltas parked by previous documents, now that delivery is safe. - for (const json of this._takeBacklog()) + // are not delivered, so stash the delta in localStorage. Any same origin + // document picks it up later, including one opened by Playwright itself + // after the page that produced it is gone. + global.addEventListener('pagehide', () => this._stashCurrent()); + // Relay stashes left by other documents, now that delivery is safe. + for (const json of this._takeStashes()) (global as any)[bindingName](json).catch(() => {}); } collect(): string[] { - const chunks = this._takeBacklog(); - const current = this._takeCurrent(); - if (current) - chunks.push(current); + const chunks = this._takeStashes(); + const delta = this._takeCurrent(); + if (delta) + chunks.push(JSON.stringify({ data: delta })); return chunks; } - private _takeCurrent(): string | undefined { + private _takeCurrent(): IstanbulCoverageDelta | undefined { const coverage: IstanbulCoverage | undefined = (this._global as any).__coverage__; if (!coverage) return undefined; @@ -72,30 +76,43 @@ export class CoverageScript { delta[file] = entry; hasFiles = true; } - return hasFiles ? JSON.stringify(delta) : undefined; + return hasFiles ? delta : undefined; } - private _takeBacklog(): string[] { + // Takes the stashes left by this session and discards the ones left by other + // sessions, e.g. by a previous run reusing a persistent profile. + private _takeStashes(): string[] { + const result: string[] = []; try { - const backlog = this._global.sessionStorage.getItem(kBacklogKey); - if (backlog) { - this._global.sessionStorage.removeItem(kBacklogKey); - return JSON.parse(backlog); + const storage = this._global.localStorage; + const sessionPrefix = kCoverageStashPrefix + this._sessionId + '.'; + const keys: string[] = []; + for (let i = 0; i < storage.length; i++) { + const key = storage.key(i); + if (key && key.startsWith(kCoverageStashPrefix)) + keys.push(key); + } + for (const key of keys) { + const json = key.startsWith(sessionPrefix) ? storage.getItem(key) : undefined; + storage.removeItem(key); + if (json) + result.push(json); } } catch { } - return []; + return result; } - private _parkCurrent() { - const json = this._takeCurrent(); - if (!json) + private _stashCurrent() { + const delta = this._takeCurrent(); + if (!delta) return; try { - const existing = this._global.sessionStorage.getItem(kBacklogKey); - const backlog = existing ? JSON.parse(existing) : []; - backlog.push(json); - this._global.sessionStorage.setItem(kBacklogKey, JSON.stringify(backlog)); + // A stash can be picked up by several documents at once, so it carries an + // id that lets the recorder discard the copies. + const id = ++this._stashOrdinal + '-' + Math.random().toString(36).slice(2); + const key = kCoverageStashPrefix + this._sessionId + '.' + id; + this._global.localStorage.setItem(key, JSON.stringify({ id, data: delta })); } catch { } } diff --git a/packages/isomorphic/istanbulCoverage.ts b/packages/isomorphic/istanbulCoverage.ts index 0d4565a3ecfd3..43b40558f12d5 100644 --- a/packages/isomorphic/istanbulCoverage.ts +++ b/packages/isomorphic/istanbulCoverage.ts @@ -61,6 +61,13 @@ export type IstanbulFileCoverageDelta = Partial & Pick, data: IstanbulCoverageDelta) { diff --git a/packages/playwright-core/src/server/coverageRecorder.ts b/packages/playwright-core/src/server/coverageRecorder.ts index 80b61d3611abb..d9fc4d39216a9 100644 --- a/packages/playwright-core/src/server/coverageRecorder.ts +++ b/packages/playwright-core/src/server/coverageRecorder.ts @@ -15,27 +15,26 @@ */ import { mergeIstanbulCoverage } from '@isomorphic/istanbulCoverage'; +import { createGuid } from '@utils/crypto'; import * as rawCoverageSource from '../generated/coverageScriptSource'; import type { BrowserContext } from './browserContext'; import type { Page } from './page'; import type { Progress } from './progress'; -import type { IstanbulFileCoverage } from '@isomorphic/istanbulCoverage'; +import type { IstanbulCoverageChunk, IstanbulFileCoverage } from '@isomorphic/istanbulCoverage'; const kCoverageBindingName = '__pwCoverageSink'; const kCoverageCollectName = '__pwCoverageCollect'; -const coverageBootstrapSource = `(() => { - const module = {}; - ${rawCoverageSource.source} - new (module.exports.CoverageScript())(window, ${JSON.stringify(kCoverageBindingName)}, ${JSON.stringify(kCoverageCollectName)}); -})()`; - const coverageCollectExpression = `window[${JSON.stringify(kCoverageCollectName)}] ? window[${JSON.stringify(kCoverageCollectName)}]() : []`; export class CoverageRecorder { private _context: BrowserContext; private _coverage = new Map(); + private _stashedChunkIds = new Set(); + // Scopes the stashes in the page storage to this recorder, so that the ones + // left behind by a previous run in a persistent profile are discarded. + private _sessionId = createGuid(); private _installed = false; private _active = false; @@ -43,6 +42,14 @@ export class CoverageRecorder { this._context = context; } + private _bootstrapSource() { + return `(() => { + const module = {}; + ${rawCoverageSource.source} + new (module.exports.CoverageScript())(window, ${JSON.stringify(kCoverageBindingName)}, ${JSON.stringify(kCoverageCollectName)}, ${JSON.stringify(this._sessionId)}); + })()`; + } + activate() { this._active = true; } @@ -59,10 +66,11 @@ export class CoverageRecorder { if (this._installed) return; this._installed = true; + const bootstrapSource = this._bootstrapSource(); await this._context.exposeBinding(progress, kCoverageBindingName, (source, json: string) => this._append(json)); - await this._context.addInitScript(progress, coverageBootstrapSource); + await this._context.addInitScript(progress, bootstrapSource); // Init scripts only affect future documents, bootstrap the existing ones. - await progress.race(this._context.safeNonStallingEvaluateInAllFrames(coverageBootstrapSource, 'main')); + await progress.race(this._context.safeNonStallingEvaluateInAllFrames(bootstrapSource, 'main')); } async flush() { @@ -90,7 +98,13 @@ export class CoverageRecorder { private _append(json: string) { try { - mergeIstanbulCoverage(this._coverage, JSON.parse(json)); + const chunk: IstanbulCoverageChunk = JSON.parse(json); + if (chunk.id) { + if (this._stashedChunkIds.has(chunk.id)) + return; + this._stashedChunkIds.add(chunk.id); + } + mergeIstanbulCoverage(this._coverage, chunk.data); } catch { } } diff --git a/tests/library/coverage.spec.ts b/tests/library/coverage.spec.ts index 5109613099ee2..9d06e5d51dc89 100644 --- a/tests/library/coverage.spec.ts +++ b/tests/library/coverage.spec.ts @@ -17,8 +17,8 @@ import { browserTest as it, expect } from '../config/browserTest'; import { parseTraceRaw } from '../config/utils'; -const coverageScript = (file: string, s0: number) => { - const fileCov = { +const fileCoverage = (file: string, s0: number) => ({ + [file]: { path: file, statementMap: { '0': { start: { line: 1, column: 0 }, end: { line: 1, column: 20 } } }, fnMap: {}, @@ -26,9 +26,10 @@ const coverageScript = (file: string, s0: number) => { s: { '0': s0 }, f: {}, b: {}, - }; - return ``; -}; + }, +}); + +const coverageScript = (file: string, s0: number) => ``; async function readCoverage(traceFile: string): Promise { const { resources } = await parseTraceRaw(traceFile); @@ -93,16 +94,16 @@ it('should report maps once and counters incrementally', async ({ browser }) => const collect = () => page.evaluate(() => (window as any).__pwCoverageCollect().map((json: string) => JSON.parse(json))); const first = await collect(); - expect(first[0]['a.js'].statementMap).toBeTruthy(); - expect(first[0]['a.js'].s).toEqual({ '0': 3 }); + expect(first[0].data['a.js'].statementMap).toBeTruthy(); + expect(first[0].data['a.js'].s).toEqual({ '0': 3 }); // Nothing was hit since the last report. expect(await collect()).toEqual([]); await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'] += 2); const third = await collect(); - expect(third[0]['a.js'].statementMap).toBe(undefined); - expect(third[0]['a.js'].s).toEqual({ '0': 2 }); + expect(third[0].data['a.js'].statementMap).toBe(undefined); + expect(third[0].data['a.js'].s).toEqual({ '0': 2 }); await context.tracing.stop(); await context.close(); @@ -127,6 +128,80 @@ it('should accumulate counters across flushes and keep never hit files', async ( expect(Object.keys(data['a.js'].statementMap)).toEqual(['0']); }); +it('should collect coverage of a page closed by in-page script', async ({ browser, server }, testInfo) => { + const context = await browser.newContext(); + await context.tracing.start({ coverage: true }); + const page = await context.newPage(); + await page.goto(server.EMPTY_PAGE); + + const [popup] = await Promise.all([ + page.waitForEvent('popup'), + page.evaluate(url => window.open(url), server.EMPTY_PAGE), + ]); + await popup.evaluate(coverage => (window as any).__coverage__ = JSON.parse(coverage), JSON.stringify(fileCoverage('popup.js', 5))); + // The popup closes itself, so its counters are only preserved by the stash. + await popup.evaluate(() => setTimeout(() => window.close(), 0)); + await popup.waitForEvent('close'); + + const traceFile = testInfo.outputPath('trace.zip'); + await context.tracing.stop({ path: traceFile }); + await context.close(); + + const data = await readCoverage(traceFile); + expect(data['popup.js'].s['0']).toBe(5); +}); + +it('should not double count a stash picked up twice', async ({ browser, server }, testInfo) => { + const context = await browser.newContext(); + await context.tracing.start({ coverage: true }); + const page = await context.newPage(); + await page.goto(server.EMPTY_PAGE); + + const [popup] = await Promise.all([ + page.waitForEvent('popup'), + page.evaluate(url => window.open(url), server.EMPTY_PAGE), + ]); + await popup.evaluate(coverage => (window as any).__coverage__ = JSON.parse(coverage), JSON.stringify(fileCoverage('popup.js', 5))); + await popup.evaluate(() => setTimeout(() => window.close(), 0)); + await popup.waitForEvent('close'); + + // Emulate two documents picking up the same stash before either removes it. + const copied = await page.evaluate(() => { + const key = Object.keys(localStorage).find(key => key.startsWith('__pwCoverage.'))!; + localStorage.setItem(key + '.copy', localStorage.getItem(key)!); + return Object.keys(localStorage).filter(key => key.startsWith('__pwCoverage.')).length; + }); + expect(copied).toBe(2); + + const traceFile = testInfo.outputPath('trace.zip'); + await context.tracing.stop({ path: traceFile }); + await context.close(); + + const data = await readCoverage(traceFile); + expect(data['popup.js'].s['0']).toBe(5); +}); + +it('should discard stashes of other sessions', async ({ browser, server }, testInfo) => { + const context = await browser.newContext(); + await context.tracing.start({ coverage: true }); + const page = await context.newPage(); + await page.goto(server.EMPTY_PAGE); + await page.evaluate(coverage => { + localStorage.setItem('__pwCoverage.other-session.1', JSON.stringify({ id: 'other', data: JSON.parse(coverage) })); + }, JSON.stringify(fileCoverage('stale.js', 7))); + + await context.tracing.flushCoverage(); + const remaining = await page.evaluate(() => Object.keys(localStorage).filter(key => key.startsWith('__pwCoverage.'))); + expect(remaining).toEqual([]); + + const traceFile = testInfo.outputPath('trace.zip'); + await context.tracing.stop({ path: traceFile }); + await context.close(); + + const data = await readCoverage(traceFile); + expect(data['stale.js']).toBe(undefined); +}); + it('should throw when flushing without coverage', async ({ browser }) => { const context = await browser.newContext(); await context.newPage(); From cd0d3f93da962b95e236ac26632e108186af87df Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Fri, 11 Sep 2026 13:56:47 -0700 Subject: [PATCH 05/21] feat(coverage): harvest coverage stashes of origins left without a page A page relays the stashes of its own origin, so the stash of a page that closed itself is only picked up while another page of that origin is still around. Track the origins of the closed pages and visit the ones that have no page left in a page of Playwright's own, reusing the origin traversal of browserContext.storageState. --- packages/injected/src/coverageScript.ts | 48 ++++++++++-------- .../src/server/browserContext.ts | 38 ++++++++------ .../src/server/coverageRecorder.ts | 50 +++++++++++++++++-- .../src/server/trace/recorder/tracing.ts | 5 +- tests/library/coverage.spec.ts | 27 ++++++++++ 5 files changed, 126 insertions(+), 42 deletions(-) diff --git a/packages/injected/src/coverageScript.ts b/packages/injected/src/coverageScript.ts index 650277e633514..84eb50b2be119 100644 --- a/packages/injected/src/coverageScript.ts +++ b/packages/injected/src/coverageScript.ts @@ -79,28 +79,8 @@ export class CoverageScript { return hasFiles ? delta : undefined; } - // Takes the stashes left by this session and discards the ones left by other - // sessions, e.g. by a previous run reusing a persistent profile. private _takeStashes(): string[] { - const result: string[] = []; - try { - const storage = this._global.localStorage; - const sessionPrefix = kCoverageStashPrefix + this._sessionId + '.'; - const keys: string[] = []; - for (let i = 0; i < storage.length; i++) { - const key = storage.key(i); - if (key && key.startsWith(kCoverageStashPrefix)) - keys.push(key); - } - for (const key of keys) { - const json = key.startsWith(sessionPrefix) ? storage.getItem(key) : undefined; - storage.removeItem(key); - if (json) - result.push(json); - } - } catch { - } - return result; + return takeCoverageStashes(this._global, this._sessionId); } private _stashCurrent() { @@ -118,6 +98,32 @@ export class CoverageScript { } } +// Takes the stashes left by this session and discards the ones left by other +// sessions, e.g. by a previous run reusing a persistent profile. Playwright +// also evaluates this on its own page to reach the origins that have no page +// left to relay them. +export function takeCoverageStashes(global: typeof globalThis, sessionId: string): string[] { + const result: string[] = []; + try { + const storage = global.localStorage; + const sessionPrefix = kCoverageStashPrefix + sessionId + '.'; + const keys: string[] = []; + for (let i = 0; i < storage.length; i++) { + const key = storage.key(i); + if (key && key.startsWith(kCoverageStashPrefix)) + keys.push(key); + } + for (const key of keys) { + const json = key.startsWith(sessionPrefix) ? storage.getItem(key) : undefined; + storage.removeItem(key); + if (json) + result.push(json); + } + } catch { + } + return result; +} + // Returns the non-zero counters and resets them, or undefined when there are none. function takeCounters(counters: { [key: string]: number }): { [key: string]: number } | undefined { let result: { [key: string]: number } | undefined; diff --git a/packages/playwright-core/src/server/browserContext.ts b/packages/playwright-core/src/server/browserContext.ts index e510a16e645e9..0d00032a59959 100644 --- a/packages/playwright-core/src/server/browserContext.ts +++ b/packages/playwright-core/src/server/browserContext.ts @@ -648,25 +648,33 @@ export abstract class BrowserContext extends Sdk // If there are still origins to save, create a blank page to iterate over origins. if (originsToSave.size) { - const page = await this.newPage(progress, true /* forStorageState */); - try { - await page.addRequestInterceptor(progress, route => { - route.fulfill({ body: '' }).catch(() => {}); - }, 'prepend'); - for (const origin of originsToSave) { - const frame = page.mainFrame(); - await frame.gotoImpl(progress, origin, {}); - const storage: SerializedStorage = await frame.evaluateExpression(progress, collectScript, { world: 'utility' }); - if (hasStorage(storage)) - result.origins.push({ origin, ...storage }); - } - } finally { - await page.close(progress); - } + await this.visitOrigins(progress, originsToSave, async (frame, origin) => { + const storage: SerializedStorage = await frame.evaluateExpression(progress, collectScript, { world: 'utility' }); + if (hasStorage(storage)) + result.origins.push({ origin, ...storage }); + }); } return result; } + // Visits each origin in a page of its own that is hidden from the client, to + // reach the storage of the origins that have no page of their own. + async visitOrigins(progress: Progress, origins: Set, callback: (frame: frames.Frame, origin: string) => Promise) { + const page = await this.newPage(progress, true /* forStorageState */); + try { + await page.addRequestInterceptor(progress, route => { + route.fulfill({ body: '' }).catch(() => {}); + }, 'prepend'); + for (const origin of origins) { + const frame = page.mainFrame(); + await frame.gotoImpl(progress, origin, {}); + await progress.race(callback(frame, origin)); + } + } finally { + await page.close(progress); + } + } + isCreatingStorageStatePage(): boolean { return this._creatingStorageStatePage; } diff --git a/packages/playwright-core/src/server/coverageRecorder.ts b/packages/playwright-core/src/server/coverageRecorder.ts index d9fc4d39216a9..4c812b670ea41 100644 --- a/packages/playwright-core/src/server/coverageRecorder.ts +++ b/packages/playwright-core/src/server/coverageRecorder.ts @@ -32,6 +32,7 @@ export class CoverageRecorder { private _context: BrowserContext; private _coverage = new Map(); private _stashedChunkIds = new Set(); + private _closedPageOrigins = new Set(); // Scopes the stashes in the page storage to this recorder, so that the ones // left behind by a previous run in a persistent profile are discarded. private _sessionId = createGuid(); @@ -73,16 +74,28 @@ export class CoverageRecorder { await progress.race(this._context.safeNonStallingEvaluateInAllFrames(bootstrapSource, 'main')); } - async flush() { + onPageClose(page: Page) { + if (!this._active) + return; + // The stash of a closed page stays in the storage of its origin. + for (const frame of page.frames()) { + const origin = frame.origin(); + if (origin) + this._closedPageOrigins.add(origin); + } + } + + async flush(progress: Progress) { for (const page of this._context.pages()) - await this.collectFromPage(page); + await progress.race(this.collectFromPage(page)); + await this._harvestClosedOrigins(progress); } // Collects pending coverage from all pages and returns it serialized, resetting the accumulator. - async take(): Promise { + async take(progress: Progress): Promise { if (!this._active) return; - await this.flush(); + await this.flush(progress); const sorted = Object.fromEntries([...this._coverage.entries()].sort(([a], [b]) => a.localeCompare(b))); this._coverage.clear(); return JSON.stringify(sorted); @@ -96,6 +109,35 @@ export class CoverageRecorder { } } + // Pages relay the stashes of their own origin, so only the origins that are + // left without a page need a page of their own to relay them. + private async _harvestClosedOrigins(progress: Progress) { + if (!this._closedPageOrigins.size || this._context.isClosingOrClosed()) + return; + const liveOrigins = new Set(); + for (const page of this._context.pages()) { + for (const frame of page.frames()) { + const origin = frame.origin(); + if (origin) + liveOrigins.add(origin); + } + } + const origins = new Set([...this._closedPageOrigins].filter(origin => !liveOrigins.has(origin))); + this._closedPageOrigins.clear(); + if (!origins.size) + return; + const takeStashesExpression = `(() => { + const module = {}; + ${rawCoverageSource.source} + return (module.exports.takeCoverageStashes())(window, ${JSON.stringify(this._sessionId)}); + })()`; + await this._context.visitOrigins(progress, origins, async frame => { + const chunks: string[] = await frame.evaluateExpression(progress, takeStashesExpression, { world: 'main' }).catch(() => []); + for (const json of chunks || []) + this._append(json); + }); + } + private _append(json: string) { try { const chunk: IstanbulCoverageChunk = JSON.parse(json); diff --git a/packages/playwright-core/src/server/trace/recorder/tracing.ts b/packages/playwright-core/src/server/trace/recorder/tracing.ts index d4dcc453a7050..4d1bb7a7b9cc5 100644 --- a/packages/playwright-core/src/server/trace/recorder/tracing.ts +++ b/packages/playwright-core/src/server/trace/recorder/tracing.ts @@ -400,7 +400,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps async flushCoverage(progress: Progress) { if (!this._coverageRecorder?.active()) throw new Error(`Coverage collection has not been started. Pass the "coverage" option to tracing.start().`); - await progress.race(this._coverageRecorder.flush()); + await this._coverageRecorder.flush(progress); } // Best-effort collection before the page is gone, no-op unless collecting. @@ -415,7 +415,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps private async _takeCoverage(progress: Progress, mode: TracingTracingStopChunkParams['mode']): Promise { if (!this._state?.recording || !this._state.options.coverage || !this._coverageRecorder) return; - const coverageJson = await progress.race(this._coverageRecorder.take()); + const coverageJson = await this._coverageRecorder.take(progress); if (mode === 'discard' || coverageJson === undefined) return; const coverageFile = path.join(this._state.tracesDir, `${this._state.traceName}-coverage-${this._state.chunkOrdinal}.json`); @@ -710,6 +710,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps } onPageClose(page: Page) { + this._coverageRecorder?.onPageClose(page); const event: trace.EventTraceEvent = { type: 'event', time: monotonicTime(), diff --git a/tests/library/coverage.spec.ts b/tests/library/coverage.spec.ts index 9d06e5d51dc89..c817feea6fcf3 100644 --- a/tests/library/coverage.spec.ts +++ b/tests/library/coverage.spec.ts @@ -202,6 +202,33 @@ it('should discard stashes of other sessions', async ({ browser, server }, testI expect(data['stale.js']).toBe(undefined); }); +it('should collect coverage of an origin left without a page', async ({ browser, server }, testInfo) => { + const context = await browser.newContext(); + await context.tracing.start({ coverage: true }); + const page = await context.newPage(); + await page.goto(server.EMPTY_PAGE); + await page.evaluate(coverage => (window as any).__coverage__ = JSON.parse(coverage), JSON.stringify(fileCoverage('opener.js', 2))); + + const [popup] = await Promise.all([ + page.waitForEvent('popup'), + page.evaluate(url => window.open(url), server.EMPTY_PAGE), + ]); + await popup.evaluate(coverage => (window as any).__coverage__ = JSON.parse(coverage), JSON.stringify(fileCoverage('popup.js', 5))); + await popup.evaluate(() => setTimeout(() => window.close(), 0)); + await popup.waitForEvent('close'); + + // Leave the origin of the stashes without a page to relay them. + await page.goto(server.CROSS_PROCESS_PREFIX + '/empty.html'); + + const traceFile = testInfo.outputPath('trace.zip'); + await context.tracing.stop({ path: traceFile }); + await context.close(); + + const data = await readCoverage(traceFile); + expect(data['popup.js'].s['0']).toBe(5); + expect(data['opener.js'].s['0']).toBe(2); +}); + it('should throw when flushing without coverage', async ({ browser }) => { const context = await browser.newContext(); await context.newPage(); From d1bc1d064c605decb376eb34af96aab819522d3c Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Fri, 11 Sep 2026 14:16:10 -0700 Subject: [PATCH 06/21] feat(coverage): collect coverage as the actions go Collect the counters of the acted upon page at the tracing instrumentation point that captures the after-action snapshot, so that an action is the most that can be lost when a page goes away without a chance to stash its counters, and so that the counters can later be attributed to the actions that produced them. With the pages swept this often, the stash in the page storage is only picked up by Playwright itself, so the coverage script no longer needs a binding to push it. Origins are noted as they are swept, not only when a page closes, to also harvest the ones left behind by a navigation. --- packages/injected/src/coverageScript.ts | 13 +-- .../src/server/coverageRecorder.ts | 48 ++++---- .../src/server/trace/recorder/tracing.ts | 8 ++ tests/library/coverage.spec.ts | 107 +++++++++++------- 4 files changed, 106 insertions(+), 70 deletions(-) diff --git a/packages/injected/src/coverageScript.ts b/packages/injected/src/coverageScript.ts index 84eb50b2be119..17a5a749aad8d 100644 --- a/packages/injected/src/coverageScript.ts +++ b/packages/injected/src/coverageScript.ts @@ -29,18 +29,15 @@ export class CoverageScript { private _reportedFiles = new Set(); private _stashOrdinal = 0; - constructor(global: typeof globalThis, bindingName: string, collectName: string, sessionId: string) { + constructor(global: typeof globalThis, collectName: string, sessionId: string) { this._global = global; this._sessionId = sessionId; (global as any)[collectName] = () => this.collect(); - // Counters die with the document, and binding calls made during unload - // are not delivered, so stash the delta in localStorage. Any same origin - // document picks it up later, including one opened by Playwright itself - // after the page that produced it is gone. + // Counters die with the document, and calls made during unload are not + // delivered, so stash the delta in localStorage. Playwright picks it up + // from any same origin document, including one it opens itself after the + // page that produced the stash is gone. global.addEventListener('pagehide', () => this._stashCurrent()); - // Relay stashes left by other documents, now that delivery is safe. - for (const json of this._takeStashes()) - (global as any)[bindingName](json).catch(() => {}); } collect(): string[] { diff --git a/packages/playwright-core/src/server/coverageRecorder.ts b/packages/playwright-core/src/server/coverageRecorder.ts index 4c812b670ea41..16b5256d158b2 100644 --- a/packages/playwright-core/src/server/coverageRecorder.ts +++ b/packages/playwright-core/src/server/coverageRecorder.ts @@ -15,7 +15,6 @@ */ import { mergeIstanbulCoverage } from '@isomorphic/istanbulCoverage'; -import { createGuid } from '@utils/crypto'; import * as rawCoverageSource from '../generated/coverageScriptSource'; import type { BrowserContext } from './browserContext'; @@ -23,7 +22,6 @@ import type { Page } from './page'; import type { Progress } from './progress'; import type { IstanbulCoverageChunk, IstanbulFileCoverage } from '@isomorphic/istanbulCoverage'; -const kCoverageBindingName = '__pwCoverageSink'; const kCoverageCollectName = '__pwCoverageCollect'; const coverageCollectExpression = `window[${JSON.stringify(kCoverageCollectName)}] ? window[${JSON.stringify(kCoverageCollectName)}]() : []`; @@ -32,10 +30,7 @@ export class CoverageRecorder { private _context: BrowserContext; private _coverage = new Map(); private _stashedChunkIds = new Set(); - private _closedPageOrigins = new Set(); - // Scopes the stashes in the page storage to this recorder, so that the ones - // left behind by a previous run in a persistent profile are discarded. - private _sessionId = createGuid(); + private _stashOrigins = new Set(); private _installed = false; private _active = false; @@ -43,11 +38,17 @@ export class CoverageRecorder { this._context = context; } + // Scopes the stashes in the page storage to this context, so that the ones + // left behind by a previous run in a persistent profile are discarded. + private _sessionId() { + return this._context.guid; + } + private _bootstrapSource() { return `(() => { const module = {}; ${rawCoverageSource.source} - new (module.exports.CoverageScript())(window, ${JSON.stringify(kCoverageBindingName)}, ${JSON.stringify(kCoverageCollectName)}, ${JSON.stringify(this._sessionId)}); + new (module.exports.CoverageScript())(window, ${JSON.stringify(kCoverageCollectName)}, ${JSON.stringify(this._sessionId())}); })()`; } @@ -68,7 +69,6 @@ export class CoverageRecorder { return; this._installed = true; const bootstrapSource = this._bootstrapSource(); - await this._context.exposeBinding(progress, kCoverageBindingName, (source, json: string) => this._append(json)); await this._context.addInitScript(progress, bootstrapSource); // Init scripts only affect future documents, bootstrap the existing ones. await progress.race(this._context.safeNonStallingEvaluateInAllFrames(bootstrapSource, 'main')); @@ -78,17 +78,19 @@ export class CoverageRecorder { if (!this._active) return; // The stash of a closed page stays in the storage of its origin. - for (const frame of page.frames()) { - const origin = frame.origin(); - if (origin) - this._closedPageOrigins.add(origin); - } + for (const frame of page.frames()) + this._noteOrigin(frame.origin()); + } + + private _noteOrigin(origin: string | undefined) { + if (origin) + this._stashOrigins.add(origin); } async flush(progress: Progress) { for (const page of this._context.pages()) await progress.race(this.collectFromPage(page)); - await this._harvestClosedOrigins(progress); + await this._harvestOriginsWithoutPage(progress); } // Collects pending coverage from all pages and returns it serialized, resetting the accumulator. @@ -102,7 +104,14 @@ export class CoverageRecorder { } async collectFromPage(page: Page) { + // A page that is going away may never answer, give up on it when it does. + await Promise.race([page.closedPromise, this._collectFromPage(page)]); + } + + private async _collectFromPage(page: Page) { for (const frame of page.frames()) { + // A document of this origin may leave a stash behind when it goes away. + this._noteOrigin(frame.origin()); const chunks: string[] = await frame.nonStallingEvaluateInExistingContext(coverageCollectExpression, 'main').catch(() => []); for (const json of chunks || []) this._append(json); @@ -111,8 +120,8 @@ export class CoverageRecorder { // Pages relay the stashes of their own origin, so only the origins that are // left without a page need a page of their own to relay them. - private async _harvestClosedOrigins(progress: Progress) { - if (!this._closedPageOrigins.size || this._context.isClosingOrClosed()) + private async _harvestOriginsWithoutPage(progress: Progress) { + if (!this._stashOrigins.size || this._context.isClosingOrClosed()) return; const liveOrigins = new Set(); for (const page of this._context.pages()) { @@ -122,14 +131,15 @@ export class CoverageRecorder { liveOrigins.add(origin); } } - const origins = new Set([...this._closedPageOrigins].filter(origin => !liveOrigins.has(origin))); - this._closedPageOrigins.clear(); + // The origins that still have a page were just drained by the sweep above. + const origins = new Set([...this._stashOrigins].filter(origin => !liveOrigins.has(origin))); + this._stashOrigins.clear(); if (!origins.size) return; const takeStashesExpression = `(() => { const module = {}; ${rawCoverageSource.source} - return (module.exports.takeCoverageStashes())(window, ${JSON.stringify(this._sessionId)}); + return (module.exports.takeCoverageStashes())(window, ${JSON.stringify(this._sessionId())}); })()`; await this._context.visitOrigins(progress, origins, async frame => { const chunks: string[] = await frame.evaluateExpression(progress, takeStashesExpression, { world: 'main' }).catch(() => []); diff --git a/packages/playwright-core/src/server/trace/recorder/tracing.ts b/packages/playwright-core/src/server/trace/recorder/tracing.ts index 4d1bb7a7b9cc5..a96f33daaea20 100644 --- a/packages/playwright-core/src/server/trace/recorder/tracing.ts +++ b/packages/playwright-core/src/server/trace/recorder/tracing.ts @@ -527,6 +527,14 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps await this._captureScreenshot(progress, page, phase); if (options?.snapshotAria) await this._captureAriaSnapshot(progress, page, phase); + // Pull the counters as the actions go, so that an action is the most that + // can be lost when the page goes away without a chance to stash them. + if (phase === 'after' && this._coverageRecorder?.active()) { + try { + await progress.race(this._coverageRecorder.collectFromPage(page)); + } catch { + } + } } private _shouldCaptureAtPhase(metadata: CallMetadata, phase: trace.ActionPhase) { diff --git a/tests/library/coverage.spec.ts b/tests/library/coverage.spec.ts index c817feea6fcf3..12201f672dc31 100644 --- a/tests/library/coverage.spec.ts +++ b/tests/library/coverage.spec.ts @@ -85,25 +85,29 @@ it('should collect coverage per trace chunk', async ({ browser }, testInfo) => { expect(data2['a.js']).toBe(undefined); }); -it('should report maps once and counters incrementally', async ({ browser }) => { +it('should report maps once and counters incrementally', async ({ browser, server }) => { const context = await browser.newContext(); await context.tracing.start({ coverage: true }); const page = await context.newPage(); - await page.setContent(coverageScript('a.js', 3)); - - const collect = () => page.evaluate(() => (window as any).__pwCoverageCollect().map((json: string) => JSON.parse(json))); + await page.goto(server.EMPTY_PAGE); - const first = await collect(); + // Actions collect the counters on their own, so hit and collect in one evaluate. + const first = await page.evaluate(coverage => { + (window as any).__coverage__ = JSON.parse(coverage); + return (window as any).__pwCoverageCollect().map((json: string) => JSON.parse(json)); + }, JSON.stringify(fileCoverage('a.js', 3))); expect(first[0].data['a.js'].statementMap).toBeTruthy(); expect(first[0].data['a.js'].s).toEqual({ '0': 3 }); - // Nothing was hit since the last report. - expect(await collect()).toEqual([]); + const second = await page.evaluate(() => { + (window as any).__coverage__['a.js'].s['0'] += 2; + return (window as any).__pwCoverageCollect().map((json: string) => JSON.parse(json)); + }); + expect(second[0].data['a.js'].statementMap).toBe(undefined); + expect(second[0].data['a.js'].s).toEqual({ '0': 2 }); - await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'] += 2); - const third = await collect(); - expect(third[0].data['a.js'].statementMap).toBe(undefined); - expect(third[0].data['a.js'].s).toEqual({ '0': 2 }); + // Nothing was hit since the last report. + expect(await page.evaluate(() => (window as any).__pwCoverageCollect())).toEqual([]); await context.tracing.stop(); await context.close(); @@ -128,20 +132,28 @@ it('should accumulate counters across flushes and keep never hit files', async ( expect(Object.keys(data['a.js'].statementMap)).toEqual(['0']); }); +async function openPopup(page: any, server: any, file: string, s0: number) { + const [popup] = await Promise.all([ + page.waitForEvent('popup'), + page.evaluate(url => window.open(url), server.EMPTY_PAGE), + ]); + await popup.evaluate(coverage => (window as any).__coverage__ = JSON.parse(coverage), JSON.stringify(fileCoverage(file, s0))); + return popup; +} + it('should collect coverage of a page closed by in-page script', async ({ browser, server }, testInfo) => { const context = await browser.newContext(); await context.tracing.start({ coverage: true }); const page = await context.newPage(); await page.goto(server.EMPTY_PAGE); - const [popup] = await Promise.all([ - page.waitForEvent('popup'), - page.evaluate(url => window.open(url), server.EMPTY_PAGE), + const popup = await openPopup(page, server, 'popup.js', 5); + // An in-page close never reaches Playwright as page.close(), the counters + // are preserved because the actions collect them as they go. + await Promise.all([ + popup.waitForEvent('close'), + popup.evaluate(() => setTimeout(() => window.close(), 0)), ]); - await popup.evaluate(coverage => (window as any).__coverage__ = JSON.parse(coverage), JSON.stringify(fileCoverage('popup.js', 5))); - // The popup closes itself, so its counters are only preserved by the stash. - await popup.evaluate(() => setTimeout(() => window.close(), 0)); - await popup.waitForEvent('close'); const traceFile = testInfo.outputPath('trace.zip'); await context.tracing.stop({ path: traceFile }); @@ -157,28 +169,20 @@ it('should not double count a stash picked up twice', async ({ browser, server } const page = await context.newPage(); await page.goto(server.EMPTY_PAGE); - const [popup] = await Promise.all([ - page.waitForEvent('popup'), - page.evaluate(url => window.open(url), server.EMPTY_PAGE), - ]); - await popup.evaluate(coverage => (window as any).__coverage__ = JSON.parse(coverage), JSON.stringify(fileCoverage('popup.js', 5))); - await popup.evaluate(() => setTimeout(() => window.close(), 0)); - await popup.waitForEvent('close'); - // Emulate two documents picking up the same stash before either removes it. - const copied = await page.evaluate(() => { - const key = Object.keys(localStorage).find(key => key.startsWith('__pwCoverage.'))!; - localStorage.setItem(key + '.copy', localStorage.getItem(key)!); - return Object.keys(localStorage).filter(key => key.startsWith('__pwCoverage.')).length; - }); - expect(copied).toBe(2); + const prefix = '__pwCoverage.' + (context as any)._guid + '.'; + await page.evaluate(({ prefix, coverage }) => { + const chunk = JSON.stringify({ id: 'stash-id', data: JSON.parse(coverage) }); + localStorage.setItem(prefix + 'one', chunk); + localStorage.setItem(prefix + 'two', chunk); + }, { prefix, coverage: JSON.stringify(fileCoverage('stashed.js', 4)) }); const traceFile = testInfo.outputPath('trace.zip'); await context.tracing.stop({ path: traceFile }); await context.close(); const data = await readCoverage(traceFile); - expect(data['popup.js'].s['0']).toBe(5); + expect(data['stashed.js'].s['0']).toBe(4); }); it('should discard stashes of other sessions', async ({ browser, server }, testInfo) => { @@ -207,17 +211,16 @@ it('should collect coverage of an origin left without a page', async ({ browser, await context.tracing.start({ coverage: true }); const page = await context.newPage(); await page.goto(server.EMPTY_PAGE); - await page.evaluate(coverage => (window as any).__coverage__ = JSON.parse(coverage), JSON.stringify(fileCoverage('opener.js', 2))); - const [popup] = await Promise.all([ - page.waitForEvent('popup'), - page.evaluate(url => window.open(url), server.EMPTY_PAGE), - ]); - await popup.evaluate(coverage => (window as any).__coverage__ = JSON.parse(coverage), JSON.stringify(fileCoverage('popup.js', 5))); - await popup.evaluate(() => setTimeout(() => window.close(), 0)); - await popup.waitForEvent('close'); + const popup = await openPopup(page, server, 'popup.js', 0); + await context.tracing.flushCoverage(); - // Leave the origin of the stashes without a page to relay them. + // The popup is hit and navigates away, leaving a stash behind. + await popup.evaluate(url => { + (window as any).__coverage__['popup.js'].s['0'] = 5; + window.location.href = url; + }, server.CROSS_PROCESS_PREFIX + '/empty.html'); + // Leave the origin of the stash without a page to relay it. await page.goto(server.CROSS_PROCESS_PREFIX + '/empty.html'); const traceFile = testInfo.outputPath('trace.zip'); @@ -226,7 +229,25 @@ it('should collect coverage of an origin left without a page', async ({ browser, const data = await readCoverage(traceFile); expect(data['popup.js'].s['0']).toBe(5); - expect(data['opener.js'].s['0']).toBe(2); +}); + +it('should pull counters as the actions go', async ({ browser, server }, testInfo) => { + const context = await browser.newContext(); + await context.tracing.start({ coverage: true }); + const page = await context.newPage(); + await page.goto(server.EMPTY_PAGE); + await page.evaluate(coverage => (window as any).__coverage__ = JSON.parse(coverage), JSON.stringify(fileCoverage('a.js', 3))); + + await page.click('body'); + // The counters are collected by the actions themselves, not only by the flush. + expect(await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'])).toBe(0); + + const traceFile = testInfo.outputPath('trace.zip'); + await context.tracing.stop({ path: traceFile }); + await context.close(); + + const data = await readCoverage(traceFile); + expect(data['a.js'].s['0']).toBe(3); }); it('should throw when flushing without coverage', async ({ browser }) => { From 6e6cc983213156810a7cd8a9d4ec01c2b3b2c97d Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Fri, 11 Sep 2026 16:22:29 -0700 Subject: [PATCH 07/21] chore(coverage): trim comments --- packages/injected/src/coverageScript.ts | 26 +++++-------------- packages/isomorphic/istanbulCoverage.ts | 14 +++------- .../src/server/browserContext.ts | 3 +-- .../src/server/coverageRecorder.ts | 13 +++------- .../src/server/trace/recorder/tracing.ts | 7 ++--- packages/playwright/src/reporters/coverage.ts | 3 +-- 6 files changed, 19 insertions(+), 47 deletions(-) diff --git a/packages/injected/src/coverageScript.ts b/packages/injected/src/coverageScript.ts index 17a5a749aad8d..fb5dcdfcf06d8 100644 --- a/packages/injected/src/coverageScript.ts +++ b/packages/injected/src/coverageScript.ts @@ -18,11 +18,7 @@ import type { IstanbulCoverage, IstanbulCoverageDelta, IstanbulFileCoverageDelta export const kCoverageStashPrefix = '__pwCoverage.'; -// Harvests istanbul counters accumulated by instrumented application code in -// `__coverage__`. Every flush serializes the counters and resets them, so -// each flush reports only the delta since the previous one. The statement, -// function and branch maps are large, so they are only reported with the -// first report of each file. +// Reports `__coverage__` as a delta: reading resets the counters, maps are sent once per file. export class CoverageScript { private _global: typeof globalThis; private _sessionId: string; @@ -33,10 +29,7 @@ export class CoverageScript { this._global = global; this._sessionId = sessionId; (global as any)[collectName] = () => this.collect(); - // Counters die with the document, and calls made during unload are not - // delivered, so stash the delta in localStorage. Playwright picks it up - // from any same origin document, including one it opens itself after the - // page that produced the stash is gone. + // Counters die with the document and unload-time calls are not delivered. global.addEventListener('pagehide', () => this._stashCurrent()); } @@ -58,8 +51,7 @@ export class CoverageScript { const s = takeCounters(fileCoverage.s); const f = takeCounters(fileCoverage.f); const b = takeBranchCounters(fileCoverage.b); - // Report every file once to account for the files that were never hit, - // afterwards only report the files that were hit since the last report. + // Every file is reported once, so that the never hit ones are accounted for. const isFirstReport = !this._reportedFiles.has(file); if (!isFirstReport && !s && !f && !b) continue; @@ -85,8 +77,7 @@ export class CoverageScript { if (!delta) return; try { - // A stash can be picked up by several documents at once, so it carries an - // id that lets the recorder discard the copies. + // Several documents can pick up the same stash, the id discards the copies. const id = ++this._stashOrdinal + '-' + Math.random().toString(36).slice(2); const key = kCoverageStashPrefix + this._sessionId + '.' + id; this._global.localStorage.setItem(key, JSON.stringify({ id, data: delta })); @@ -95,10 +86,8 @@ export class CoverageScript { } } -// Takes the stashes left by this session and discards the ones left by other -// sessions, e.g. by a previous run reusing a persistent profile. Playwright -// also evaluates this on its own page to reach the origins that have no page -// left to relay them. +// Also evaluated by Playwright itself on the origins that have no page left. +// Stashes of other sessions are stale, e.g. left by a previous run in a persistent profile. export function takeCoverageStashes(global: typeof globalThis, sessionId: string): string[] { const result: string[] = []; try { @@ -121,7 +110,6 @@ export function takeCoverageStashes(global: typeof globalThis, sessionId: string return result; } -// Returns the non-zero counters and resets them, or undefined when there are none. function takeCounters(counters: { [key: string]: number }): { [key: string]: number } | undefined { let result: { [key: string]: number } | undefined; for (const key of Object.keys(counters)) { @@ -136,7 +124,7 @@ function takeCounters(counters: { [key: string]: number }): { [key: string]: num return result; } -// Branch counters are positional, so hit branches are reported with the whole array. +// Branch counters are positional, so a hit branch is reported with the whole array. function takeBranchCounters(counters: { [key: string]: number[] }): { [key: string]: number[] } | undefined { let result: { [key: string]: number[] } | undefined; for (const key of Object.keys(counters)) { diff --git a/packages/isomorphic/istanbulCoverage.ts b/packages/isomorphic/istanbulCoverage.ts index 43b40558f12d5..8a1b8bd24d454 100644 --- a/packages/isomorphic/istanbulCoverage.ts +++ b/packages/isomorphic/istanbulCoverage.ts @@ -14,9 +14,7 @@ * limitations under the License. */ -// The istanbul coverage format, as produced by istanbul-instrumented code in -// the `__coverage__` global and consumed by istanbul report tooling. -// Mirrors the shapes from istanbul-lib-coverage. +// The istanbul coverage format, mirrors the shapes from istanbul-lib-coverage. export type IstanbulLocation = { line: number; @@ -55,21 +53,17 @@ export type IstanbulFileCoverage = { export type IstanbulCoverage = { [file: string]: IstanbulFileCoverage }; -// Collected counters, reported as a delta since the previous report. The maps -// are only sent with the first report of each file, so they are optional here. +// The maps are only sent with the first report of each file. export type IstanbulFileCoverageDelta = Partial & Pick; export type IstanbulCoverageDelta = { [file: string]: IstanbulFileCoverageDelta }; -// A single report from a document. Chunks stashed in the page storage carry an -// id, because they can be picked up by several documents at once. export type IstanbulCoverageChunk = { data: IstanbulCoverageDelta; id?: string; }; -// Merges istanbul hit counts: counters add up, map entries are taken from the -// first report that carries them. Assumes all data comes from the same build. +// Counters add up, maps are taken from the first report that carries them. export function mergeIstanbulCoverage(into: Map, data: IstanbulCoverageDelta) { for (const [file, fileCov] of Object.entries(data)) { let existing = into.get(file); @@ -77,7 +71,7 @@ export function mergeIstanbulCoverage(into: Map, d existing = { path: fileCov.path, statementMap: {}, fnMap: {}, branchMap: {}, s: {}, f: {}, b: {} }; into.set(file, existing); } - // A counters-only report can arrive first if the report carrying the maps was lost. + // The maps can arrive late if the report that carried them was lost. if (fileCov.statementMap && !Object.keys(existing.statementMap).length) { existing.statementMap = fileCov.statementMap; existing.fnMap = fileCov.fnMap || {}; diff --git a/packages/playwright-core/src/server/browserContext.ts b/packages/playwright-core/src/server/browserContext.ts index 0d00032a59959..a57e34c20e094 100644 --- a/packages/playwright-core/src/server/browserContext.ts +++ b/packages/playwright-core/src/server/browserContext.ts @@ -657,8 +657,7 @@ export abstract class BrowserContext extends Sdk return result; } - // Visits each origin in a page of its own that is hidden from the client, to - // reach the storage of the origins that have no page of their own. + // Reaches the storage of the origins that have no page, in a hidden page. async visitOrigins(progress: Progress, origins: Set, callback: (frame: frames.Frame, origin: string) => Promise) { const page = await this.newPage(progress, true /* forStorageState */); try { diff --git a/packages/playwright-core/src/server/coverageRecorder.ts b/packages/playwright-core/src/server/coverageRecorder.ts index 16b5256d158b2..a39fb5da4786f 100644 --- a/packages/playwright-core/src/server/coverageRecorder.ts +++ b/packages/playwright-core/src/server/coverageRecorder.ts @@ -38,8 +38,7 @@ export class CoverageRecorder { this._context = context; } - // Scopes the stashes in the page storage to this context, so that the ones - // left behind by a previous run in a persistent profile are discarded. + // Scopes the stashes in the page storage to this context. private _sessionId() { return this._context.guid; } @@ -77,7 +76,6 @@ export class CoverageRecorder { onPageClose(page: Page) { if (!this._active) return; - // The stash of a closed page stays in the storage of its origin. for (const frame of page.frames()) this._noteOrigin(frame.origin()); } @@ -93,7 +91,6 @@ export class CoverageRecorder { await this._harvestOriginsWithoutPage(progress); } - // Collects pending coverage from all pages and returns it serialized, resetting the accumulator. async take(progress: Progress): Promise { if (!this._active) return; @@ -104,13 +101,12 @@ export class CoverageRecorder { } async collectFromPage(page: Page) { - // A page that is going away may never answer, give up on it when it does. + // A page that is going away may never answer. await Promise.race([page.closedPromise, this._collectFromPage(page)]); } private async _collectFromPage(page: Page) { for (const frame of page.frames()) { - // A document of this origin may leave a stash behind when it goes away. this._noteOrigin(frame.origin()); const chunks: string[] = await frame.nonStallingEvaluateInExistingContext(coverageCollectExpression, 'main').catch(() => []); for (const json of chunks || []) @@ -118,8 +114,7 @@ export class CoverageRecorder { } } - // Pages relay the stashes of their own origin, so only the origins that are - // left without a page need a page of their own to relay them. + // Pages relay the stashes of their own origin, the rest need a page of their own. private async _harvestOriginsWithoutPage(progress: Progress) { if (!this._stashOrigins.size || this._context.isClosingOrClosed()) return; @@ -131,7 +126,7 @@ export class CoverageRecorder { liveOrigins.add(origin); } } - // The origins that still have a page were just drained by the sweep above. + // The origins that still have a page were drained by the sweep above. const origins = new Set([...this._stashOrigins].filter(origin => !liveOrigins.has(origin))); this._stashOrigins.clear(); if (!origins.size) diff --git a/packages/playwright-core/src/server/trace/recorder/tracing.ts b/packages/playwright-core/src/server/trace/recorder/tracing.ts index a96f33daaea20..699c43ecb8cdd 100644 --- a/packages/playwright-core/src/server/trace/recorder/tracing.ts +++ b/packages/playwright-core/src/server/trace/recorder/tracing.ts @@ -403,15 +403,13 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps await this._coverageRecorder.flush(progress); } - // Best-effort collection before the page is gone, no-op unless collecting. async flushCoverageBeforePageClose(page: Page) { if (!this._coverageRecorder?.active()) return; await this._coverageRecorder.collectFromPage(page); } - // Collects pending coverage into a file that will become the "coverage.json" - // entry of the chunk, while the pages can still be evaluated in. + // Collected before the chunk stops, while the pages can still be evaluated in. private async _takeCoverage(progress: Progress, mode: TracingTracingStopChunkParams['mode']): Promise { if (!this._state?.recording || !this._state.options.coverage || !this._coverageRecorder) return; @@ -527,8 +525,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps await this._captureScreenshot(progress, page, phase); if (options?.snapshotAria) await this._captureAriaSnapshot(progress, page, phase); - // Pull the counters as the actions go, so that an action is the most that - // can be lost when the page goes away without a chance to stash them. + // Collecting as the actions go, an action is the most that can be lost. if (phase === 'after' && this._coverageRecorder?.active()) { try { await progress.race(this._coverageRecorder.collectFromPage(page)); diff --git a/packages/playwright/src/reporters/coverage.ts b/packages/playwright/src/reporters/coverage.ts index 80e0d4e4a0843..af62ed4d9e606 100644 --- a/packages/playwright/src/reporters/coverage.ts +++ b/packages/playwright/src/reporters/coverage.ts @@ -100,8 +100,7 @@ class CoverageReporter implements ReporterV2 { writeLine(lines.join('\n')); } - // Istanbul report libraries are not shipped with Playwright. Use them for the - // html report when the project has them installed. + // Istanbul report libraries are not shipped with Playwright, use the project's if any. private _tryWriteHtmlReport(mergedJson: IstanbulCoverage, outputDir: string): boolean { try { const resolveFrom = (name: string) => require.resolve(name, { paths: [this._options.configDir] }); From aa76e03069ea0fe181f044da899691c2abc65061 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Fri, 11 Sep 2026 17:01:26 -0700 Subject: [PATCH 08/21] chore(coverage): collect through instrumentation, name the trace entry Collecting after each action used to hang off the snapshot capture, which is gated on the snapshot metainfo of the method, and the page close case fell outside that gate and reached into tracing from page.ts. Both now go through instrumentation: onAfterCall collects for the page the action ran in, and a new onPageWillClose collects before the page is gone, which also excludes the pages Playwright opens for itself. Name the trace entry trace.coverage, so that merging the chunks of several contexts into the trace of a test keeps every one of them instead of silently dropping all but the first. Do not collect for a chunk that is being discarded, and write no entry when there is nothing to report. Sweep the pages and their frames concurrently, and keep the coverage stash out of browserContext.storageState. --- docs/src/api/class-tracing.md | 2 +- packages/injected/src/coverageScript.ts | 16 ++-- packages/injected/src/storageScript.ts | 5 +- packages/isomorphic/istanbulCoverage.ts | 7 ++ packages/playwright-client/types/types.d.ts | 2 +- .../src/server/coverageRecorder.ts | 85 +++++++++---------- .../src/server/instrumentation.ts | 2 + packages/playwright-core/src/server/page.ts | 4 +- .../src/server/trace/recorder/tracing.ts | 64 +++++++------- packages/playwright-core/types/types.d.ts | 2 +- packages/playwright/src/reporters/coverage.ts | 73 ++++++++-------- tests/config/utils.ts | 12 +++ tests/library/browsertype-connect.spec.ts | 2 +- tests/library/coverage.spec.ts | 28 +++--- tests/playwright-test/coverage.spec.ts | 38 ++++++--- 15 files changed, 189 insertions(+), 153 deletions(-) diff --git a/docs/src/api/class-tracing.md b/docs/src/api/class-tracing.md index ee4e653e48f55..be3ac85b35da5 100644 --- a/docs/src/api/class-tracing.md +++ b/docs/src/api/class-tracing.md @@ -151,7 +151,7 @@ a timeline preview. * langs: js - `coverage` <[boolean]> -**Experimental.** Whether to collect code coverage from istanbul-instrumented application code. Build the application with an istanbul instrumentation plugin, for example [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right before navigations, and stores them as `coverage.json` in istanbul format inside the trace file. Use [`method: Tracing.flushCoverage`] to collect counters from a page that is about to be closed by in-page script. +**Experimental.** Whether to collect code coverage from istanbul-instrumented application code. Build the application with an istanbul instrumentation plugin, for example [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right before navigations, and stores them in istanbul format inside the trace file. Use [`method: Tracing.flushCoverage`] to collect counters from a page that is about to be closed by in-page script. ### option: Tracing.start.snapshots * since: v1.12 diff --git a/packages/injected/src/coverageScript.ts b/packages/injected/src/coverageScript.ts index fb5dcdfcf06d8..fda4199fa9f61 100644 --- a/packages/injected/src/coverageScript.ts +++ b/packages/injected/src/coverageScript.ts @@ -14,9 +14,9 @@ * limitations under the License. */ -import type { IstanbulCoverage, IstanbulCoverageDelta, IstanbulFileCoverageDelta } from '@isomorphic/istanbulCoverage'; +import { kCoverageStashPrefix } from '@isomorphic/istanbulCoverage'; -export const kCoverageStashPrefix = '__pwCoverage.'; +import type { IstanbulCoverage, IstanbulCoverageDelta, IstanbulFileCoverageDelta } from '@isomorphic/istanbulCoverage'; // Reports `__coverage__` as a delta: reading resets the counters, maps are sent once per file. export class CoverageScript { @@ -34,7 +34,7 @@ export class CoverageScript { } collect(): string[] { - const chunks = this._takeStashes(); + const chunks = takeCoverageStashes(this._global, this._sessionId); const delta = this._takeCurrent(); if (delta) chunks.push(JSON.stringify({ data: delta })); @@ -46,7 +46,6 @@ export class CoverageScript { if (!coverage) return undefined; const delta: IstanbulCoverageDelta = {}; - let hasFiles = false; for (const [file, fileCoverage] of Object.entries(coverage)) { const s = takeCounters(fileCoverage.s); const f = takeCounters(fileCoverage.f); @@ -63,13 +62,8 @@ export class CoverageScript { this._reportedFiles.add(file); } delta[file] = entry; - hasFiles = true; } - return hasFiles ? delta : undefined; - } - - private _takeStashes(): string[] { - return takeCoverageStashes(this._global, this._sessionId); + return Object.keys(delta).length ? delta : undefined; } private _stashCurrent() { @@ -134,7 +128,7 @@ function takeBranchCounters(counters: { [key: string]: number[] }): { [key: stri if (!result) result = {}; result[key] = counts.slice(); - counters[key] = counts.map(() => 0); + counts.fill(0); } return result; } diff --git a/packages/injected/src/storageScript.ts b/packages/injected/src/storageScript.ts index f7e4d3780923f..efb3054cef573 100644 --- a/packages/injected/src/storageScript.ts +++ b/packages/injected/src/storageScript.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { kCoverageStashPrefix } from '@isomorphic/istanbulCoverage'; import { parseEvaluationResultValue, serializeAsCallArgument, typedArrayToBase64 } from '@isomorphic/utilityScriptSerializers'; import type { IndexedDBDatabase, OPFSEntry, OriginStorage, SetOriginStorage } from '@protocol/structs'; @@ -170,7 +171,9 @@ export class StorageScript { } async collect(record: { indexedDB: boolean, opfs: boolean }): Promise { - const localStorage = Object.keys(this._global.localStorage).map(name => ({ name, value: this._global.localStorage.getItem(name)! })); + const localStorage = Object.keys(this._global.localStorage) + .filter(name => !name.startsWith(kCoverageStashPrefix)) + .map(name => ({ name, value: this._global.localStorage.getItem(name)! })); const result: SerializedStorage = { localStorage }; if (record.indexedDB) { try { diff --git a/packages/isomorphic/istanbulCoverage.ts b/packages/isomorphic/istanbulCoverage.ts index 8a1b8bd24d454..e9e80b0f558c2 100644 --- a/packages/isomorphic/istanbulCoverage.ts +++ b/packages/isomorphic/istanbulCoverage.ts @@ -58,11 +58,18 @@ export type IstanbulFileCoverageDelta = Partial & Pick): IstanbulCoverage { + return Object.fromEntries([...coverage.entries()].sort(([a], [b]) => a.localeCompare(b))); +} + // Counters add up, maps are taken from the first report that carries them. export function mergeIstanbulCoverage(into: Map, data: IstanbulCoverageDelta) { for (const [file, fileCov] of Object.entries(data)) { diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts index fcfe492ef689f..c2925fdc0c53e 100644 --- a/packages/playwright-client/types/types.d.ts +++ b/packages/playwright-client/types/types.d.ts @@ -23286,7 +23286,7 @@ export interface Tracing { * [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or * [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the * `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right - * before navigations, and stores them as `coverage.json` in istanbul format inside the trace file. Use + * before navigations, and stores them in istanbul format inside the trace file. Use * [tracing.flushCoverage()](https://playwright.dev/docs/api/class-tracing#tracing-flush-coverage) to collect counters * from a page that is about to be closed by in-page script. */ diff --git a/packages/playwright-core/src/server/coverageRecorder.ts b/packages/playwright-core/src/server/coverageRecorder.ts index a39fb5da4786f..7d8eebf8703f2 100644 --- a/packages/playwright-core/src/server/coverageRecorder.ts +++ b/packages/playwright-core/src/server/coverageRecorder.ts @@ -14,11 +14,11 @@ * limitations under the License. */ -import { mergeIstanbulCoverage } from '@isomorphic/istanbulCoverage'; +import { mergeIstanbulCoverage, sortedIstanbulCoverage } from '@isomorphic/istanbulCoverage'; import * as rawCoverageSource from '../generated/coverageScriptSource'; import type { BrowserContext } from './browserContext'; -import type { Page } from './page'; +import type { InitScript, Page } from './page'; import type { Progress } from './progress'; import type { IstanbulCoverageChunk, IstanbulFileCoverage } from '@isomorphic/istanbulCoverage'; @@ -31,51 +31,44 @@ export class CoverageRecorder { private _coverage = new Map(); private _stashedChunkIds = new Set(); private _stashOrigins = new Set(); - private _installed = false; - private _active = false; + private _initScript: InitScript | undefined; constructor(context: BrowserContext) { this._context = context; } - // Scopes the stashes in the page storage to this context. - private _sessionId() { - return this._context.guid; - } - - private _bootstrapSource() { + private _moduleExpression(call: string) { return `(() => { const module = {}; ${rawCoverageSource.source} - new (module.exports.CoverageScript())(window, ${JSON.stringify(kCoverageCollectName)}, ${JSON.stringify(this._sessionId())}); + ${call} })()`; } - activate() { - this._active = true; - } - - deactivate() { - this._active = false; - } - - active() { - return this._active; + // The stashes in the page storage are scoped to the context that collects them. + private _sessionId() { + return JSON.stringify(this._context.guid); } async install(progress: Progress) { - if (this._installed) + if (this._initScript) return; - this._installed = true; - const bootstrapSource = this._bootstrapSource(); - await this._context.addInitScript(progress, bootstrapSource); + const source = this._moduleExpression(`new (module.exports.CoverageScript())(window, ${JSON.stringify(kCoverageCollectName)}, ${this._sessionId()});`); + this._initScript = await this._context.addInitScript(progress, source); // Init scripts only affect future documents, bootstrap the existing ones. - await progress.race(this._context.safeNonStallingEvaluateInAllFrames(bootstrapSource, 'main')); + await progress.race(this._context.safeNonStallingEvaluateInAllFrames(source, 'main')); + } + + async uninstall() { + const initScript = this._initScript; + this._initScript = undefined; + this._stashOrigins.clear(); + this._stashedChunkIds.clear(); + this._coverage.clear(); + await initScript?.dispose().catch(() => {}); } onPageClose(page: Page) { - if (!this._active) - return; for (const frame of page.frames()) this._noteOrigin(frame.origin()); } @@ -86,32 +79,40 @@ export class CoverageRecorder { } async flush(progress: Progress) { - for (const page of this._context.pages()) - await progress.race(this.collectFromPage(page)); + await progress.race(Promise.all(this._context.pages().map(page => this.collectFromPage(page)))); await this._harvestOriginsWithoutPage(progress); } - async take(progress: Progress): Promise { - if (!this._active) + async take(progress: Progress, mode: 'keep' | 'discard'): Promise { + if (mode === 'discard') { + // The coverage of a chunk is discarded together with it. + this._coverage.clear(); return; + } await this.flush(progress); - const sorted = Object.fromEntries([...this._coverage.entries()].sort(([a], [b]) => a.localeCompare(b))); + this._stashedChunkIds.clear(); + if (!this._coverage.size) + return; + const sorted = sortedIstanbulCoverage(this._coverage); this._coverage.clear(); return JSON.stringify(sorted); } async collectFromPage(page: Page) { + // Pages of Playwright's own making never run the application under test. + if (page.isStorageStatePage) + return; // A page that is going away may never answer. await Promise.race([page.closedPromise, this._collectFromPage(page)]); } private async _collectFromPage(page: Page) { - for (const frame of page.frames()) { + await Promise.all(page.frames().map(async frame => { this._noteOrigin(frame.origin()); - const chunks: string[] = await frame.nonStallingEvaluateInExistingContext(coverageCollectExpression, 'main').catch(() => []); - for (const json of chunks || []) + const chunks: string[] = await frame.nonStallingRawEvaluateInExistingMainContext(coverageCollectExpression).catch(() => []); + for (const json of chunks) this._append(json); - } + })); } // Pages relay the stashes of their own origin, the rest need a page of their own. @@ -131,14 +132,10 @@ export class CoverageRecorder { this._stashOrigins.clear(); if (!origins.size) return; - const takeStashesExpression = `(() => { - const module = {}; - ${rawCoverageSource.source} - return (module.exports.takeCoverageStashes())(window, ${JSON.stringify(this._sessionId())}); - })()`; + const source = this._moduleExpression(`return (module.exports.takeCoverageStashes())(window, ${this._sessionId()});`); await this._context.visitOrigins(progress, origins, async frame => { - const chunks: string[] = await frame.evaluateExpression(progress, takeStashesExpression, { world: 'main' }).catch(() => []); - for (const json of chunks || []) + const chunks: string[] = await frame.evaluateExpression(progress, source, { world: 'main' }).catch(() => []); + for (const json of chunks) this._append(json); }); } diff --git a/packages/playwright-core/src/server/instrumentation.ts b/packages/playwright-core/src/server/instrumentation.ts index a2de6a9806fbf..6f82daf56ab76 100644 --- a/packages/playwright-core/src/server/instrumentation.ts +++ b/packages/playwright-core/src/server/instrumentation.ts @@ -111,6 +111,7 @@ export interface Instrumentation { onCallLog(sdkObject: SdkObject, metadata: CallMetadata, logName: string, message: string): void; onAfterCall(progress: Progress, sdkObject: SdkObject): Promise; onPageOpen(page: Page): void; + onPageWillClose(page: Page): Promise; onPageClose(page: Page): void; onBrowserOpen(browser: Browser): void; onBrowserClose(browser: Browser): void; @@ -124,6 +125,7 @@ export interface InstrumentationListener { onCallLog?(sdkObject: SdkObject, metadata: CallMetadata, logName: string, message: string): void; onAfterCall?(progress: Progress, sdkObject: SdkObject): Promise; onPageOpen?(page: Page): void; + onPageWillClose?(page: Page): Promise; onPageClose?(page: Page): void; onBrowserOpen?(browser: Browser): void; onBrowserClose?(browser: Browser): void; diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts index bebb3aa54bb44..ea7beb3422524 100644 --- a/packages/playwright-core/src/server/page.ts +++ b/packages/playwright-core/src/server/page.ts @@ -821,7 +821,6 @@ export class Page extends SdkObject { } async close(progress: Progress, options: { reason?: string } = {}) { - await progress.race(this.browserContext.tracing.flushCoverageBeforePageClose(this).catch(() => {})); await progress.race(this._close(options)); } @@ -832,6 +831,9 @@ export class Page extends SdkObject { if (options.reason) this._closeReason = options.reason; + if (!this.isStorageStatePage) + await this.instrumentation.onPageWillClose(this).catch(() => {}); + await this.screencast.handlePageOrContextClose(); if (this._lifecycle !== 'closing') { diff --git a/packages/playwright-core/src/server/trace/recorder/tracing.ts b/packages/playwright-core/src/server/trace/recorder/tracing.ts index 699c43ecb8cdd..06afcec83e55b 100644 --- a/packages/playwright-core/src/server/trace/recorder/tracing.ts +++ b/packages/playwright-core/src/server/trace/recorder/tracing.ts @@ -187,10 +187,8 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps // Tracing is 10x bigger if we include scripts in every trace. if (options.snapshotDom) this._harTracer.start({ omitScripts: !options.live }); - if (options.coverage && this._context instanceof BrowserContext) { + if (options.coverage && this._context instanceof BrowserContext) this._coverageRecorder ??= new CoverageRecorder(this._context); - this._coverageRecorder.activate(); - } this._started = true; } @@ -348,7 +346,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps this._closeAllGroups(); this._harTracer.stop(); this.flushHarEntries(); - this._coverageRecorder?.deactivate(); + await this._coverageRecorder?.uninstall(); await this._fs.sync().finally(() => { this._state = undefined; }); @@ -397,24 +395,24 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps this._groupEnd(); } - async flushCoverage(progress: Progress) { - if (!this._coverageRecorder?.active()) - throw new Error(`Coverage collection has not been started. Pass the "coverage" option to tracing.start().`); - await this._coverageRecorder.flush(progress); + private _activeCoverageRecorder(): CoverageRecorder | undefined { + return this._state?.options.coverage ? this._coverageRecorder : undefined; } - async flushCoverageBeforePageClose(page: Page) { - if (!this._coverageRecorder?.active()) - return; - await this._coverageRecorder.collectFromPage(page); + async flushCoverage(progress: Progress) { + const recorder = this._activeCoverageRecorder(); + if (!recorder) + throw new Error(`Coverage collection has not been started. Pass the "coverage" option to tracing.start().`); + await recorder.flush(progress); } // Collected before the chunk stops, while the pages can still be evaluated in. private async _takeCoverage(progress: Progress, mode: TracingTracingStopChunkParams['mode']): Promise { - if (!this._state?.recording || !this._state.options.coverage || !this._coverageRecorder) + const recorder = this._activeCoverageRecorder(); + if (!this._state?.recording || !recorder) return; - const coverageJson = await this._coverageRecorder.take(progress); - if (mode === 'discard' || coverageJson === undefined) + const coverageJson = await recorder.take(progress, mode === 'discard' ? 'discard' : 'keep'); + if (coverageJson === undefined) return; const coverageFile = path.join(this._state.tracesDir, `${this._state.traceName}-coverage-${this._state.chunkOrdinal}.json`); this._fs.writeFile(coverageFile, coverageJson); @@ -489,7 +487,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps entries.push({ name: 'trace.trace', value: this._state.traceFile }); entries.push({ name: 'trace.network', value: newNetworkFile }); if (coverageFile) - entries.push({ name: 'coverage.json', value: coverageFile }); + entries.push({ name: 'trace.coverage', value: coverageFile }); for (const file of new Set([...this._state.chunkFiles, ...this._state.crossChunkFiles])) entries.push({ name: file, value: path.join(this._state.tracesDir, file) }); @@ -525,13 +523,6 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps await this._captureScreenshot(progress, page, phase); if (options?.snapshotAria) await this._captureAriaSnapshot(progress, page, phase); - // Collecting as the actions go, an action is the most that can be lost. - if (phase === 'after' && this._coverageRecorder?.active()) { - try { - await progress.race(this._coverageRecorder.collectFromPage(page)); - } catch { - } - } } private _shouldCaptureAtPhase(metadata: CallMetadata, phase: trace.ActionPhase) { @@ -604,18 +595,31 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps this._appendTraceEvent(event); } - onAfterCall(progress: Progress, sdkObject: SdkObject) { - // IMPORTANT: no awaits in this method, this._appendTraceEvent must be called synchronously. + async onAfterCall(progress: Progress, sdkObject: SdkObject) { + // IMPORTANT: no awaits before _appendTraceEvent, it must be called synchronously. const { metadata } = progress; if (!this._state?.callsInProgress.has(metadata.id)) - return Promise.resolve(); - this._state?.callsInProgress.delete(metadata.id); + return; + this._state.callsInProgress.delete(metadata.id); const event = createAfterActionTraceEvent(metadata); if (!event) - return Promise.resolve(); + return; this._temporarilyDisableThrottling(sdkObject.attribution.page); this._appendTraceEvent(event); - return this._captureSnapshot(progress, sdkObject, 'after'); + await this._captureSnapshot(progress, sdkObject, 'after'); + // Collecting as the actions go, an action is the most that can be lost. + const page = sdkObject.attribution.page; + const recorder = this._activeCoverageRecorder(); + if (page && recorder) { + try { + await progress.race(recorder.collectFromPage(page)); + } catch { + } + } + } + + async onPageWillClose(page: Page) { + await this._activeCoverageRecorder()?.collectFromPage(page).catch(() => {}); } onEntryStarted(entry: har.Entry) { @@ -715,7 +719,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps } onPageClose(page: Page) { - this._coverageRecorder?.onPageClose(page); + this._activeCoverageRecorder()?.onPageClose(page); const event: trace.EventTraceEvent = { type: 'event', time: monotonicTime(), diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index fcfe492ef689f..c2925fdc0c53e 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -23286,7 +23286,7 @@ export interface Tracing { * [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or * [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the * `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right - * before navigations, and stores them as `coverage.json` in istanbul format inside the trace file. Use + * before navigations, and stores them in istanbul format inside the trace file. Use * [tracing.flushCoverage()](https://playwright.dev/docs/api/class-tracing#tracing-flush-coverage) to collect counters * from a page that is about to be closed by in-page script. */ diff --git a/packages/playwright/src/reporters/coverage.ts b/packages/playwright/src/reporters/coverage.ts index af62ed4d9e606..f392f5b18d06f 100644 --- a/packages/playwright/src/reporters/coverage.ts +++ b/packages/playwright/src/reporters/coverage.ts @@ -17,8 +17,9 @@ import fs from 'fs'; import path from 'path'; -import { mergeIstanbulCoverage } from '@isomorphic/istanbulCoverage'; +import { mergeIstanbulCoverage, sortedIstanbulCoverage } from '@isomorphic/istanbulCoverage'; import { ZipFile } from '@utils/zipFile'; +import { terminalScreen } from './base'; import { resolveReporterOutputPath } from '../util'; import type { IstanbulCoverage, IstanbulFileCoverage } from '@isomorphic/istanbulCoverage'; @@ -55,15 +56,27 @@ class CoverageReporter implements ReporterV2 { async onEnd() { const coverage = new Map(); - for (const traceFile of this._traceFiles) { - const zipFile = new ZipFile(traceFile); - try { - const entries = await zipFile.entries(); - if (entries.includes('coverage.json')) - mergeIstanbulCoverage(coverage, JSON.parse((await zipFile.read('coverage.json')).toString('utf8'))); - } catch { - } finally { - zipFile.close(); + // Traces of several contexts are merged into one, each keeping its own coverage entry. + const traceFiles = [...this._traceFiles]; + while (traceFiles.length) { + const batch = traceFiles.splice(0, 16).map(async traceFile => { + const zipFile = new ZipFile(traceFile); + try { + const entries = (await zipFile.entries()).filter(entry => entry.match(/(^|-)trace\.coverage$/)); + return await Promise.all(entries.map(entry => zipFile.read(entry))); + } catch { + return []; + } finally { + zipFile.close(); + } + }); + for (const buffers of await Promise.all(batch)) { + for (const buffer of buffers) { + try { + mergeIstanbulCoverage(coverage, JSON.parse(buffer.toString('utf8'))); + } catch { + } + } } } @@ -80,7 +93,7 @@ class CoverageReporter implements ReporterV2 { const outputDir = resolveReporterOutputPath('coverage', this._options.configDir, this._options.outputDir); await fs.promises.mkdir(outputDir, { recursive: true }); - const mergedJson: IstanbulCoverage = Object.fromEntries([...coverage.entries()].sort(([a], [b]) => a.localeCompare(b))); + const mergedJson = sortedIstanbulCoverage(coverage); await fs.promises.writeFile(path.join(outputDir, 'coverage-final.json'), JSON.stringify(mergedJson)); await fs.promises.writeFile(path.join(outputDir, 'lcov.info'), lcovReport(mergedJson)); const hasHtml = this._tryWriteHtmlReport(mergedJson, outputDir); @@ -127,32 +140,23 @@ function computeSummary(mergedJson: IstanbulCoverage) { const functions: Metric = { covered: 0, total: 0 }; const lines: Metric = { covered: 0, total: 0 }; for (const fileCov of Object.values(mergedJson)) { - for (const count of Object.values(fileCov.s)) { - ++statements.total; - if (count > 0) - ++statements.covered; - } - for (const count of Object.values(fileCov.f)) { - ++functions.total; - if (count > 0) - ++functions.covered; - } - for (const counts of Object.values(fileCov.b)) { - for (const count of counts) { - ++branches.total; - if (count > 0) - ++branches.covered; - } - } - for (const count of lineCoverage(fileCov).values()) { - ++lines.total; - if (count > 0) - ++lines.covered; - } + countHits(statements, Object.values(fileCov.s)); + countHits(functions, Object.values(fileCov.f)); + for (const counts of Object.values(fileCov.b)) + countHits(branches, counts); + countHits(lines, lineCoverage(fileCov).values()); } return { statements, branches, functions, lines }; } +function countHits(metric: Metric, counts: Iterable) { + for (const count of counts) { + ++metric.total; + if (count > 0) + ++metric.covered; + } +} + function lineCoverage(fileCov: IstanbulFileCoverage): Map { const lines = new Map(); for (const [key, statement] of Object.entries(fileCov.statementMap)) { @@ -210,8 +214,7 @@ function lcovReport(mergedJson: IstanbulCoverage): string { } function writeLine(line: string) { - // eslint-disable-next-line no-restricted-properties - process.stdout.write(line + '\n'); + terminalScreen.stdout.write(line + '\n'); } export default CoverageReporter; diff --git a/tests/config/utils.ts b/tests/config/utils.ts index b28fc36365428..0a4148ba7747a 100644 --- a/tests/config/utils.ts +++ b/tests/config/utils.ts @@ -104,6 +104,18 @@ export function suppressCertificateWarning() { }; } +// Traces of several contexts are merged into one, each keeping its own coverage entry. +export async function parseTraceCoverage(file: string): Promise { + const { resources } = await parseTraceRaw(file); + const entries = [...resources.keys()].filter(name => name.match(/(^|-)trace\.coverage$/)); + if (!entries.length) + return undefined; + const coverage = new Map(); + for (const entry of entries) + iso.mergeIstanbulCoverage(coverage, JSON.parse(resources.get(entry)!.toString())); + return iso.sortedIstanbulCoverage(coverage); +} + export async function parseTraceRaw(file: string): Promise<{ events: any[], resources: Map, actions: string[], actionObjects: ActionTraceEvent[], stacks: Map }> { const zipFS = new utils.ZipFile(file); const resources = new Map(); diff --git a/tests/library/browsertype-connect.spec.ts b/tests/library/browsertype-connect.spec.ts index 4db584f5187ff..f73f647636c74 100644 --- a/tests/library/browsertype-connect.spec.ts +++ b/tests/library/browsertype-connect.spec.ts @@ -757,7 +757,7 @@ for (const kind of ['launchServer', 'run-server'] as const) { await browser.close(); const { resources } = await parseTraceRaw(testInfo.outputPath('trace1.zip')); - const data = JSON.parse(resources.get('coverage.json')!.toString()); + const data = JSON.parse(resources.get('trace.coverage')!.toString()); expect(data['a.js'].s['0']).toBe(3); }); diff --git a/tests/library/coverage.spec.ts b/tests/library/coverage.spec.ts index 12201f672dc31..3e4fbc65bc3c9 100644 --- a/tests/library/coverage.spec.ts +++ b/tests/library/coverage.spec.ts @@ -15,7 +15,7 @@ */ import { browserTest as it, expect } from '../config/browserTest'; -import { parseTraceRaw } from '../config/utils'; +import { parseTraceCoverage } from '../config/utils'; const fileCoverage = (file: string, s0: number) => ({ [file]: { @@ -31,12 +31,6 @@ const fileCoverage = (file: string, s0: number) => ({ const coverageScript = (file: string, s0: number) => ``; -async function readCoverage(traceFile: string): Promise { - const { resources } = await parseTraceRaw(traceFile); - const entry = resources.get('coverage.json'); - return entry ? JSON.parse(entry.toString()) : undefined; -} - it('should collect istanbul coverage into the trace', async ({ browser }, testInfo) => { const context = await browser.newContext(); await context.tracing.start({ coverage: true }); @@ -54,7 +48,7 @@ it('should collect istanbul coverage into the trace', async ({ browser }, testIn await context.tracing.stop({ path: traceFile }); await context.close(); - const data = await readCoverage(traceFile); + const data = await parseTraceCoverage(traceFile); expect(data['a.js'].s['0']).toBe(3); expect(data['b.js'].s['0']).toBe(2); }); @@ -76,11 +70,11 @@ it('should collect coverage per trace chunk', async ({ browser }, testInfo) => { await context.tracing.stop(); await context.close(); - const data1 = await readCoverage(traceFile1); + const data1 = await parseTraceCoverage(traceFile1); expect(data1['a.js'].s['0']).toBe(5); expect(data1['b.js']).toBe(undefined); - const data2 = await readCoverage(traceFile2); + const data2 = await parseTraceCoverage(traceFile2); expect(data2['b.js'].s['0']).toBe(7); expect(data2['a.js']).toBe(undefined); }); @@ -127,7 +121,7 @@ it('should accumulate counters across flushes and keep never hit files', async ( await context.tracing.stop({ path: traceFile }); await context.close(); - const data = await readCoverage(traceFile); + const data = await parseTraceCoverage(traceFile); expect(data['a.js'].s['0']).toBe(7); expect(Object.keys(data['a.js'].statementMap)).toEqual(['0']); }); @@ -159,7 +153,7 @@ it('should collect coverage of a page closed by in-page script', async ({ browse await context.tracing.stop({ path: traceFile }); await context.close(); - const data = await readCoverage(traceFile); + const data = await parseTraceCoverage(traceFile); expect(data['popup.js'].s['0']).toBe(5); }); @@ -181,7 +175,7 @@ it('should not double count a stash picked up twice', async ({ browser, server } await context.tracing.stop({ path: traceFile }); await context.close(); - const data = await readCoverage(traceFile); + const data = await parseTraceCoverage(traceFile); expect(data['stashed.js'].s['0']).toBe(4); }); @@ -202,8 +196,8 @@ it('should discard stashes of other sessions', async ({ browser, server }, testI await context.tracing.stop({ path: traceFile }); await context.close(); - const data = await readCoverage(traceFile); - expect(data['stale.js']).toBe(undefined); + // The stale stash was the only coverage, so nothing was collected at all. + expect(await parseTraceCoverage(traceFile)).toBe(undefined); }); it('should collect coverage of an origin left without a page', async ({ browser, server }, testInfo) => { @@ -227,7 +221,7 @@ it('should collect coverage of an origin left without a page', async ({ browser, await context.tracing.stop({ path: traceFile }); await context.close(); - const data = await readCoverage(traceFile); + const data = await parseTraceCoverage(traceFile); expect(data['popup.js'].s['0']).toBe(5); }); @@ -246,7 +240,7 @@ it('should pull counters as the actions go', async ({ browser, server }, testInf await context.tracing.stop({ path: traceFile }); await context.close(); - const data = await readCoverage(traceFile); + const data = await parseTraceCoverage(traceFile); expect(data['a.js'].s['0']).toBe(3); }); diff --git a/tests/playwright-test/coverage.spec.ts b/tests/playwright-test/coverage.spec.ts index 6156effe3c3a0..3c9f26af61619 100644 --- a/tests/playwright-test/coverage.spec.ts +++ b/tests/playwright-test/coverage.spec.ts @@ -16,7 +16,7 @@ import fs from 'fs'; import { test, expect } from './playwright-test-fixtures'; -import { parseTraceRaw } from '../config/utils'; +import { parseTraceCoverage } from '../config/utils'; const coverageHelper = ` const makeCoverage = (file, s0) => ({ @@ -40,12 +40,6 @@ const coverageHelper = ` const coverageScript = (file, s0) => ''; `; -async function readCoverage(tracePath: string): Promise { - const { resources } = await parseTraceRaw(tracePath); - const entry = resources.get('coverage.json'); - return entry ? JSON.parse(entry.toString()) : undefined; -} - test('should collect istanbul coverage into the trace', async ({ runInlineTest }) => { const result = await runInlineTest({ 'playwright.config.ts': ` @@ -74,7 +68,7 @@ test('should collect istanbul coverage into the trace', async ({ runInlineTest } expect(result.exitCode).toBe(0); expect(result.passed).toBe(1); - const data = await readCoverage(test.info().outputPath('test-results', 'a-pass', 'trace.zip')); + const data = await parseTraceCoverage(test.info().outputPath('test-results', 'a-pass', 'trace.zip')); expect(data['src/app.js'].s['0']).toBe(7); expect(data['src/app.js'].s['1']).toBe(0); expect(data['src/app.js'].f['0']).toBe(2); @@ -100,7 +94,7 @@ test('should collect coverage from frames', async ({ runInlineTest }) => { expect(result.exitCode).toBe(0); expect(result.passed).toBe(1); - const data = await readCoverage(test.info().outputPath('test-results', 'a-pass', 'trace.zip')); + const data = await parseTraceCoverage(test.info().outputPath('test-results', 'a-pass', 'trace.zip')); expect(data['src/app.js'].s['0']).toBe(1); expect(data['src/frame.js'].s['0']).toBe(3); }); @@ -184,6 +178,30 @@ test('should not collect coverage without the option', async ({ runInlineTest }) expect(result.exitCode).toBe(0); expect(result.passed).toBe(1); - const data = await readCoverage(test.info().outputPath('test-results', 'a-pass', 'trace.zip')); + const data = await parseTraceCoverage(test.info().outputPath('test-results', 'a-pass', 'trace.zip')); expect(data).toBe(undefined); }); + +test('should keep the coverage of every context of a test', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { use: { trace: { mode: 'on', coverage: true } } }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + ${coverageHelper} + test('pass', async ({ page, browser }) => { + await page.setContent(coverageScript('src/first.js', 1)); + const second = await browser.newContext(); + await (await second.newPage()).setContent(coverageScript('src/second.js', 4)); + await second.close(); + }); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + + const data = await parseTraceCoverage(test.info().outputPath('test-results', 'a-pass', 'trace.zip')); + expect(data['src/first.js'].s['0']).toBe(1); + expect(data['src/second.js'].s['0']).toBe(4); +}); From e4f13c9ae54453a7cdfe9b8003fc1f7a5724a91d Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Fri, 11 Sep 2026 17:07:20 -0700 Subject: [PATCH 09/21] docs(coverage): drop the experimental note from the coverage API --- docs/src/api/class-tracing.md | 4 ++-- docs/src/test-api/class-testoptions.md | 2 +- packages/playwright-client/types/types.d.ts | 8 ++++---- packages/playwright-core/types/types.d.ts | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/src/api/class-tracing.md b/docs/src/api/class-tracing.md index be3ac85b35da5..0241832b95b15 100644 --- a/docs/src/api/class-tracing.md +++ b/docs/src/api/class-tracing.md @@ -151,7 +151,7 @@ a timeline preview. * langs: js - `coverage` <[boolean]> -**Experimental.** Whether to collect code coverage from istanbul-instrumented application code. Build the application with an istanbul instrumentation plugin, for example [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right before navigations, and stores them in istanbul format inside the trace file. Use [`method: Tracing.flushCoverage`] to collect counters from a page that is about to be closed by in-page script. +Whether to collect code coverage from istanbul-instrumented application code. Build the application with an istanbul instrumentation plugin, for example [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right before navigations, and stores them in istanbul format inside the trace file. Use [`method: Tracing.flushCoverage`] to collect counters from a page that is about to be closed by in-page script. ### option: Tracing.start.snapshots * since: v1.12 @@ -490,7 +490,7 @@ Closes the last group created by [`method: Tracing.group`]. * since: v1.64 * langs: js -**Experimental.** Collects istanbul coverage counters accumulated by all pages of the browser context into the coverage recorded by the trace. Requires tracing with the [`option: Tracing.start.coverage`] option to be started first. +Collects istanbul coverage counters accumulated by all pages of the browser context into the coverage recorded by the trace. Requires tracing with the [`option: Tracing.start.coverage`] option to be started first. Coverage is collected automatically when the trace chunk is stopped and when a page is closed with [`method: Page.close`]. Call this method before a page is closed by in-page script, for example a popup closing itself, so that its counters are not lost. diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.md index 82cc51f6089ef..06cfe0e883f0c 100644 --- a/docs/src/test-api/class-testoptions.md +++ b/docs/src/test-api/class-testoptions.md @@ -673,7 +673,7 @@ export default defineConfig({ - `dom` ?<[boolean]> Capture DOM snapshot on every action and record network activity. Optional. - `aria` ?<[boolean]> Capture aria snapshot of the page on every action. Optional. - `screen` ?<[boolean]> Capture a screenshot of the page on every action. Optional. - - `coverage` ?<[boolean]> **Experimental.** Whether to collect coverage from istanbul-instrumented application code into the trace. Defaults to false. Optional. + - `coverage` ?<[boolean]> Whether to collect coverage from istanbul-instrumented application code into the trace. Defaults to false. Optional. - `sources` ?<[boolean]> Whether to include source files for trace actions. Defaults to true. Optional. Whether to record trace for each test. Defaults to `'off'`. The initial run of a test is the "first run"; subsequent runs caused by [retries](../test-retries.md) are "retries". diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts index c2925fdc0c53e..40ecd084a845e 100644 --- a/packages/playwright-client/types/types.d.ts +++ b/packages/playwright-client/types/types.d.ts @@ -23205,8 +23205,8 @@ export interface Touchscreen { */ export interface Tracing { /** - * **Experimental.** Collects istanbul coverage counters accumulated by all pages of the browser context into the - * coverage recorded by the trace. Requires tracing with the + * Collects istanbul coverage counters accumulated by all pages of the browser context into the coverage recorded by + * the trace. Requires tracing with the * [`coverage`](https://playwright.dev/docs/api/class-tracing#tracing-start-option-coverage) option to be started * first. * @@ -23281,8 +23281,8 @@ export interface Tracing { */ start(options?: { /** - * **Experimental.** Whether to collect code coverage from istanbul-instrumented application code. Build the - * application with an istanbul instrumentation plugin, for example + * Whether to collect code coverage from istanbul-instrumented application code. Build the application with an + * istanbul instrumentation plugin, for example * [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or * [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the * `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index c2925fdc0c53e..40ecd084a845e 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -23205,8 +23205,8 @@ export interface Touchscreen { */ export interface Tracing { /** - * **Experimental.** Collects istanbul coverage counters accumulated by all pages of the browser context into the - * coverage recorded by the trace. Requires tracing with the + * Collects istanbul coverage counters accumulated by all pages of the browser context into the coverage recorded by + * the trace. Requires tracing with the * [`coverage`](https://playwright.dev/docs/api/class-tracing#tracing-start-option-coverage) option to be started * first. * @@ -23281,8 +23281,8 @@ export interface Tracing { */ start(options?: { /** - * **Experimental.** Whether to collect code coverage from istanbul-instrumented application code. Build the - * application with an istanbul instrumentation plugin, for example + * Whether to collect code coverage from istanbul-instrumented application code. Build the application with an + * istanbul instrumentation plugin, for example * [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or * [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the * `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right From 58300b816a5e624c992893ced8ca66e3b786c5fe Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Fri, 11 Sep 2026 17:48:09 -0700 Subject: [PATCH 10/21] chore(coverage): capture coverage as part of the after snapshot Keeps onAfterCall synchronous, drops the api call wrapper around flushCoverage and the visitOrigins comment. --- .../playwright-core/src/client/tracing.ts | 4 +-- .../src/server/browserContext.ts | 1 - .../src/server/trace/recorder/tracing.ts | 33 ++++++++++--------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/packages/playwright-core/src/client/tracing.ts b/packages/playwright-core/src/client/tracing.ts index 679be72ac027f..11ea7397d472a 100644 --- a/packages/playwright-core/src/client/tracing.ts +++ b/packages/playwright-core/src/client/tracing.ts @@ -80,9 +80,7 @@ export class Tracing extends ChannelOwner implements ap } async flushCoverage() { - await this._wrapApiCall(async () => { - await this._channel.tracingFlushCoverage({}, kNoTimeout); - }); + await this._channel.tracingFlushCoverage({}, kNoTimeout); } private async _startCollectingStacks(traceName: string, live: boolean) { diff --git a/packages/playwright-core/src/server/browserContext.ts b/packages/playwright-core/src/server/browserContext.ts index a57e34c20e094..32bc5e10f8167 100644 --- a/packages/playwright-core/src/server/browserContext.ts +++ b/packages/playwright-core/src/server/browserContext.ts @@ -657,7 +657,6 @@ export abstract class BrowserContext extends Sdk return result; } - // Reaches the storage of the origins that have no page, in a hidden page. async visitOrigins(progress: Progress, origins: Set, callback: (frame: frames.Frame, origin: string) => Promise) { const page = await this.newPage(progress, true /* forStorageState */); try { diff --git a/packages/playwright-core/src/server/trace/recorder/tracing.ts b/packages/playwright-core/src/server/trace/recorder/tracing.ts index 06afcec83e55b..f37546d453246 100644 --- a/packages/playwright-core/src/server/trace/recorder/tracing.ts +++ b/packages/playwright-core/src/server/trace/recorder/tracing.ts @@ -523,6 +523,18 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps await this._captureScreenshot(progress, page, phase); if (options?.snapshotAria) await this._captureAriaSnapshot(progress, page, phase); + if (options?.coverage && phase === 'after') + await this._captureCoverage(progress, page); + } + + private async _captureCoverage(progress: Progress, page: Page): Promise { + const recorder = this._coverageRecorder; + if (!recorder) + return; + try { + await progress.race(recorder.collectFromPage(page)); + } catch { + } } private _shouldCaptureAtPhase(metadata: CallMetadata, phase: trace.ActionPhase) { @@ -595,27 +607,18 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps this._appendTraceEvent(event); } - async onAfterCall(progress: Progress, sdkObject: SdkObject) { - // IMPORTANT: no awaits before _appendTraceEvent, it must be called synchronously. + onAfterCall(progress: Progress, sdkObject: SdkObject) { + // IMPORTANT: no awaits in this method, this._appendTraceEvent must be called synchronously. const { metadata } = progress; if (!this._state?.callsInProgress.has(metadata.id)) - return; - this._state.callsInProgress.delete(metadata.id); + return Promise.resolve(); + this._state?.callsInProgress.delete(metadata.id); const event = createAfterActionTraceEvent(metadata); if (!event) - return; + return Promise.resolve(); this._temporarilyDisableThrottling(sdkObject.attribution.page); this._appendTraceEvent(event); - await this._captureSnapshot(progress, sdkObject, 'after'); - // Collecting as the actions go, an action is the most that can be lost. - const page = sdkObject.attribution.page; - const recorder = this._activeCoverageRecorder(); - if (page && recorder) { - try { - await progress.race(recorder.collectFromPage(page)); - } catch { - } - } + return this._captureSnapshot(progress, sdkObject, 'after'); } async onPageWillClose(page: Page) { From e523663a172d7ad5ce8c6d75b8ebbdaa67b57bcd Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Tue, 15 Sep 2026 16:47:07 -0700 Subject: [PATCH 11/21] chore(coverage): drop the stash ordinal, the random suffix is enough --- packages/injected/src/coverageScript.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/injected/src/coverageScript.ts b/packages/injected/src/coverageScript.ts index fda4199fa9f61..630fbfd689663 100644 --- a/packages/injected/src/coverageScript.ts +++ b/packages/injected/src/coverageScript.ts @@ -23,7 +23,6 @@ export class CoverageScript { private _global: typeof globalThis; private _sessionId: string; private _reportedFiles = new Set(); - private _stashOrdinal = 0; constructor(global: typeof globalThis, collectName: string, sessionId: string) { this._global = global; @@ -72,7 +71,7 @@ export class CoverageScript { return; try { // Several documents can pick up the same stash, the id discards the copies. - const id = ++this._stashOrdinal + '-' + Math.random().toString(36).slice(2); + const id = Math.random().toString(36).slice(2); const key = kCoverageStashPrefix + this._sessionId + '.' + id; this._global.localStorage.setItem(key, JSON.stringify({ id, data: delta })); } catch { From 57952e3d7a0fc0c8e6d659cb50c4000306ea66b8 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Tue, 15 Sep 2026 16:48:14 -0700 Subject: [PATCH 12/21] chore(coverage): drop comments that restate the code --- packages/injected/src/coverageScript.ts | 1 - packages/isomorphic/istanbulCoverage.ts | 3 --- packages/playwright-core/src/server/coverageRecorder.ts | 4 ---- 3 files changed, 8 deletions(-) diff --git a/packages/injected/src/coverageScript.ts b/packages/injected/src/coverageScript.ts index 630fbfd689663..440992e904cd5 100644 --- a/packages/injected/src/coverageScript.ts +++ b/packages/injected/src/coverageScript.ts @@ -79,7 +79,6 @@ export class CoverageScript { } } -// Also evaluated by Playwright itself on the origins that have no page left. // Stashes of other sessions are stale, e.g. left by a previous run in a persistent profile. export function takeCoverageStashes(global: typeof globalThis, sessionId: string): string[] { const result: string[] = []; diff --git a/packages/isomorphic/istanbulCoverage.ts b/packages/isomorphic/istanbulCoverage.ts index e9e80b0f558c2..7cc9cc4547722 100644 --- a/packages/isomorphic/istanbulCoverage.ts +++ b/packages/isomorphic/istanbulCoverage.ts @@ -45,7 +45,6 @@ export type IstanbulFileCoverage = { statementMap: { [key: string]: IstanbulRange }; fnMap: { [key: string]: IstanbulFunctionMapping }; branchMap: { [key: string]: IstanbulBranchMapping }; - // Hit counts, keyed by the corresponding map entries above. s: { [key: string]: number }; f: { [key: string]: number }; b: { [key: string]: number[] }; @@ -58,7 +57,6 @@ export type IstanbulFileCoverageDelta = Partial & Pick a.localeCompare(b))); } -// Counters add up, maps are taken from the first report that carries them. export function mergeIstanbulCoverage(into: Map, data: IstanbulCoverageDelta) { for (const [file, fileCov] of Object.entries(data)) { let existing = into.get(file); diff --git a/packages/playwright-core/src/server/coverageRecorder.ts b/packages/playwright-core/src/server/coverageRecorder.ts index 7d8eebf8703f2..06cf03e7da447 100644 --- a/packages/playwright-core/src/server/coverageRecorder.ts +++ b/packages/playwright-core/src/server/coverageRecorder.ts @@ -45,7 +45,6 @@ export class CoverageRecorder { })()`; } - // The stashes in the page storage are scoped to the context that collects them. private _sessionId() { return JSON.stringify(this._context.guid); } @@ -85,7 +84,6 @@ export class CoverageRecorder { async take(progress: Progress, mode: 'keep' | 'discard'): Promise { if (mode === 'discard') { - // The coverage of a chunk is discarded together with it. this._coverage.clear(); return; } @@ -99,7 +97,6 @@ export class CoverageRecorder { } async collectFromPage(page: Page) { - // Pages of Playwright's own making never run the application under test. if (page.isStorageStatePage) return; // A page that is going away may never answer. @@ -115,7 +112,6 @@ export class CoverageRecorder { })); } - // Pages relay the stashes of their own origin, the rest need a page of their own. private async _harvestOriginsWithoutPage(progress: Progress) { if (!this._stashOrigins.size || this._context.isClosingOrClosed()) return; From 38743aa2dd76607f21f98685cd21de225e8a826b Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Tue, 15 Sep 2026 17:22:13 -0700 Subject: [PATCH 13/21] chore(coverage): harvest the origins the context has visited The recorder only noted origins it happened to see during a pull or at page close, missing origins passed through in between. --- .../src/server/browserContext.ts | 4 ++++ .../src/server/coverageRecorder.ts | 18 ++---------------- .../src/server/trace/recorder/tracing.ts | 1 - 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/packages/playwright-core/src/server/browserContext.ts b/packages/playwright-core/src/server/browserContext.ts index 32bc5e10f8167..f3e399f5b35c7 100644 --- a/packages/playwright-core/src/server/browserContext.ts +++ b/packages/playwright-core/src/server/browserContext.ts @@ -614,6 +614,10 @@ export abstract class BrowserContext extends Sdk this._origins.add(origin); } + visitedOrigins(): ReadonlySet { + return this._origins; + } + async storageState(progress: Progress, { indexedDB = false, opfs = false, credentials = false }: { indexedDB?: boolean, opfs?: boolean, credentials?: boolean } = {}): Promise { const result: channels.BrowserContextStorageStateResult = { cookies: await this.cookies(progress), diff --git a/packages/playwright-core/src/server/coverageRecorder.ts b/packages/playwright-core/src/server/coverageRecorder.ts index 06cf03e7da447..0091395b4f056 100644 --- a/packages/playwright-core/src/server/coverageRecorder.ts +++ b/packages/playwright-core/src/server/coverageRecorder.ts @@ -30,7 +30,6 @@ export class CoverageRecorder { private _context: BrowserContext; private _coverage = new Map(); private _stashedChunkIds = new Set(); - private _stashOrigins = new Set(); private _initScript: InitScript | undefined; constructor(context: BrowserContext) { @@ -61,22 +60,11 @@ export class CoverageRecorder { async uninstall() { const initScript = this._initScript; this._initScript = undefined; - this._stashOrigins.clear(); this._stashedChunkIds.clear(); this._coverage.clear(); await initScript?.dispose().catch(() => {}); } - onPageClose(page: Page) { - for (const frame of page.frames()) - this._noteOrigin(frame.origin()); - } - - private _noteOrigin(origin: string | undefined) { - if (origin) - this._stashOrigins.add(origin); - } - async flush(progress: Progress) { await progress.race(Promise.all(this._context.pages().map(page => this.collectFromPage(page)))); await this._harvestOriginsWithoutPage(progress); @@ -105,7 +93,6 @@ export class CoverageRecorder { private async _collectFromPage(page: Page) { await Promise.all(page.frames().map(async frame => { - this._noteOrigin(frame.origin()); const chunks: string[] = await frame.nonStallingRawEvaluateInExistingMainContext(coverageCollectExpression).catch(() => []); for (const json of chunks) this._append(json); @@ -113,7 +100,7 @@ export class CoverageRecorder { } private async _harvestOriginsWithoutPage(progress: Progress) { - if (!this._stashOrigins.size || this._context.isClosingOrClosed()) + if (this._context.isClosingOrClosed()) return; const liveOrigins = new Set(); for (const page of this._context.pages()) { @@ -124,8 +111,7 @@ export class CoverageRecorder { } } // The origins that still have a page were drained by the sweep above. - const origins = new Set([...this._stashOrigins].filter(origin => !liveOrigins.has(origin))); - this._stashOrigins.clear(); + const origins = new Set([...this._context.visitedOrigins()].filter(origin => !liveOrigins.has(origin))); if (!origins.size) return; const source = this._moduleExpression(`return (module.exports.takeCoverageStashes())(window, ${this._sessionId()});`); diff --git a/packages/playwright-core/src/server/trace/recorder/tracing.ts b/packages/playwright-core/src/server/trace/recorder/tracing.ts index f37546d453246..f3d93d6fe10e7 100644 --- a/packages/playwright-core/src/server/trace/recorder/tracing.ts +++ b/packages/playwright-core/src/server/trace/recorder/tracing.ts @@ -722,7 +722,6 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps } onPageClose(page: Page) { - this._activeCoverageRecorder()?.onPageClose(page); const event: trace.EventTraceEvent = { type: 'event', time: monotonicTime(), From 1bc18f852253d31251bb48df70de02af114daa99 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Tue, 15 Sep 2026 17:25:34 -0700 Subject: [PATCH 14/21] test(coverage): fold the runner coverage tests into three Frames and multi-context cases join the plumbing and reporter tests, the option-off case moves to the library suite. --- tests/library/coverage.spec.ts | 13 ++++ tests/playwright-test/coverage.spec.ts | 89 +++++--------------------- 2 files changed, 29 insertions(+), 73 deletions(-) diff --git a/tests/library/coverage.spec.ts b/tests/library/coverage.spec.ts index 3e4fbc65bc3c9..5353a93c18049 100644 --- a/tests/library/coverage.spec.ts +++ b/tests/library/coverage.spec.ts @@ -244,6 +244,19 @@ it('should pull counters as the actions go', async ({ browser, server }, testInf expect(data['a.js'].s['0']).toBe(3); }); +it('should not collect coverage without the option', async ({ browser }, testInfo) => { + const context = await browser.newContext(); + await context.tracing.start(); + const page = await context.newPage(); + await page.setContent(coverageScript('a.js', 1)); + + const traceFile = testInfo.outputPath('trace.zip'); + await context.tracing.stop({ path: traceFile }); + await context.close(); + + expect(await parseTraceCoverage(traceFile)).toBe(undefined); +}); + it('should throw when flushing without coverage', async ({ browser }) => { const context = await browser.newContext(); await context.newPage(); diff --git a/tests/playwright-test/coverage.spec.ts b/tests/playwright-test/coverage.spec.ts index 3c9f26af61619..f00a17b582ef1 100644 --- a/tests/playwright-test/coverage.spec.ts +++ b/tests/playwright-test/coverage.spec.ts @@ -54,11 +54,17 @@ test('should collect istanbul coverage into the trace', async ({ runInlineTest } const http = require('http'); const server = http.createServer((req, res) => { res.setHeader('Content-Type', 'text/html'); - res.end(coverageScript('src/app.js', req.url === '/page1' ? 5 : 2)); + if (req.url === '/frame') + res.end(coverageScript('src/frame.js', 3)); + else if (req.url === '/page1') + res.end(coverageScript('src/app.js', 5) + ''); + else + res.end(coverageScript('src/app.js', 2)); }); await new Promise(f => server.listen(0, '127.0.0.1', f)); const origin = 'http://127.0.0.1:' + server.address().port; await page.goto(origin + '/page1'); + await expect(page.locator('iframe')).toBeVisible(); // Navigating away parks the first document's counters for the next one. await page.goto(origin + '/page2'); server.close(); @@ -73,29 +79,6 @@ test('should collect istanbul coverage into the trace', async ({ runInlineTest } expect(data['src/app.js'].s['1']).toBe(0); expect(data['src/app.js'].f['0']).toBe(2); expect(data['src/app.js'].b['0']).toEqual([2, 0]); -}); - -test('should collect coverage from frames', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'playwright.config.ts': ` - module.exports = { - use: { trace: { mode: 'on', coverage: true } }, - }; - `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - ${coverageHelper} - test('pass', async ({ page }) => { - await page.setContent(coverageScript('src/app.js', 1) + ''); - await expect(page.locator('iframe')).toBeVisible(); - }); - `, - }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - - const data = await parseTraceCoverage(test.info().outputPath('test-results', 'a-pass', 'trace.zip')); - expect(data['src/app.js'].s['0']).toBe(1); expect(data['src/frame.js'].s['0']).toBe(3); }); @@ -117,8 +100,12 @@ test('should aggregate coverage with the coverage reporter', async ({ runInlineT 'b.test.ts': ` import { test, expect } from '@playwright/test'; ${coverageHelper} - test('second', async ({ page }) => { + test('second', async ({ page, browser }) => { await page.setContent(coverageScript('src/app.js', 2)); + // Every context of a test keeps its own coverage entry in the trace. + const second = await browser.newContext(); + await (await second.newPage()).setContent(coverageScript('src/second.js', 4)); + await second.close(); }); `, }); @@ -130,14 +117,16 @@ test('should aggregate coverage with the coverage reporter', async ({ runInlineT expect(final['src/app.js'].s['1']).toBe(0); expect(final['src/app.js'].f['0']).toBe(2); expect(final['src/app.js'].b['0']).toEqual([2, 0]); + expect(final['src/second.js'].s['0']).toBe(4); const lcov = fs.readFileSync(test.info().outputPath('coverage-report', 'lcov.info'), 'utf8'); expect(lcov).toContain('SF:src/app.js'); expect(lcov).toContain('DA:1,7'); expect(lcov).toContain('FNDA:2,foo'); + expect(lcov).toContain('SF:src/second.js'); - expect(result.output).toContain('Code coverage (1 files)'); - expect(result.output).toContain('statements: 50.00% (1/2)'); + expect(result.output).toContain('Code coverage (2 files)'); + expect(result.output).toContain('statements: 50.00% (2/4)'); }); test('should warn when no coverage was collected', async ({ runInlineTest }) => { @@ -159,49 +148,3 @@ test('should warn when no coverage was collected', async ({ runInlineTest }) => expect(result.output).toContain('No code coverage was collected'); expect(fs.existsSync(test.info().outputPath('coverage-report', 'coverage-final.json'))).toBe(false); }); - -test('should not collect coverage without the option', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'playwright.config.ts': ` - module.exports = { - use: { trace: 'on' }, - }; - `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - ${coverageHelper} - test('pass', async ({ page }) => { - await page.setContent(coverageScript('src/app.js', 1)); - }); - `, - }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - - const data = await parseTraceCoverage(test.info().outputPath('test-results', 'a-pass', 'trace.zip')); - expect(data).toBe(undefined); -}); - -test('should keep the coverage of every context of a test', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'playwright.config.ts': ` - module.exports = { use: { trace: { mode: 'on', coverage: true } } }; - `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - ${coverageHelper} - test('pass', async ({ page, browser }) => { - await page.setContent(coverageScript('src/first.js', 1)); - const second = await browser.newContext(); - await (await second.newPage()).setContent(coverageScript('src/second.js', 4)); - await second.close(); - }); - `, - }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - - const data = await parseTraceCoverage(test.info().outputPath('test-results', 'a-pass', 'trace.zip')); - expect(data['src/first.js'].s['0']).toBe(1); - expect(data['src/second.js'].s['0']).toBe(4); -}); From 36a5415a107cadb1e1a2a6edf7eede071530488f Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Wed, 16 Sep 2026 11:44:03 -0700 Subject: [PATCH 15/21] chore(coverage): recreate the recorder per tracing session The recorder lives from start to stop, and uninstall unhooks the pagehide listener in the live documents. --- packages/injected/src/coverageScript.ts | 13 +++++++++++-- .../src/server/coverageRecorder.ts | 9 +++------ .../src/server/trace/recorder/tracing.ts | 16 +++++++--------- tests/library/coverage.spec.ts | 17 +++++++++++++++++ 4 files changed, 38 insertions(+), 17 deletions(-) diff --git a/packages/injected/src/coverageScript.ts b/packages/injected/src/coverageScript.ts index 440992e904cd5..f4ed205973a58 100644 --- a/packages/injected/src/coverageScript.ts +++ b/packages/injected/src/coverageScript.ts @@ -21,15 +21,24 @@ import type { IstanbulCoverage, IstanbulCoverageDelta, IstanbulFileCoverageDelta // Reports `__coverage__` as a delta: reading resets the counters, maps are sent once per file. export class CoverageScript { private _global: typeof globalThis; + private _collectName: string; private _sessionId: string; private _reportedFiles = new Set(); + private _onPageHide = () => this._stashCurrent(); constructor(global: typeof globalThis, collectName: string, sessionId: string) { this._global = global; + this._collectName = collectName; this._sessionId = sessionId; - (global as any)[collectName] = () => this.collect(); + (global as any)[collectName]?.dispose(); + (global as any)[collectName] = Object.assign(() => this.collect(), { dispose: () => this.dispose() }); // Counters die with the document and unload-time calls are not delivered. - global.addEventListener('pagehide', () => this._stashCurrent()); + global.addEventListener('pagehide', this._onPageHide); + } + + dispose() { + this._global.removeEventListener('pagehide', this._onPageHide); + delete (this._global as any)[this._collectName]; } collect(): string[] { diff --git a/packages/playwright-core/src/server/coverageRecorder.ts b/packages/playwright-core/src/server/coverageRecorder.ts index 0091395b4f056..d29bbdbbb8ed8 100644 --- a/packages/playwright-core/src/server/coverageRecorder.ts +++ b/packages/playwright-core/src/server/coverageRecorder.ts @@ -25,6 +25,7 @@ import type { IstanbulCoverageChunk, IstanbulFileCoverage } from '@isomorphic/is const kCoverageCollectName = '__pwCoverageCollect'; const coverageCollectExpression = `window[${JSON.stringify(kCoverageCollectName)}] ? window[${JSON.stringify(kCoverageCollectName)}]() : []`; +const coverageDisposeExpression = `window[${JSON.stringify(kCoverageCollectName)}]?.dispose()`; export class CoverageRecorder { private _context: BrowserContext; @@ -53,16 +54,12 @@ export class CoverageRecorder { return; const source = this._moduleExpression(`new (module.exports.CoverageScript())(window, ${JSON.stringify(kCoverageCollectName)}, ${this._sessionId()});`); this._initScript = await this._context.addInitScript(progress, source); - // Init scripts only affect future documents, bootstrap the existing ones. await progress.race(this._context.safeNonStallingEvaluateInAllFrames(source, 'main')); } async uninstall() { - const initScript = this._initScript; - this._initScript = undefined; - this._stashedChunkIds.clear(); - this._coverage.clear(); - await initScript?.dispose().catch(() => {}); + await this._initScript?.dispose().catch(() => {}); + await this._context.safeNonStallingEvaluateInAllFrames(coverageDisposeExpression, 'main').catch(() => {}); } async flush(progress: Progress) { diff --git a/packages/playwright-core/src/server/trace/recorder/tracing.ts b/packages/playwright-core/src/server/trace/recorder/tracing.ts index f3d93d6fe10e7..1469f2d246242 100644 --- a/packages/playwright-core/src/server/trace/recorder/tracing.ts +++ b/packages/playwright-core/src/server/trace/recorder/tracing.ts @@ -188,7 +188,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps if (options.snapshotDom) this._harTracer.start({ omitScripts: !options.live }); if (options.coverage && this._context instanceof BrowserContext) - this._coverageRecorder ??= new CoverageRecorder(this._context); + this._coverageRecorder = new CoverageRecorder(this._context); this._started = true; } @@ -346,7 +346,9 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps this._closeAllGroups(); this._harTracer.stop(); this.flushHarEntries(); - await this._coverageRecorder?.uninstall(); + const coverageRecorder = this._coverageRecorder; + this._coverageRecorder = undefined; + await coverageRecorder?.uninstall(); await this._fs.sync().finally(() => { this._state = undefined; }); @@ -395,12 +397,8 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps this._groupEnd(); } - private _activeCoverageRecorder(): CoverageRecorder | undefined { - return this._state?.options.coverage ? this._coverageRecorder : undefined; - } - async flushCoverage(progress: Progress) { - const recorder = this._activeCoverageRecorder(); + const recorder = this._coverageRecorder; if (!recorder) throw new Error(`Coverage collection has not been started. Pass the "coverage" option to tracing.start().`); await recorder.flush(progress); @@ -408,7 +406,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps // Collected before the chunk stops, while the pages can still be evaluated in. private async _takeCoverage(progress: Progress, mode: TracingTracingStopChunkParams['mode']): Promise { - const recorder = this._activeCoverageRecorder(); + const recorder = this._coverageRecorder; if (!this._state?.recording || !recorder) return; const coverageJson = await recorder.take(progress, mode === 'discard' ? 'discard' : 'keep'); @@ -622,7 +620,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps } async onPageWillClose(page: Page) { - await this._activeCoverageRecorder()?.collectFromPage(page).catch(() => {}); + await this._coverageRecorder?.collectFromPage(page).catch(() => {}); } onEntryStarted(entry: har.Entry) { diff --git a/tests/library/coverage.spec.ts b/tests/library/coverage.spec.ts index 5353a93c18049..9d88da5dcbb88 100644 --- a/tests/library/coverage.spec.ts +++ b/tests/library/coverage.spec.ts @@ -257,6 +257,23 @@ it('should not collect coverage without the option', async ({ browser }, testInf expect(await parseTraceCoverage(traceFile)).toBe(undefined); }); +it('should stop collecting when tracing stops', async ({ browser, server }, testInfo) => { + const context = await browser.newContext(); + await context.tracing.start({ coverage: true }); + const page = await context.newPage(); + await page.goto(server.EMPTY_PAGE); + await page.evaluate(coverage => (window as any).__coverage__ = JSON.parse(coverage), JSON.stringify(fileCoverage('a.js', 1))); + await context.tracing.stop(); + + expect(await page.evaluate(() => typeof (window as any).__pwCoverageCollect)).toBe('undefined'); + // Leaving the document no longer stashes the counters. + await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'] = 5); + await page.goto(server.PREFIX + '/title.html'); + expect(await page.evaluate(() => Object.keys(localStorage).filter(key => key.startsWith('__pwCoverage.')))).toEqual([]); + expect(await page.evaluate(() => typeof (window as any).__pwCoverageCollect)).toBe('undefined'); + await context.close(); +}); + it('should throw when flushing without coverage', async ({ browser }) => { const context = await browser.newContext(); await context.newPage(); From d7122d8f4692931b95aad0feb015995b3bf37d51 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Wed, 16 Sep 2026 11:50:08 -0700 Subject: [PATCH 16/21] chore(coverage): rename the in-page collect to take --- packages/injected/src/coverageScript.ts | 14 +++++++------- .../src/server/coverageRecorder.ts | 16 +++++++++------- tests/library/coverage.spec.ts | 10 +++++----- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/packages/injected/src/coverageScript.ts b/packages/injected/src/coverageScript.ts index f4ed205973a58..431cf5bcb21c4 100644 --- a/packages/injected/src/coverageScript.ts +++ b/packages/injected/src/coverageScript.ts @@ -21,27 +21,27 @@ import type { IstanbulCoverage, IstanbulCoverageDelta, IstanbulFileCoverageDelta // Reports `__coverage__` as a delta: reading resets the counters, maps are sent once per file. export class CoverageScript { private _global: typeof globalThis; - private _collectName: string; + private _takeName: string; private _sessionId: string; private _reportedFiles = new Set(); private _onPageHide = () => this._stashCurrent(); - constructor(global: typeof globalThis, collectName: string, sessionId: string) { + constructor(global: typeof globalThis, takeName: string, sessionId: string) { this._global = global; - this._collectName = collectName; + this._takeName = takeName; this._sessionId = sessionId; - (global as any)[collectName]?.dispose(); - (global as any)[collectName] = Object.assign(() => this.collect(), { dispose: () => this.dispose() }); + (global as any)[takeName]?.dispose(); + (global as any)[takeName] = Object.assign(() => this.take(), { dispose: () => this.dispose() }); // Counters die with the document and unload-time calls are not delivered. global.addEventListener('pagehide', this._onPageHide); } dispose() { this._global.removeEventListener('pagehide', this._onPageHide); - delete (this._global as any)[this._collectName]; + delete (this._global as any)[this._takeName]; } - collect(): string[] { + take(): string[] { const chunks = takeCoverageStashes(this._global, this._sessionId); const delta = this._takeCurrent(); if (delta) diff --git a/packages/playwright-core/src/server/coverageRecorder.ts b/packages/playwright-core/src/server/coverageRecorder.ts index d29bbdbbb8ed8..f0a19d5f9da19 100644 --- a/packages/playwright-core/src/server/coverageRecorder.ts +++ b/packages/playwright-core/src/server/coverageRecorder.ts @@ -22,10 +22,10 @@ import type { InitScript, Page } from './page'; import type { Progress } from './progress'; import type { IstanbulCoverageChunk, IstanbulFileCoverage } from '@isomorphic/istanbulCoverage'; -const kCoverageCollectName = '__pwCoverageCollect'; +const kCoverageTakeName = '__pwCoverageTake'; -const coverageCollectExpression = `window[${JSON.stringify(kCoverageCollectName)}] ? window[${JSON.stringify(kCoverageCollectName)}]() : []`; -const coverageDisposeExpression = `window[${JSON.stringify(kCoverageCollectName)}]?.dispose()`; +const coverageTakeExpression = `window[${JSON.stringify(kCoverageTakeName)}] ? window[${JSON.stringify(kCoverageTakeName)}]() : []`; +const coverageDisposeExpression = `window[${JSON.stringify(kCoverageTakeName)}]?.dispose()`; export class CoverageRecorder { private _context: BrowserContext; @@ -52,14 +52,16 @@ export class CoverageRecorder { async install(progress: Progress) { if (this._initScript) return; - const source = this._moduleExpression(`new (module.exports.CoverageScript())(window, ${JSON.stringify(kCoverageCollectName)}, ${this._sessionId()});`); + const source = this._moduleExpression(`new (module.exports.CoverageScript())(window, ${JSON.stringify(kCoverageTakeName)}, ${this._sessionId()});`); this._initScript = await this._context.addInitScript(progress, source); await progress.race(this._context.safeNonStallingEvaluateInAllFrames(source, 'main')); } async uninstall() { - await this._initScript?.dispose().catch(() => {}); - await this._context.safeNonStallingEvaluateInAllFrames(coverageDisposeExpression, 'main').catch(() => {}); + await Promise.all([ + this._initScript?.dispose(), + this._context.safeNonStallingEvaluateInAllFrames(coverageDisposeExpression, 'main'), + ]).catch(() => {}); } async flush(progress: Progress) { @@ -90,7 +92,7 @@ export class CoverageRecorder { private async _collectFromPage(page: Page) { await Promise.all(page.frames().map(async frame => { - const chunks: string[] = await frame.nonStallingRawEvaluateInExistingMainContext(coverageCollectExpression).catch(() => []); + const chunks: string[] = await frame.nonStallingRawEvaluateInExistingMainContext(coverageTakeExpression).catch(() => []); for (const json of chunks) this._append(json); })); diff --git a/tests/library/coverage.spec.ts b/tests/library/coverage.spec.ts index 9d88da5dcbb88..f7c7a90cfe748 100644 --- a/tests/library/coverage.spec.ts +++ b/tests/library/coverage.spec.ts @@ -88,20 +88,20 @@ it('should report maps once and counters incrementally', async ({ browser, serve // Actions collect the counters on their own, so hit and collect in one evaluate. const first = await page.evaluate(coverage => { (window as any).__coverage__ = JSON.parse(coverage); - return (window as any).__pwCoverageCollect().map((json: string) => JSON.parse(json)); + return (window as any).__pwCoverageTake().map((json: string) => JSON.parse(json)); }, JSON.stringify(fileCoverage('a.js', 3))); expect(first[0].data['a.js'].statementMap).toBeTruthy(); expect(first[0].data['a.js'].s).toEqual({ '0': 3 }); const second = await page.evaluate(() => { (window as any).__coverage__['a.js'].s['0'] += 2; - return (window as any).__pwCoverageCollect().map((json: string) => JSON.parse(json)); + return (window as any).__pwCoverageTake().map((json: string) => JSON.parse(json)); }); expect(second[0].data['a.js'].statementMap).toBe(undefined); expect(second[0].data['a.js'].s).toEqual({ '0': 2 }); // Nothing was hit since the last report. - expect(await page.evaluate(() => (window as any).__pwCoverageCollect())).toEqual([]); + expect(await page.evaluate(() => (window as any).__pwCoverageTake())).toEqual([]); await context.tracing.stop(); await context.close(); @@ -265,12 +265,12 @@ it('should stop collecting when tracing stops', async ({ browser, server }, test await page.evaluate(coverage => (window as any).__coverage__ = JSON.parse(coverage), JSON.stringify(fileCoverage('a.js', 1))); await context.tracing.stop(); - expect(await page.evaluate(() => typeof (window as any).__pwCoverageCollect)).toBe('undefined'); + expect(await page.evaluate(() => typeof (window as any).__pwCoverageTake)).toBe('undefined'); // Leaving the document no longer stashes the counters. await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'] = 5); await page.goto(server.PREFIX + '/title.html'); expect(await page.evaluate(() => Object.keys(localStorage).filter(key => key.startsWith('__pwCoverage.')))).toEqual([]); - expect(await page.evaluate(() => typeof (window as any).__pwCoverageCollect)).toBe('undefined'); + expect(await page.evaluate(() => typeof (window as any).__pwCoverageTake)).toBe('undefined'); await context.close(); }); From afd7b805c0ebb37b6a3e4e7e644b6d9d8b953440 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Wed, 16 Sep 2026 11:58:26 -0700 Subject: [PATCH 17/21] chore(coverage): reset the counters when tracing starts Only the hits after the start count, the maps still go out with the first report. --- packages/injected/src/coverageScript.ts | 10 ++++++++++ .../src/server/trace/recorder/tracing.ts | 2 +- tests/library/coverage.spec.ts | 17 +++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/packages/injected/src/coverageScript.ts b/packages/injected/src/coverageScript.ts index 431cf5bcb21c4..f1f2442b3cd5c 100644 --- a/packages/injected/src/coverageScript.ts +++ b/packages/injected/src/coverageScript.ts @@ -32,10 +32,20 @@ export class CoverageScript { this._sessionId = sessionId; (global as any)[takeName]?.dispose(); (global as any)[takeName] = Object.assign(() => this.take(), { dispose: () => this.dispose() }); + this._resetCounters(); // Counters die with the document and unload-time calls are not delivered. global.addEventListener('pagehide', this._onPageHide); } + private _resetCounters() { + const coverage: IstanbulCoverage | undefined = (this._global as any).__coverage__; + for (const fileCoverage of Object.values(coverage || {})) { + takeCounters(fileCoverage.s); + takeCounters(fileCoverage.f); + takeBranchCounters(fileCoverage.b); + } + } + dispose() { this._global.removeEventListener('pagehide', this._onPageHide); delete (this._global as any)[this._takeName]; diff --git a/packages/playwright-core/src/server/trace/recorder/tracing.ts b/packages/playwright-core/src/server/trace/recorder/tracing.ts index 1469f2d246242..db3278073c920 100644 --- a/packages/playwright-core/src/server/trace/recorder/tracing.ts +++ b/packages/playwright-core/src/server/trace/recorder/tracing.ts @@ -521,7 +521,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps await this._captureScreenshot(progress, page, phase); if (options?.snapshotAria) await this._captureAriaSnapshot(progress, page, phase); - if (options?.coverage && phase === 'after') + if (phase === 'after') await this._captureCoverage(progress, page); } diff --git a/tests/library/coverage.spec.ts b/tests/library/coverage.spec.ts index f7c7a90cfe748..ad9e3290b9069 100644 --- a/tests/library/coverage.spec.ts +++ b/tests/library/coverage.spec.ts @@ -257,6 +257,23 @@ it('should not collect coverage without the option', async ({ browser }, testInf expect(await parseTraceCoverage(traceFile)).toBe(undefined); }); +it('should count only the hits after start', async ({ browser, server }, testInfo) => { + const context = await browser.newContext(); + const page = await context.newPage(); + await page.goto(server.EMPTY_PAGE); + await page.evaluate(coverage => (window as any).__coverage__ = JSON.parse(coverage), JSON.stringify(fileCoverage('a.js', 3))); + + await context.tracing.start({ coverage: true }); + await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'] += 2); + const traceFile = testInfo.outputPath('trace.zip'); + await context.tracing.stop({ path: traceFile }); + await context.close(); + + const data = await parseTraceCoverage(traceFile); + expect(data['a.js'].s['0']).toBe(2); + expect(Object.keys(data['a.js'].statementMap)).toEqual(['0']); +}); + it('should stop collecting when tracing stops', async ({ browser, server }, testInfo) => { const context = await browser.newContext(); await context.tracing.start({ coverage: true }); From a0da93500750d47d81840f3c1fe2b222d85067bd Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Wed, 16 Sep 2026 12:00:35 -0700 Subject: [PATCH 18/21] chore(coverage): drop the dispose of a previous script instance Uninstall already unhooks the previous instance in every live document. --- packages/injected/src/coverageScript.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/injected/src/coverageScript.ts b/packages/injected/src/coverageScript.ts index f1f2442b3cd5c..24714c9c94706 100644 --- a/packages/injected/src/coverageScript.ts +++ b/packages/injected/src/coverageScript.ts @@ -30,7 +30,6 @@ export class CoverageScript { this._global = global; this._takeName = takeName; this._sessionId = sessionId; - (global as any)[takeName]?.dispose(); (global as any)[takeName] = Object.assign(() => this.take(), { dispose: () => this.dispose() }); this._resetCounters(); // Counters die with the document and unload-time calls are not delivered. From 06b8a5f17638fab27b0bf7cb59c22ca39e7beff4 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Wed, 16 Sep 2026 12:36:25 -0700 Subject: [PATCH 19/21] feat(coverage): surface stash failures, drop flushCoverage from the API A failed stash is recorded in the page storage and thrown by the next take, the error is reported when the chunk stops. The two tests that were failing on Firefox and WebKit no longer click an empty body or navigate inside the action whose pull races the navigation. --- docs/src/api/class-tracing.md | 10 +--- packages/injected/src/coverageScript.ts | 55 ++++++++++++------- packages/isomorphic/istanbulCoverage.ts | 1 + packages/isomorphic/protocolMetainfo.ts | 1 - packages/playwright-client/types/types.d.ts | 16 +----- .../playwright-core/src/client/channels.d.ts | 4 -- .../playwright-core/src/client/tracing.ts | 4 -- .../playwright-core/src/server/channels.d.ts | 4 -- .../src/server/coverageRecorder.ts | 17 +++++- .../server/dispatchers/tracingDispatcher.ts | 4 -- .../src/server/trace/recorder/tracing.ts | 18 +++--- packages/playwright-core/types/types.d.ts | 16 +----- packages/protocol/spec/tracing.yml | 4 -- packages/protocol/src/validator.ts | 2 - tests/library/coverage.spec.ts | 49 +++++++++++------ 15 files changed, 94 insertions(+), 111 deletions(-) diff --git a/docs/src/api/class-tracing.md b/docs/src/api/class-tracing.md index 0241832b95b15..906aece2961f6 100644 --- a/docs/src/api/class-tracing.md +++ b/docs/src/api/class-tracing.md @@ -151,7 +151,7 @@ a timeline preview. * langs: js - `coverage` <[boolean]> -Whether to collect code coverage from istanbul-instrumented application code. Build the application with an istanbul instrumentation plugin, for example [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right before navigations, and stores them in istanbul format inside the trace file. Use [`method: Tracing.flushCoverage`] to collect counters from a page that is about to be closed by in-page script. +Whether to collect code coverage from istanbul-instrumented application code. Build the application with an istanbul instrumentation plugin, for example [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right before navigations and page closes, and stores them in istanbul format inside the trace file. ### option: Tracing.start.snapshots * since: v1.12 @@ -486,14 +486,6 @@ Specifies a custom location for the group to be shown in the trace viewer. Defau Closes the last group created by [`method: Tracing.group`]. -## async method: Tracing.flushCoverage -* since: v1.64 -* langs: js - -Collects istanbul coverage counters accumulated by all pages of the browser context into the coverage recorded by the trace. Requires tracing with the [`option: Tracing.start.coverage`] option to be started first. - -Coverage is collected automatically when the trace chunk is stopped and when a page is closed with [`method: Page.close`]. Call this method before a page is closed by in-page script, for example a popup closing itself, so that its counters are not lost. - ## async method: Tracing.stop * since: v1.12 diff --git a/packages/injected/src/coverageScript.ts b/packages/injected/src/coverageScript.ts index 24714c9c94706..1852ece1602be 100644 --- a/packages/injected/src/coverageScript.ts +++ b/packages/injected/src/coverageScript.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { kCoverageStashPrefix } from '@isomorphic/istanbulCoverage'; +import { kCoverageStashError, kCoverageStashPrefix } from '@isomorphic/istanbulCoverage'; import type { IstanbulCoverage, IstanbulCoverageDelta, IstanbulFileCoverageDelta } from '@isomorphic/istanbulCoverage'; @@ -87,35 +87,50 @@ export class CoverageScript { const delta = this._takeCurrent(); if (!delta) return; + // Several documents can pick up the same stash, the id discards the copies. + const id = Math.random().toString(36).slice(2); try { - // Several documents can pick up the same stash, the id discards the copies. - const id = Math.random().toString(36).slice(2); - const key = kCoverageStashPrefix + this._sessionId + '.' + id; - this._global.localStorage.setItem(key, JSON.stringify({ id, data: delta })); - } catch { + this._global.localStorage.setItem(kCoverageStashPrefix + this._sessionId + '.' + id, JSON.stringify({ id, data: delta })); + } catch (error) { + // The counters are lost, the next take surfaces the error. + try { + this._global.localStorage.setItem(stashErrorKey(this._sessionId), String(error)); + } catch { + } } } } +function stashErrorKey(sessionId: string) { + return kCoverageStashPrefix + 'error.' + sessionId; +} + // Stashes of other sessions are stale, e.g. left by a previous run in a persistent profile. export function takeCoverageStashes(global: typeof globalThis, sessionId: string): string[] { const result: string[] = []; + let storage: Storage; try { - const storage = global.localStorage; - const sessionPrefix = kCoverageStashPrefix + sessionId + '.'; - const keys: string[] = []; - for (let i = 0; i < storage.length; i++) { - const key = storage.key(i); - if (key && key.startsWith(kCoverageStashPrefix)) - keys.push(key); - } - for (const key of keys) { - const json = key.startsWith(sessionPrefix) ? storage.getItem(key) : undefined; - storage.removeItem(key); - if (json) - result.push(json); - } + storage = global.localStorage; } catch { + return result; + } + const error = storage.getItem(stashErrorKey(sessionId)); + if (error) { + storage.removeItem(stashErrorKey(sessionId)); + throw new Error(kCoverageStashError + ': ' + error); + } + const sessionPrefix = kCoverageStashPrefix + sessionId + '.'; + const keys: string[] = []; + for (let i = 0; i < storage.length; i++) { + const key = storage.key(i); + if (key && key.startsWith(kCoverageStashPrefix)) + keys.push(key); + } + for (const key of keys) { + const json = key.startsWith(sessionPrefix) ? storage.getItem(key) : undefined; + storage.removeItem(key); + if (json) + result.push(json); } return result; } diff --git a/packages/isomorphic/istanbulCoverage.ts b/packages/isomorphic/istanbulCoverage.ts index 7cc9cc4547722..7c0111c6f4371 100644 --- a/packages/isomorphic/istanbulCoverage.ts +++ b/packages/isomorphic/istanbulCoverage.ts @@ -58,6 +58,7 @@ export type IstanbulFileCoverageDelta = Partial & Pick([ ['Tracing.tracingStartChunk', { title: 'Start tracing', group: 'configuration', }], ['Tracing.tracingGroup', { title: 'Trace "{name}"', renderParams: ['name'], }], ['Tracing.tracingGroupEnd', { title: 'Group end', }], - ['Tracing.tracingFlushCoverage', { title: 'Flush coverage', group: 'configuration', }], ['Tracing.tracingStopChunk', { title: 'Stop tracing', group: 'configuration', }], ['Tracing.tracingStop', { title: 'Stop tracing', group: 'configuration', }], ['Tracing.harStart', { internal: true, }], diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts index 40ecd084a845e..5fcd6b1c96abe 100644 --- a/packages/playwright-client/types/types.d.ts +++ b/packages/playwright-client/types/types.d.ts @@ -23204,18 +23204,6 @@ export interface Touchscreen { * */ export interface Tracing { - /** - * Collects istanbul coverage counters accumulated by all pages of the browser context into the coverage recorded by - * the trace. Requires tracing with the - * [`coverage`](https://playwright.dev/docs/api/class-tracing#tracing-start-option-coverage) option to be started - * first. - * - * Coverage is collected automatically when the trace chunk is stopped and when a page is closed with - * [page.close([options])](https://playwright.dev/docs/api/class-page#page-close). Call this method before a page is - * closed by in-page script, for example a popup closing itself, so that its counters are not lost. - */ - flushCoverage(): Promise; - /** * **NOTE** Use `test.step` instead when available. * @@ -23286,9 +23274,7 @@ export interface Tracing { * [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or * [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the * `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right - * before navigations, and stores them in istanbul format inside the trace file. Use - * [tracing.flushCoverage()](https://playwright.dev/docs/api/class-tracing#tracing-flush-coverage) to collect counters - * from a page that is about to be closed by in-page script. + * before navigations and page closes, and stores them in istanbul format inside the trace file. */ coverage?: boolean; diff --git a/packages/playwright-core/src/client/channels.d.ts b/packages/playwright-core/src/client/channels.d.ts index 40da002b32d2c..6eee244b3afe8 100644 --- a/packages/playwright-core/src/client/channels.d.ts +++ b/packages/playwright-core/src/client/channels.d.ts @@ -5064,7 +5064,6 @@ export interface TracingChannel extends TracingEventTarget, Channel { tracingStartChunk(params: TracingTracingStartChunkParams, options: TimeoutOptions): Promise; tracingGroup(params: TracingTracingGroupParams, options: TimeoutOptions): Promise; tracingGroupEnd(params: TracingTracingGroupEndParams, options: TimeoutOptions): Promise; - tracingFlushCoverage(params: TracingTracingFlushCoverageParams, options: TimeoutOptions): Promise; tracingStopChunk(params: TracingTracingStopChunkParams, options: TimeoutOptions): Promise; tracingStop(params: TracingTracingStopParams, options: TimeoutOptions): Promise; harStart(params: TracingHarStartParams, options: TimeoutOptions): Promise; @@ -5119,9 +5118,6 @@ export type TracingTracingGroupResult = void; export type TracingTracingGroupEndParams = {}; export type TracingTracingGroupEndOptions = {}; export type TracingTracingGroupEndResult = void; -export type TracingTracingFlushCoverageParams = {}; -export type TracingTracingFlushCoverageOptions = {}; -export type TracingTracingFlushCoverageResult = void; export type TracingTracingStopChunkParams = { mode: 'archive' | 'discard' | 'entries', }; diff --git a/packages/playwright-core/src/client/tracing.ts b/packages/playwright-core/src/client/tracing.ts index 11ea7397d472a..928f8615e8133 100644 --- a/packages/playwright-core/src/client/tracing.ts +++ b/packages/playwright-core/src/client/tracing.ts @@ -79,10 +79,6 @@ export class Tracing extends ChannelOwner implements ap await this._channel.tracingGroupEnd({}, kNoTimeout); } - async flushCoverage() { - await this._channel.tracingFlushCoverage({}, kNoTimeout); - } - private async _startCollectingStacks(traceName: string, live: boolean) { if (!this._isTracing) { this._isTracing = true; diff --git a/packages/playwright-core/src/server/channels.d.ts b/packages/playwright-core/src/server/channels.d.ts index 7655960f56a63..7edbe1ba7fa5a 100644 --- a/packages/playwright-core/src/server/channels.d.ts +++ b/packages/playwright-core/src/server/channels.d.ts @@ -5065,7 +5065,6 @@ export interface TracingChannel extends TracingEventTarget, Channel { tracingStartChunk(params: TracingTracingStartChunkParams, progress: Progress): Promise; tracingGroup(params: TracingTracingGroupParams, progress: Progress): Promise; tracingGroupEnd(params: TracingTracingGroupEndParams, progress: Progress): Promise; - tracingFlushCoverage(params: TracingTracingFlushCoverageParams, progress: Progress): Promise; tracingStopChunk(params: TracingTracingStopChunkParams, progress: Progress): Promise; tracingStop(params: TracingTracingStopParams, progress: Progress): Promise; harStart(params: TracingHarStartParams, progress: Progress): Promise; @@ -5120,9 +5119,6 @@ export type TracingTracingGroupResult = void; export type TracingTracingGroupEndParams = {}; export type TracingTracingGroupEndOptions = {}; export type TracingTracingGroupEndResult = void; -export type TracingTracingFlushCoverageParams = {}; -export type TracingTracingFlushCoverageOptions = {}; -export type TracingTracingFlushCoverageResult = void; export type TracingTracingStopChunkParams = { mode: 'archive' | 'discard' | 'entries', }; diff --git a/packages/playwright-core/src/server/coverageRecorder.ts b/packages/playwright-core/src/server/coverageRecorder.ts index f0a19d5f9da19..2726546c6c46d 100644 --- a/packages/playwright-core/src/server/coverageRecorder.ts +++ b/packages/playwright-core/src/server/coverageRecorder.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { mergeIstanbulCoverage, sortedIstanbulCoverage } from '@isomorphic/istanbulCoverage'; +import { kCoverageStashError, mergeIstanbulCoverage, sortedIstanbulCoverage } from '@isomorphic/istanbulCoverage'; import * as rawCoverageSource from '../generated/coverageScriptSource'; import type { BrowserContext } from './browserContext'; @@ -31,6 +31,7 @@ export class CoverageRecorder { private _context: BrowserContext; private _coverage = new Map(); private _stashedChunkIds = new Set(); + private _stashError: Error | undefined; private _initScript: InitScript | undefined; constructor(context: BrowserContext) { @@ -75,6 +76,8 @@ export class CoverageRecorder { return; } await this.flush(progress); + if (this._stashError) + throw this._stashError; this._stashedChunkIds.clear(); if (!this._coverage.size) return; @@ -92,7 +95,7 @@ export class CoverageRecorder { private async _collectFromPage(page: Page) { await Promise.all(page.frames().map(async frame => { - const chunks: string[] = await frame.nonStallingRawEvaluateInExistingMainContext(coverageTakeExpression).catch(() => []); + const chunks: string[] = await frame.nonStallingRawEvaluateInExistingMainContext(coverageTakeExpression).catch(error => this._takeFailed(error)); for (const json of chunks) this._append(json); })); @@ -115,12 +118,20 @@ export class CoverageRecorder { return; const source = this._moduleExpression(`return (module.exports.takeCoverageStashes())(window, ${this._sessionId()});`); await this._context.visitOrigins(progress, origins, async frame => { - const chunks: string[] = await frame.evaluateExpression(progress, source, { world: 'main' }).catch(() => []); + const chunks: string[] = await frame.evaluateExpression(progress, source, { world: 'main' }).catch(error => this._takeFailed(error)); for (const json of chunks) this._append(json); }); } + // Evaluation fails for pages on their way out, only the stash error matters. + private _takeFailed(error: Error): string[] { + const message = error.message?.split('\n')[0].replace(/^Error: /, ''); + if (message?.includes(kCoverageStashError)) + this._stashError ??= new Error(message); + return []; + } + private _append(json: string) { try { const chunk: IstanbulCoverageChunk = JSON.parse(json); diff --git a/packages/playwright-core/src/server/dispatchers/tracingDispatcher.ts b/packages/playwright-core/src/server/dispatchers/tracingDispatcher.ts index fc3c0d990d88c..950ab8401bb89 100644 --- a/packages/playwright-core/src/server/dispatchers/tracingDispatcher.ts +++ b/packages/playwright-core/src/server/dispatchers/tracingDispatcher.ts @@ -56,10 +56,6 @@ export class TracingDispatcher extends Dispatcher { - await this._object.flushCoverage(progress); - } - async tracingStopChunk(params: channels.TracingTracingStopChunkParams, progress: Progress): Promise { const { artifact, entries } = await this._object.stopChunk(progress, params); return { artifact: artifact ? ArtifactDispatcher.from(this, artifact) : undefined, entries }; diff --git a/packages/playwright-core/src/server/trace/recorder/tracing.ts b/packages/playwright-core/src/server/trace/recorder/tracing.ts index db3278073c920..df55fcaee8dab 100644 --- a/packages/playwright-core/src/server/trace/recorder/tracing.ts +++ b/packages/playwright-core/src/server/trace/recorder/tracing.ts @@ -397,13 +397,6 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps this._groupEnd(); } - async flushCoverage(progress: Progress) { - const recorder = this._coverageRecorder; - if (!recorder) - throw new Error(`Coverage collection has not been started. Pass the "coverage" option to tracing.start().`); - await recorder.flush(progress); - } - // Collected before the chunk stops, while the pages can still be evaluated in. private async _takeCoverage(progress: Progress, mode: TracingTracingStopChunkParams['mode']): Promise { const recorder = this._coverageRecorder; @@ -422,8 +415,17 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps throw new Error(`Tracing is already stopping`); this._isStopping = true; try { - const coverageFile = await this._takeCoverage(progress, params.mode); + let coverageFile: string | undefined; + let coverageError: Error | undefined; + try { + coverageFile = await this._takeCoverage(progress, params.mode); + } catch (error) { + coverageError = error; + } + // The chunk is torn down either way, the coverage error is reported after. const result = this._stopChunk(params, coverageFile); + if (coverageError) + throw coverageError; if (!result) return {}; diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index 40ecd084a845e..5fcd6b1c96abe 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -23204,18 +23204,6 @@ export interface Touchscreen { * */ export interface Tracing { - /** - * Collects istanbul coverage counters accumulated by all pages of the browser context into the coverage recorded by - * the trace. Requires tracing with the - * [`coverage`](https://playwright.dev/docs/api/class-tracing#tracing-start-option-coverage) option to be started - * first. - * - * Coverage is collected automatically when the trace chunk is stopped and when a page is closed with - * [page.close([options])](https://playwright.dev/docs/api/class-page#page-close). Call this method before a page is - * closed by in-page script, for example a popup closing itself, so that its counters are not lost. - */ - flushCoverage(): Promise; - /** * **NOTE** Use `test.step` instead when available. * @@ -23286,9 +23274,7 @@ export interface Tracing { * [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or * [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the * `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right - * before navigations, and stores them in istanbul format inside the trace file. Use - * [tracing.flushCoverage()](https://playwright.dev/docs/api/class-tracing#tracing-flush-coverage) to collect counters - * from a page that is about to be closed by in-page script. + * before navigations and page closes, and stores them in istanbul format inside the trace file. */ coverage?: boolean; diff --git a/packages/protocol/spec/tracing.yml b/packages/protocol/spec/tracing.yml index ed9bafb812174..054e06d8bb627 100644 --- a/packages/protocol/spec/tracing.yml +++ b/packages/protocol/spec/tracing.yml @@ -54,10 +54,6 @@ Tracing: tracingGroupEnd: title: Group end - tracingFlushCoverage: - title: Flush coverage - group: configuration - tracingStopChunk: title: Stop tracing group: configuration diff --git a/packages/protocol/src/validator.ts b/packages/protocol/src/validator.ts index 0cf8a41e9bd03..86f1e91255607 100644 --- a/packages/protocol/src/validator.ts +++ b/packages/protocol/src/validator.ts @@ -3087,8 +3087,6 @@ scheme.TracingTracingGroupParams = tObject({ scheme.TracingTracingGroupResult = tOptional(tObject({})); scheme.TracingTracingGroupEndParams = tOptional(tObject({})); scheme.TracingTracingGroupEndResult = tOptional(tObject({})); -scheme.TracingTracingFlushCoverageParams = tOptional(tObject({})); -scheme.TracingTracingFlushCoverageResult = tOptional(tObject({})); scheme.TracingTracingStopChunkParams = tObject({ mode: tEnum(['archive', 'discard', 'entries']), }); diff --git a/tests/library/coverage.spec.ts b/tests/library/coverage.spec.ts index ad9e3290b9069..53441d41b1537 100644 --- a/tests/library/coverage.spec.ts +++ b/tests/library/coverage.spec.ts @@ -37,7 +37,6 @@ it('should collect istanbul coverage into the trace', async ({ browser }, testIn const page = await context.newPage(); await page.setContent(coverageScript('a.js', 3)); - await context.tracing.flushCoverage(); // Coverage of a closed page is collected automatically. const page2 = await context.newPage(); @@ -107,14 +106,12 @@ it('should report maps once and counters incrementally', async ({ browser, serve await context.close(); }); -it('should accumulate counters across flushes and keep never hit files', async ({ browser }, testInfo) => { +it('should accumulate counters across pulls and keep never hit files', async ({ browser }, testInfo) => { const context = await browser.newContext(); await context.tracing.start({ coverage: true }); const page = await context.newPage(); await page.setContent(coverageScript('a.js', 0)); - await context.tracing.flushCoverage(); await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'] += 4); - await context.tracing.flushCoverage(); await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'] += 3); const traceFile = testInfo.outputPath('trace.zip'); @@ -129,7 +126,7 @@ it('should accumulate counters across flushes and keep never hit files', async ( async function openPopup(page: any, server: any, file: string, s0: number) { const [popup] = await Promise.all([ page.waitForEvent('popup'), - page.evaluate(url => window.open(url), server.EMPTY_PAGE), + page.evaluate(url => (window as any).__popup = window.open(url), server.EMPTY_PAGE), ]); await popup.evaluate(coverage => (window as any).__coverage__ = JSON.parse(coverage), JSON.stringify(fileCoverage(file, s0))); return popup; @@ -188,7 +185,6 @@ it('should discard stashes of other sessions', async ({ browser, server }, testI localStorage.setItem('__pwCoverage.other-session.1', JSON.stringify({ id: 'other', data: JSON.parse(coverage) })); }, JSON.stringify(fileCoverage('stale.js', 7))); - await context.tracing.flushCoverage(); const remaining = await page.evaluate(() => Object.keys(localStorage).filter(key => key.startsWith('__pwCoverage.'))); expect(remaining).toEqual([]); @@ -207,13 +203,18 @@ it('should collect coverage of an origin left without a page', async ({ browser, await page.goto(server.EMPTY_PAGE); const popup = await openPopup(page, server, 'popup.js', 0); - await context.tracing.flushCoverage(); - - // The popup is hit and navigates away, leaving a stash behind. await popup.evaluate(url => { - (window as any).__coverage__['popup.js'].s['0'] = 5; - window.location.href = url; + (window as any).__go = () => { + (window as any).__coverage__['popup.js'].s['0'] = 5; + window.location.href = url; + }; }, server.CROSS_PROCESS_PREFIX + '/empty.html'); + // The popup is hit and navigates away, leaving a stash behind. Triggered from + // the opener, so that the pull after the action does not race the navigation. + await Promise.all([ + popup.waitForURL(server.CROSS_PROCESS_PREFIX + '/empty.html'), + page.evaluate(() => (window as any).__popup.__go()), + ]); // Leave the origin of the stash without a page to relay it. await page.goto(server.CROSS_PROCESS_PREFIX + '/empty.html'); @@ -230,10 +231,8 @@ it('should pull counters as the actions go', async ({ browser, server }, testInf await context.tracing.start({ coverage: true }); const page = await context.newPage(); await page.goto(server.EMPTY_PAGE); - await page.evaluate(coverage => (window as any).__coverage__ = JSON.parse(coverage), JSON.stringify(fileCoverage('a.js', 3))); - - await page.click('body'); - // The counters are collected by the actions themselves, not only by the flush. + await page.setContent(coverageScript('a.js', 3)); + // The counters are collected by the action itself, not only by the final flush. expect(await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'])).toBe(0); const traceFile = testInfo.outputPath('trace.zip'); @@ -291,9 +290,23 @@ it('should stop collecting when tracing stops', async ({ browser, server }, test await context.close(); }); -it('should throw when flushing without coverage', async ({ browser }) => { +it('should surface a failure to stash the coverage', async ({ browser, server }, testInfo) => { const context = await browser.newContext(); - await context.newPage(); - await expect(context.tracing.flushCoverage()).rejects.toThrow('Coverage collection has not been started'); + await context.tracing.start({ coverage: true }); + const page = await context.newPage(); + await page.goto(server.EMPTY_PAGE); + await page.evaluate(coverage => { + (window as any).__coverage__ = JSON.parse(coverage); + // The stash does not fit, the error record still does. + const setItem = Storage.prototype.setItem; + Storage.prototype.setItem = function(key: string, value: string) { + if (value.length > 100) + throw new DOMException('The quota has been exceeded.', 'QuotaExceededError'); + setItem.call(this, key, value); + }; + window.dispatchEvent(new Event('pagehide')); + }, JSON.stringify(fileCoverage('a.js', 3))); + + await expect(context.tracing.stop({ path: testInfo.outputPath('trace.zip') })).rejects.toThrow('Failed to stash the coverage: QuotaExceededError: The quota has been exceeded.'); await context.close(); }); From 496936594c44157bbc5e5a1317c373e8e8da20d3 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Wed, 16 Sep 2026 13:12:50 -0700 Subject: [PATCH 20/21] chore(instrumentation): rename onPageClose to onPageDidClose Pairs with onPageWillClose. --- packages/playwright-core/src/server/debugController.ts | 2 +- packages/playwright-core/src/server/instrumentation.ts | 4 ++-- packages/playwright-core/src/server/page.ts | 2 +- packages/playwright-core/src/server/playwright.ts | 2 +- packages/playwright-core/src/server/trace/recorder/tracing.ts | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/playwright-core/src/server/debugController.ts b/packages/playwright-core/src/server/debugController.ts index 7ad8b2d4cc1da..48bca18fcce56 100644 --- a/packages/playwright-core/src/server/debugController.ts +++ b/packages/playwright-core/src/server/debugController.ts @@ -68,7 +68,7 @@ export class DebugController extends SdkObject { if (enabled && !this._trackHierarchyListener) { this._trackHierarchyListener = { onPageOpen: () => this._emitSnapshot(false), - onPageClose: () => this._emitSnapshot(false), + onPageDidClose: () => this._emitSnapshot(false), }; this._playwright.instrumentation.addListener(this._trackHierarchyListener, null); this._emitSnapshot(true); diff --git a/packages/playwright-core/src/server/instrumentation.ts b/packages/playwright-core/src/server/instrumentation.ts index 6f82daf56ab76..a9aa05b3f99ec 100644 --- a/packages/playwright-core/src/server/instrumentation.ts +++ b/packages/playwright-core/src/server/instrumentation.ts @@ -112,7 +112,7 @@ export interface Instrumentation { onAfterCall(progress: Progress, sdkObject: SdkObject): Promise; onPageOpen(page: Page): void; onPageWillClose(page: Page): Promise; - onPageClose(page: Page): void; + onPageDidClose(page: Page): void; onBrowserOpen(browser: Browser): void; onBrowserClose(browser: Browser): void; onDialog(dialog: Dialog): void; @@ -126,7 +126,7 @@ export interface InstrumentationListener { onAfterCall?(progress: Progress, sdkObject: SdkObject): Promise; onPageOpen?(page: Page): void; onPageWillClose?(page: Page): Promise; - onPageClose?(page: Page): void; + onPageDidClose?(page: Page): void; onBrowserOpen?(browser: Browser): void; onBrowserClose?(browser: Browser): void; onDialog?(dialog: Dialog): void; diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts index ea7beb3422524..0c5d10398148f 100644 --- a/packages/playwright-core/src/server/page.ts +++ b/packages/playwright-core/src/server/page.ts @@ -316,7 +316,7 @@ export class Page extends SdkObject { this.browserContext.emit(BrowserContext.Events.PageClosed, this); this.closedPromise.resolve(); if (!this.isStorageStatePage) - this.instrumentation.onPageClose(this); + this.instrumentation.onPageDidClose(this); } _didCrash() { diff --git a/packages/playwright-core/src/server/playwright.ts b/packages/playwright-core/src/server/playwright.ts index dc129c7e23714..9b04c951a011b 100644 --- a/packages/playwright-core/src/server/playwright.ts +++ b/packages/playwright-core/src/server/playwright.ts @@ -56,7 +56,7 @@ export class Playwright extends SdkObject { onBrowserOpen: browser => this._allBrowsers.add(browser), onBrowserClose: browser => this._allBrowsers.delete(browser), onPageOpen: page => this._allPages.add(page), - onPageClose: page => this._allPages.delete(page), + onPageDidClose: page => this._allPages.delete(page), }, null); this.chromium = new Chromium(this, new BidiChromium(this)); this.firefox = new Firefox(this, new BidiFirefox(this)); diff --git a/packages/playwright-core/src/server/trace/recorder/tracing.ts b/packages/playwright-core/src/server/trace/recorder/tracing.ts index df55fcaee8dab..67b163b1fe5bf 100644 --- a/packages/playwright-core/src/server/trace/recorder/tracing.ts +++ b/packages/playwright-core/src/server/trace/recorder/tracing.ts @@ -721,7 +721,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps this._appendTraceEvent(event); } - onPageClose(page: Page) { + onPageDidClose(page: Page) { const event: trace.EventTraceEvent = { type: 'event', time: monotonicTime(), From 3d655ff340e888383f51302f3b1efff53f9b85d8 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Wed, 16 Sep 2026 13:12:50 -0700 Subject: [PATCH 21/21] test(coverage): drop the connect coverage test --- tests/library/browsertype-connect.spec.ts | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/tests/library/browsertype-connect.spec.ts b/tests/library/browsertype-connect.spec.ts index f73f647636c74..269745b8d1b5a 100644 --- a/tests/library/browsertype-connect.spec.ts +++ b/tests/library/browsertype-connect.spec.ts @@ -741,26 +741,6 @@ for (const kind of ['launchServer', 'run-server'] as const) { expect(sourceFile).toEqual(thisFile); }); - test('should record trace with coverage', async ({ connect, startRemoteServer, trace }, testInfo) => { - test.skip(trace === 'on'); - const remoteServer = await startRemoteServer(kind); - const browser = await connect(remoteServer.wsEndpoint()); - const context = await browser.newContext(); - const page = await context.newPage(); - - await context.tracing.start({ coverage: true }); - const fileCov = { path: 'a.js', statementMap: { '0': { start: { line: 1, column: 0 }, end: { line: 1, column: 20 } } }, fnMap: {}, branchMap: {}, s: { '0': 3 }, f: {}, b: {} }; - await page.setContent(``); - await context.tracing.stop({ path: testInfo.outputPath('trace1.zip') }); - - await context.close(); - await browser.close(); - - const { resources } = await parseTraceRaw(testInfo.outputPath('trace1.zip')); - const data = JSON.parse(resources.get('trace.coverage')!.toString()); - expect(data['a.js'].s['0']).toBe(3); - }); - test('should fulfill with global fetch result', async ({ connect, startRemoteServer, playwright, server }) => { const remoteServer = await startRemoteServer(kind); const browser = await connect(remoteServer.wsEndpoint());