diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.md index 0c9ab8942d158..52ba2867ce2a5 100644 --- a/docs/src/test-cli-js.md +++ b/docs/src/test-cli-js.md @@ -103,6 +103,7 @@ npx playwright test --ui | `--quiet` | Suppress stdio. | | `--repeat-each ` | Run each test `N` times (default: 1). | | `--reporter ` | Reporter to use, comma-separated, can be "dot", "line", "list", or others (default: "list" locally and "dot" on CI). You can also pass a path to a custom reporter file. | +| `--reporter-only-failures` | Report only failed, flaky, and interrupted tests. Terminal reporters print failure details live when each attempt finishes and retain the full run summary. HTML, JSON, JUnit, and Perfetto reports contain only the included results. Blob reports remain complete. Custom reporters receive the `onlyFailures` constructor option. `--list` output is unchanged. | | `--retries ` | Maximum retry count for flaky tests, zero for no retries (default: no retries). | | `--run-agents ` | Run agents to generate the code for `page.perform`. Possible values are "missing", "all" and "none" (default: "none"). See [test-agents](./test-agents.md). | | `--shard ` | Shard tests and execute only the selected shard, specified in the form "current/all", 1-based, e.g., "3/5". | @@ -320,6 +321,7 @@ npx playwright merge-reports ./reports | :--- | :--- | | `-c, --config ` | Configuration file. Can be used to specify additional configuration for the output report | | `--reporter ` | Reporter to use, comma-separated, can be "list", "line", "dot", "json", "junit", "null", "github", "html", "blob" (default: "list" locally and "dot" on CI) | +| `--reporter-only-failures` | Report only failed, flaky, and interrupted tests in terminal, HTML, JSON, JUnit, and Perfetto output. Blob reports remain complete. Custom reporters receive the `onlyFailures` constructor option. | ### Clear Cache diff --git a/docs/src/test-reporter-api/class-reporter.md b/docs/src/test-reporter-api/class-reporter.md index 68a2a21dab9b5..64fa01a3b1fdf 100644 --- a/docs/src/test-reporter-api/class-reporter.md +++ b/docs/src/test-reporter-api/class-reporter.md @@ -37,11 +37,11 @@ module.exports = MyReporter; ```js tab=js-ts title="my-awesome-reporter.ts" import type { - Reporter, FullConfig, Suite, TestCase, TestResult, FullResult + Reporter, ReporterOptions, FullConfig, Suite, TestCase, TestResult, FullResult } from '@playwright/test/reporter'; class MyReporter implements Reporter { - constructor(options: { customOption?: string } = {}) { + constructor(options: ReporterOptions & { customOption?: string } = {}) { console.log(`my-awesome-reporter setup with customOption set to ${options.customOption}`); } @@ -74,6 +74,16 @@ export default defineConfig({ }); ``` +**Reporter options** + +The reporter constructor receives its configured options together with common options described by the `ReporterOptions` type: + +| Option | Description | +|---|---| +| `onlyFailures` | Whether to limit this reporter's output to failed, flaky, and interrupted tests. Can be configured for each reporter. `--reporter-only-failures` supplies `true`, overriding configuration. Each reporter constructor has the final say, including environment overrides. | + +This option does not filter or delay reporter callbacks. Reporters still receive the complete suite, every test result, and all timing information. + Here is a typical order of reporter calls: * [`method: Reporter.onBegin`] is called once with a root suite that contains all other suites and tests. Learn more about [suites hierarchy][Suite]. * [`method: Reporter.onTestBegin`] is called for each test run. It is given a [TestCase] that is executed, and a [TestResult] that is almost empty. Test result will be populated while the test runs (for example, with steps and stdio) and will get final `status` once the test finishes. diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.md index f05ac3568eb47..de58ab3f8442e 100644 --- a/docs/src/test-reporters-js.md +++ b/docs/src/test-reporters-js.md @@ -50,6 +50,35 @@ export default defineConfig({ }); ``` +### Report only failures + +Use `--reporter-only-failures` to report only failed, flaky, and interrupted tests: + +```bash +npx playwright test --reporter-only-failures +npx playwright merge-reports --reporter=html --reporter-only-failures ./blob-report +``` + +The `dot`, `line`, and `list` reporters use the same terminal output when `onlyFailures` is enabled. Each failed attempt's details are printed as soon as that attempt finishes, including failures that will be retried. There is no progress output, and failure details are not repeated at the end. The final summary retains the full run counts and timing information. + +Generated HTML, JSON, JUnit, and Perfetto reports are written at the end of the run and omit passing tests, tests with expected failures, and skipped tests. Their test counts describe the included results. + +Blob reports always retain every test, result, and attachment so they can be replayed into either complete or filtered reports. Apply `--reporter-only-failures` when merging to filter the generated reports without changing the blob data. + +The flag overrides reporter configuration. Each terminal reporter then resolves its environment settings in its constructor. Custom reporters receive `onlyFailures: true` in their constructor options and can choose how to respect it. Their callbacks still receive all tests and results. `--list` output is unchanged. Test stdout and stderr remain controlled by [`property: TestConfig.quiet`]. + +You can also configure `onlyFailures` separately for each reporter. For example, keep terminal output concise while retaining a complete HTML report: + +```js title="playwright.config.ts" +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + reporter: [['list', { onlyFailures: true }], ['html']], +}); +``` + +For terminal reporters, the corresponding `PLAYWRIGHT_*_ONLY_FAILURES` environment variable overrides both the configuration option and the command line flag, including when set to `false` or `0`. Precedence is environment, then command line, then configuration. + ## Built-in reporters All built-in reporters show detailed information about failures, and mostly differ in verbosity for successful runs. @@ -107,6 +136,18 @@ export default defineConfig({ }); ``` +You can hide progress output while keeping failure details and the final summary: + +```js title="playwright.config.ts" +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + reporter: [['list', { onlyFailures: true }]], +}); +``` + +Failure details are always printed live when each attempt finishes. This takes precedence over both `printFailuresInline` and `printSteps` and does not suppress stdout or stderr from tests. + You can omit test tags that are automatically appended to test titles: ```js title="playwright.config.ts" @@ -133,6 +174,7 @@ List report supports the following configuration options and environment variabl |---|---|---|---| | `PLAYWRIGHT_LIST_PRINT_STEPS` | `printSteps` | Whether to print each step on its own line. | `false` | `PLAYWRIGHT_LIST_PRINT_FAILURES_INLINE` | `printFailuresInline` | Whether to print failure details immediately after a failed test instead of at the end. | `false` +| `PLAYWRIGHT_LIST_ONLY_FAILURES` | `onlyFailures` | Whether to print failure details live instead of progress output, followed by the final summary. | `false` | `PLAYWRIGHT_LIST_OMIT_TAGS` | `omitTags` | Whether to omit test tags that are automatically appended to test titles. | `false` | `PLAYWRIGHT_LIST_PRINT_WORKER_INDEX` | `printWorkerIndex` | Whether to prefix every line of the output with the index of the worker that produced it. | `false` | `PLAYWRIGHT_FORCE_TTY` | | Whether to produce output suitable for a live terminal. Supports `true`, `1`, `false`, `0`, `[WIDTH]`, and `[WIDTH]x[HEIGHT]`. `[WIDTH]` and `[WIDTH]x[HEIGHT]` specifies the TTY dimensions. | `true` when terminal is in TTY mode, `false` otherwise. @@ -170,10 +212,23 @@ Running 124 tests using 6 workers [23/124] gitignore.spec.ts - should respect nested .gitignore ``` +You can hide progress output while keeping failure details and the final summary: + +```js title="playwright.config.ts" +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + reporter: [['line', { onlyFailures: true }]], +}); +``` + +This does not suppress stdout or stderr from tests. Use [`property: TestConfig.quiet`] to suppress that output. + Line report supports the following configuration options and environment variables: | Environment Variable Name | Reporter Config Option| Description | Default |---|---|---|---| +| `PLAYWRIGHT_LINE_ONLY_FAILURES` | `onlyFailures` | Whether to print failure details live instead of progress output, followed by the final summary. | `false` | `PLAYWRIGHT_LINE_OMIT_TAGS` | `omitTags` | Whether to omit test tags that are automatically appended to test titles. | `false` | `PLAYWRIGHT_FORCE_TTY` | | Whether to produce output suitable for a live terminal. Supports `true`, `1`, `false`, `0`, `[WIDTH]`, and `[WIDTH]x[HEIGHT]`. `[WIDTH]` and `[WIDTH]x[HEIGHT]` specifies the TTY dimensions. | `true` when terminal is in TTY mode, `false` otherwise. | `FORCE_COLOR` | | Whether to produce colored output. | `true` when terminal is in TTY mode, `false` otherwise. @@ -214,10 +269,23 @@ One character is displayed for each test that has run, indicating its status: | `T` | Timed out | `°` | Skipped +You can hide progress output while keeping failure details and the final summary: + +```js title="playwright.config.ts" +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + reporter: [['dot', { onlyFailures: true }]], +}); +``` + +This does not suppress stdout or stderr from tests. Use [`property: TestConfig.quiet`] to suppress that output. + Dot report supports the following configuration options and environment variables: | Environment Variable Name | Reporter Config Option| Description | Default |---|---|---|---| +| `PLAYWRIGHT_DOT_ONLY_FAILURES` | `onlyFailures` | Whether to print failure details live instead of progress output, followed by the final summary. | `false` | `PLAYWRIGHT_DOT_OMIT_TAGS` | `omitTags` | Whether to omit test tags that are automatically appended to test titles. | `false` | `PLAYWRIGHT_FORCE_TTY` | | Whether to produce output suitable for a live terminal. Supports `true`, `1`, `false`, `0`, `[WIDTH]`, and `[WIDTH]x[HEIGHT]`. `[WIDTH]` and `[WIDTH]x[HEIGHT]` specifies the TTY dimensions. | `true` when terminal is in TTY mode, `false` otherwise. | `FORCE_COLOR` | | Whether to produce colored output. | `true` when terminal is in TTY mode, `false` otherwise. diff --git a/packages/playwright/src/cli/reportActions.ts b/packages/playwright/src/cli/reportActions.ts index 3b8943ebc5243..599de78429498 100644 --- a/packages/playwright/src/cli/reportActions.ts +++ b/packages/playwright/src/cli/reportActions.ts @@ -29,7 +29,8 @@ export async function showReport(report: string | undefined, host: string, port: export async function mergeReports(reportDir: string | undefined, opts: { [key: string]: any }) { const configFile = opts.config; - const config = configFile ? await configLoader.loadConfigFromFile(configFile) : await configLoader.loadEmptyConfigForMergeReports(); + const overrides = { reporterOnlyFailures: opts.reporterOnlyFailures ? true : undefined }; + const config = configFile ? await configLoader.loadConfigFromFile(configFile, overrides) : await configLoader.loadEmptyConfigForMergeReports(overrides); const dir = path.resolve(process.cwd(), reportDir || ''); const dirStat = await fs.promises.stat(dir).catch(e => null); diff --git a/packages/playwright/src/cli/testActions.ts b/packages/playwright/src/cli/testActions.ts index c03abcff8064f..04ba1c44d1bbd 100644 --- a/packages/playwright/src/cli/testActions.ts +++ b/packages/playwright/src/cli/testActions.ts @@ -126,6 +126,7 @@ function overridesFromOptions(options: { [key: string]: any }): ipc.ConfigCLIOve repeatEach: options.repeatEach ? parseInt(options.repeatEach, 10) : undefined, retries: options.retries ? parseInt(options.retries, 10) : undefined, reporter: resolveReporterOption(options.reporter), + reporterOnlyFailures: options.reporterOnlyFailures ? true : undefined, additionalReporters: resolveReporterOption(options.addReporter), shard: resolveShardOption(options.shard), shuffle: resolveShuffleOption(options.shuffle), diff --git a/packages/playwright/src/common/configLoader.ts b/packages/playwright/src/common/configLoader.ts index 6e2fa63afb98d..fdc9c6239a1e7 100644 --- a/packages/playwright/src/common/configLoader.ts +++ b/packages/playwright/src/common/configLoader.ts @@ -367,7 +367,7 @@ export async function loadConfigFromFile(configFile: string | undefined, overrid return await loadConfig(resolveConfigLocation(configFile), overrides, ignoreDeps); } -export async function loadEmptyConfigForMergeReports() { +export async function loadEmptyConfigForMergeReports(overrides?: ConfigCLIOverrides) { // Merge reports is "different" for no good reason. It should not pick up local config from the cwd. - return await loadConfig({ configDir: process.cwd() }); + return await loadConfig({ configDir: process.cwd() }, overrides); } diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts index 872e06b30eabc..523682307263f 100644 --- a/packages/playwright/src/common/ipc.ts +++ b/packages/playwright/src/common/ipc.ts @@ -36,6 +36,7 @@ export type ConfigCLIOverrides = { repeatEach?: number; retries?: number; reporter?: ReporterDescription[]; + reporterOnlyFailures?: boolean; additionalReporters?: ReporterDescription[]; shard?: { current: number, total: number }; shuffle?: string; diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index 9042f965224aa..471a4bff990c5 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -129,6 +129,7 @@ function addMergeReportsCommand(program: Command) { }); command.option('-c, --config ', `Configuration file. Can be used to specify additional configuration for the output report.`); command.option('--reporter ', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${config.defaultReporter}")`); + command.option('--reporter-only-failures', 'Only report failures with built-in reporters'); command.addHelpText('afterAll', ` Arguments [dir]: Directory containing blob reports. @@ -230,6 +231,7 @@ const testOptions: [string, { description: string, choices?: string[], preset?: ['--quiet', { description: `Suppress stdio` }], ['--repeat-each ', { description: `Run each test N times (default: 1)` }], ['--reporter ', { description: `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${config.defaultReporter}")` }], + ['--reporter-only-failures', { description: `Only report failures with built-in reporters` }], ['--retries ', { description: `Maximum retry count for flaky tests, zero for no retries (default: no retries)` }], ['--run-agents ', { description: `Run agents to generate the code for page.perform`, choices: ['missing', 'all', 'none'], preset: 'none' }], ['--shard ', { description: `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"` }], diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts index 00bbb0172bed9..8b1792a8842f0 100644 --- a/packages/playwright/src/reporters/base.ts +++ b/packages/playwright/src/reporters/base.ts @@ -27,7 +27,7 @@ import { fitToWidth } from '@utils/stringWidth'; import { resolveReporterOutputPath, stripAnsiEscapes } from '../util'; import type { ReporterV2 } from './reporterV2'; -import type { FullConfig, FullResult, Location, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter'; +import type { FullConfig, FullResult, Location, ReporterOptions, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter'; import type { Colors } from '@isomorphic/colors'; export type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' }; @@ -49,7 +49,7 @@ type TestSummary = { fatalErrors: TestError[]; }; -export type CommonReporterOptions = { +export type CommonReporterOptions = ReporterOptions & { configDir: string, _mode?: 'list' | 'test' | 'merge', _commandHash?: string, @@ -159,7 +159,7 @@ export const internalScreen: Screen = { resolveFiles: 'rootDir', }; -export type TerminalReporterOptions = { +export type TerminalReporterOptions = ReporterOptions & { screen?: TerminalScreen; omitFailures?: boolean; includeTestId?: boolean; @@ -168,19 +168,19 @@ export type TerminalReporterOptions = { }; export class TerminalReporter implements ReporterV2 { + readonly options: TerminalReporterOptions; screen: TerminalScreen; config!: FullConfig; suite!: Suite; totalTestCount = 0; result!: FullResult; private fileDurations = new Map }>(); - private _options: TerminalReporterOptions; private _fatalErrors: TestError[] = []; private _failureCount: number = 0; constructor(options: TerminalReporterOptions = {}) { this.screen = options.screen ?? terminalScreen; - this._options = options; + this.options = options; } version(): 'v2' { @@ -333,7 +333,7 @@ export class TerminalReporter implements ReporterV2 { epilogue(full: boolean) { const summary = this.generateSummary(); const summaryMessage = this.generateSummaryMessage(summary); - if (full && summary.failuresToPrint.length && !this._options.omitFailures) + if (full && summary.failuresToPrint.length && !this.options.omitFailures) this._printFailures(summary.failuresToPrint); this._printSlowTests(); this._printSummary(summaryMessage); @@ -365,15 +365,15 @@ export class TerminalReporter implements ReporterV2 { } formatTestTitle(test: TestCase, step?: TestStep): string { - return formatTestTitle(this.screen, this.config, test, step, this._options); + return formatTestTitle(this.screen, this.config, test, step, this.options); } formatTestHeader(test: TestCase, options: { indent?: string, index?: number, mode?: 'default' | 'error' } = {}): string { - return formatTestHeader(this.screen, this.config, test, { ...options, includeTestId: this._options.includeTestId, omitTags: this._options.omitTags }); + return formatTestHeader(this.screen, this.config, test, { ...options, includeTestId: this.options.includeTestId, omitTags: this.options.omitTags }); } formatFailure(test: TestCase, index?: number): string { - return formatFailure(this.screen, this.config, test, index, this._options); + return formatFailure(this.screen, this.config, test, index, this.options); } formatError(error: TestError): ErrorDetails { @@ -389,6 +389,34 @@ export class TerminalReporter implements ReporterV2 { } } +export function isFailure(test: TestCase): boolean { + const outcome = test.outcome(); + return outcome === 'unexpected' || outcome === 'flaky' || test.results.some(result => result.status === 'interrupted'); +} + +export type TestFilter = (test: TestCase) => boolean; + +export function createTestFilter(options: ReporterOptions): TestFilter | undefined { + return options.onlyFailures ? isFailure : undefined; +} + +export function* visitTests(suite: Suite, filter?: TestFilter): Generator { + for (const entry of suite.entries()) { + if (entry.type !== 'test') + yield* visitTests(entry, filter); + else if (!filter || filter(entry)) + yield entry; + } +} + +export function filterSuites(suites: Suite[], filter?: TestFilter): Suite[] { + return filter ? suites.filter(suite => !visitTests(suite, filter).next().done) : suites; +} + +export function filterSuiteEntries(suite: Suite, filter?: TestFilter): (Suite | TestCase)[] { + return filter ? suite.entries().filter(entry => entry.type === 'test' ? filter(entry) : !visitTests(entry, filter).next().done) : suite.entries(); +} + function formatResultErrors(screen: Screen, test: TestCase, result: TestResult): string { const lines: string[] = []; if (test.outcome() === 'unexpected') { diff --git a/packages/playwright/src/reporters/dot.ts b/packages/playwright/src/reporters/dot.ts index dd3812b44a8ae..f9f2057c2f3bb 100644 --- a/packages/playwright/src/reporters/dot.ts +++ b/packages/playwright/src/reporters/dot.ts @@ -26,7 +26,11 @@ class DotReporter extends TerminalReporter { private _counter = 0; constructor(options?: DotReporterOptions & CommonReporterOptions & TerminalReporterOptions) { - super({ ...options, omitTags: getAsBooleanFromENV('PLAYWRIGHT_DOT_OMIT_TAGS', options?.omitTags) }); + super({ + ...options, + onlyFailures: getAsBooleanFromENV('PLAYWRIGHT_DOT_ONLY_FAILURES', options?.onlyFailures), + omitTags: getAsBooleanFromENV('PLAYWRIGHT_DOT_OMIT_TAGS', options?.omitTags), + }); } override onBegin(suite: Suite) { diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts index 30abcc8254e21..1f6a32902c990 100644 --- a/packages/playwright/src/reporters/html.ts +++ b/packages/playwright/src/reporters/html.ts @@ -36,11 +36,12 @@ import { extractZip } from '@utils/third_party/extractZip'; // `import('vite')`) is DCE'd in release builds. declare const __PW_HMR__: boolean; -import { CommonReporterOptions, formatError, formatResultFailure, internalScreen } from './base'; +import { createTestFilter, filterSuiteEntries, filterSuites, formatError, formatResultFailure, internalScreen } from './base'; import * as babel from '../transform/babelBundle'; import { resolveReporterOutputPath, stripAnsiEscapes } from '../util'; import type { ReportConfigureParams, ReportEndParams, ReporterV2 } from './reporterV2'; +import type { CommonReporterOptions, TestFilter } from './base'; import type { HtmlReporterOptions as HtmlReporterConfigOptions, Metadata, TestAnnotation } from '../../types/test'; import type * as api from '../../types/testReporter'; import type { HTMLReport, HTMLReportOptions, Location, Stats, TestAttachment, TestCase, TestCaseSummary, TestFile, TestFileSummary, TestResult, TestStep } from '@html-reporter/types'; @@ -164,7 +165,7 @@ class HtmlReporter implements ReporterV2 { noSnippets, noCopyPrompt, mergeFiles, - }); + }, createTestFilter(this._options)); this._buildResult = await builder.build(this.config.metadata, projectSuites, result, this._topLevelErrors, this._machines); } @@ -336,13 +337,15 @@ class HtmlBuilder { private _attachmentsBaseURL: string; private _options: HTMLReportOptions; private _doNotInlineAssets: boolean; + private _testFilter: TestFilter | undefined; - constructor(yazl: typeof import('yazl'), config: api.FullConfig, outputDir: string, attachmentsBaseURL: string, doNotInlineAssets: boolean, options: HTMLReportOptions) { + constructor(yazl: typeof import('yazl'), config: api.FullConfig, outputDir: string, attachmentsBaseURL: string, doNotInlineAssets: boolean, options: HTMLReportOptions, testFilter?: TestFilter) { this._dataZipFile = new yazl.ZipFile(); this._config = config; this._reportFolder = outputDir; this._options = options; this._doNotInlineAssets = doNotInlineAssets; + this._testFilter = testFilter; fs.mkdirSync(this._reportFolder, { recursive: true }); this._attachmentsBaseURL = attachmentsBaseURL; } @@ -351,7 +354,7 @@ class HtmlBuilder { const data: DataMap = new Map(); for (const projectSuite of projectSuites) { const projectName = projectSuite.project()!.name; - for (const fileSuite of projectSuite.suites) { + for (const fileSuite of filterSuites(projectSuite.suites, this._testFilter)) { const fileName = this._relativeLocation(fileSuite.location)!.file; this._createEntryForSuite(data, projectName, fileSuite, fileName, true); } @@ -502,7 +505,7 @@ class HtmlBuilder { private _processSuite(suite: api.Suite, projectName: string, path: string[], deep: boolean, outTests: TestEntry[]) { const newPath = [...path, suite.title]; - suite.entries().forEach(e => { + filterSuiteEntries(suite, this._testFilter).forEach(e => { if (e.type === 'test') outTests.push(this._createTestEntry(e, projectName, newPath)); else if (deep) diff --git a/packages/playwright/src/reporters/json.ts b/packages/playwright/src/reporters/json.ts index 406df9ba87d55..08119c5334058 100644 --- a/packages/playwright/src/reporters/json.ts +++ b/packages/playwright/src/reporters/json.ts @@ -20,10 +20,11 @@ import path from 'path'; import { MultiMap } from '@isomorphic/multimap'; import { toPosixPath } from '@utils/fileUtils'; -import { formatError, nonTerminalScreen, prepareErrorStack, resolveOutputFile, CommonReporterOptions } from './base'; +import { createTestFilter, filterSuiteEntries, formatError, nonTerminalScreen, prepareErrorStack, resolveOutputFile, visitTests } from './base'; import { config } from '../common'; import type { ReporterV2 } from './reporterV2'; +import type { CommonReporterOptions, TestFilter } from './base'; import type { JsonReporterOptions } from '../../types/test'; import type { FullConfig, FullResult, JSONReport, JSONReportError, JSONReportSpec, JSONReportSuite, JSONReportTest, JSONReportTestResult, JSONReportTestStep, Location, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter'; @@ -32,9 +33,11 @@ class JSONReporter implements ReporterV2 { suite!: Suite; private _errors: TestError[] = []; private _resolvedOutputFile: string | undefined; + private _testFilter: TestFilter | undefined; constructor(options: JsonReporterOptions & CommonReporterOptions) { this._resolvedOutputFile = resolveOutputFile('JSON', options)?.outputFile; + this._testFilter = createTestFilter(options); } version(): 'v2' { @@ -92,7 +95,7 @@ class JSONReporter implements ReporterV2 { flaky: 0, }, }; - for (const test of this.suite.allTests()) + for (const test of visitTests(this.suite, this._testFilter)) ++report.stats[test.outcome()]; return report; } @@ -162,13 +165,15 @@ class JSONReporter implements ReporterV2 { } private _serializeSuite(projectId: string, projectName: string, suite: Suite): null | JSONReportSuite { - if (!suite.allTests().length) + const entries = filterSuiteEntries(suite, this._testFilter); + const suites = entries.filter(entry => entry.type !== 'test').map(suite => this._serializeSuite(projectId, projectName, suite)).filter(s => s) as JSONReportSuite[]; + const tests = entries.filter(entry => entry.type === 'test'); + if (!tests.length && !suites.length) return null; - const suites = suite.suites.map(suite => this._serializeSuite(projectId, projectName, suite)).filter(s => s) as JSONReportSuite[]; return { title: suite.title, ...this._relativeLocation(suite.location), - specs: suite.tests.map(test => this._serializeTestSpec(projectId, projectName, test)), + specs: tests.map(test => this._serializeTestSpec(projectId, projectName, test)), suites: suites.length ? suites : undefined, }; } diff --git a/packages/playwright/src/reporters/junit.ts b/packages/playwright/src/reporters/junit.ts index 82df28228344b..de98989a47ec1 100644 --- a/packages/playwright/src/reporters/junit.ts +++ b/packages/playwright/src/reporters/junit.ts @@ -19,10 +19,11 @@ import path from 'path'; import { getAsBooleanFromENV } from '@utils/env'; -import { CommonReporterOptions, formatFailure, nonTerminalScreen, resolveOutputFile } from './base'; +import { createTestFilter, filterSuites, formatFailure, nonTerminalScreen, resolveOutputFile, visitTests } from './base'; import { stripAnsiEscapes } from '../util'; import type { ReporterV2 } from './reporterV2'; +import type { CommonReporterOptions, TestFilter } from './base'; import type { JUnitReporterOptions } from '../../types/test'; import type { FullConfig, FullResult, Suite, TestCase, TestResult } from '../../types/testReporter'; @@ -40,8 +41,10 @@ class JUnitReporter implements ReporterV2 { private includeProjectInTestName = false; private includeRetries = false; private omitTags = false; + private testFilter: TestFilter | undefined; constructor(options: JUnitReporterOptions & CommonReporterOptions) { + this.testFilter = createTestFilter(options); this.stripANSIControlSequences = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_STRIP_ANSI', !!options.stripANSIControlSequences); this.includeProjectInTestName = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_INCLUDE_PROJECT_IN_TEST_NAME', !!options.includeProjectInTestName); this.includeRetries = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_INCLUDE_RETRIES', !!options.includeRetries); @@ -70,7 +73,7 @@ class JUnitReporter implements ReporterV2 { async onEnd(result: FullResult) { const children: XMLEntry[] = []; for (const projectSuite of this.suite.suites) { - for (const fileSuite of projectSuite.suites) + for (const fileSuite of filterSuites(projectSuite.suites, this.testFilter)) children.push(await this._buildTestSuite(projectSuite.title, fileSuite)); } const tokens: string[] = []; @@ -110,7 +113,7 @@ class JUnitReporter implements ReporterV2 { const children: XMLEntry[] = []; const testCaseNamePrefix = projectName && this.includeProjectInTestName ? `[${projectName}] ` : ''; - for (const test of suite.allTests()){ + for (const test of visitTests(suite, this.testFilter)) { ++tests; if (test.outcome() === 'skipped') ++skipped; diff --git a/packages/playwright/src/reporters/line.ts b/packages/playwright/src/reporters/line.ts index c703d3cc01e18..dac52ff85dfb9 100644 --- a/packages/playwright/src/reporters/line.ts +++ b/packages/playwright/src/reporters/line.ts @@ -29,7 +29,11 @@ class LineReporter extends TerminalReporter { private _didBegin = false; constructor(options?: LineReporterOptions & CommonReporterOptions & TerminalReporterOptions) { - super({ ...options, omitTags: getAsBooleanFromENV('PLAYWRIGHT_LINE_OMIT_TAGS', options?.omitTags) }); + super({ + ...options, + onlyFailures: getAsBooleanFromENV('PLAYWRIGHT_LINE_ONLY_FAILURES', options?.onlyFailures), + omitTags: getAsBooleanFromENV('PLAYWRIGHT_LINE_OMIT_TAGS', options?.omitTags), + }); } override onBegin(suite: Suite) { diff --git a/packages/playwright/src/reporters/list.ts b/packages/playwright/src/reporters/list.ts index a6a31bbf50274..e939207eb0c25 100644 --- a/packages/playwright/src/reporters/list.ts +++ b/packages/playwright/src/reporters/list.ts @@ -45,7 +45,12 @@ class ListReporter extends TerminalReporter { constructor(options?: ListReporterOptions & CommonReporterOptions & TerminalReporterOptions) { const printFailuresInline = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_FAILURES_INLINE', options?.printFailuresInline); - super({ ...options, omitTags: getAsBooleanFromENV('PLAYWRIGHT_LIST_OMIT_TAGS', options?.omitTags), lastResult: printFailuresInline }); + super({ + ...options, + onlyFailures: getAsBooleanFromENV('PLAYWRIGHT_LIST_ONLY_FAILURES', options?.onlyFailures), + omitTags: getAsBooleanFromENV('PLAYWRIGHT_LIST_OMIT_TAGS', options?.omitTags), + lastResult: printFailuresInline, + }); this._printSteps = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_STEPS', options?.printSteps); this._printFailuresInline = printFailuresInline; this._printWorkerIndex = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_WORKER_INDEX', options?.printWorkerIndex); diff --git a/packages/playwright/src/reporters/onlyFailures.ts b/packages/playwright/src/reporters/onlyFailures.ts new file mode 100644 index 0000000000000..006249ca060dd --- /dev/null +++ b/packages/playwright/src/reporters/onlyFailures.ts @@ -0,0 +1,104 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { markErrorsAsReported, TerminalReporter } from './base'; + +import type { FullResult, TestCase, TestError, TestResult } from '../../types/testReporter'; +import type { TerminalReporterOptions } from './base'; + +class OnlyFailuresTestReporter extends TerminalReporter { + private _failureIndex = new Map(); + private _needNewLine = false; + + constructor(options: TerminalReporterOptions = {}) { + super({ ...options, lastResult: true }); + } + + override onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) { + super.onStdOut(chunk, test, result); + this._dumpToStdio(chunk, this.screen.stdout); + } + + override onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult) { + super.onStdErr(chunk, test, result); + this._dumpToStdio(chunk, this.screen.stderr); + } + + private _dumpToStdio(chunk: string | Buffer, stream: NodeJS.WriteStream) { + if (this.config.quiet) + return; + stream.write(chunk); + if (chunk.length) + this._needNewLine = !chunk.toString().endsWith('\n'); + } + + override onTestEnd(test: TestCase, result: TestResult) { + super.onTestEnd(test, result); + if (result.status !== 'skipped' && result.status !== test.expectedStatus) + this._printFailure(test); + } + + private _printFailure(test: TestCase) { + let index = this._failureIndex.get(test); + if (index === undefined) { + index = this._failureIndex.size + 1; + this._failureIndex.set(test, index); + } + const message = this.formatFailure(test, index); + if (message.trim()) { + this._maybeWriteNewLine(); + this.writeLine(message); + } + } + + override onError(error: TestError) { + super.onError(error); + this._maybeWriteNewLine(); + this.writeLine(this.formatError(error).message); + } + + async onTestPaused(test: TestCase, result: TestResult) { + // Without TTY, the user cannot interrupt the pause. + if (!process.stdin.isTTY && !process.env.PW_TEST_DEBUG_REPORTERS) + return; + + this._maybeWriteNewLine(); + if (test.outcome() === 'unexpected') { + this._printFailure(test); + markErrorsAsReported(result); + this.writeLine(this.screen.colors.yellow(' Paused on error. Press Ctrl+C to end.')); + } else { + this.writeLine(this.screen.colors.yellow(this.formatTestHeader(test, { indent: ' ' }))); + this.writeLine(this.screen.colors.yellow(' Paused at test end. Press Ctrl+C to end.')); + } + await new Promise(() => {}); + } + + override async onEnd(result: FullResult) { + await super.onEnd(result); + this._maybeWriteNewLine(); + this.epilogue(false); + } + + private _maybeWriteNewLine() { + if (this._needNewLine) { + this.writeLine(); + this._needNewLine = false; + } + } +} + +export default OnlyFailuresTestReporter; diff --git a/packages/playwright/src/reporters/perfetto.ts b/packages/playwright/src/reporters/perfetto.ts index 2f0f59e27ed6f..07e42177cf535 100644 --- a/packages/playwright/src/reporters/perfetto.ts +++ b/packages/playwright/src/reporters/perfetto.ts @@ -21,10 +21,11 @@ import zlib from 'zlib'; import { toPosixPath } from '@utils/fileUtils'; import { getPlaywrightVersion } from 'playwright-core/lib/coreBundle'; -import { formatError, nonTerminalScreen, resolveOutputFile, CommonReporterOptions } from './base'; +import { createTestFilter, formatError, nonTerminalScreen, resolveOutputFile, visitTests } from './base'; import { stripAnsiEscapes } from '../util'; import type { ReporterV2 } from './reporterV2'; +import type { CommonReporterOptions, TestFilter } from './base'; import type { Writable } from 'stream'; import type { PerfettoReporterOptions } from '../../types/test'; import type { FullConfig, FullResult, Location, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter'; @@ -64,8 +65,10 @@ class PerfettoReporter implements ReporterV2 { private _events: TraceEvent[] = []; private _laneEndTime = new Map(); private _globalErrors: { error: TestError, timestamp: number }[] = []; + private _testFilter: TestFilter | undefined; constructor(options: PerfettoReporterOptions & CommonReporterOptions) { + this._testFilter = createTestFilter(options); this._resolvedOutputFile = resolveOutputFile('PERFETTO', { ...options, default: { @@ -97,7 +100,7 @@ class PerfettoReporter implements ReporterV2 { async onEnd(result: FullResult) { const entries: { test: TestCase, result: TestResult }[] = []; - for (const test of this._suite?.allTests() ?? []) { + for (const test of this._suite ? visitTests(this._suite, this._testFilter) : []) { for (const testResult of test.results) entries.push({ test, result: testResult }); } diff --git a/packages/playwright/src/runner/reporters.ts b/packages/playwright/src/runner/reporters.ts index 34cff70b12c59..c2c9fc2342b0f 100644 --- a/packages/playwright/src/runner/reporters.ts +++ b/packages/playwright/src/runner/reporters.ts @@ -29,6 +29,7 @@ import JUnitReporter from '../reporters/junit'; import LineReporter from '../reporters/line'; import ListReporter from '../reporters/list'; import ListModeReporter from '../reporters/listModeReporter'; +import OnlyFailuresTestReporter from '../reporters/onlyFailures'; import PerfettoReporter from '../reporters/perfetto'; import { wrapReporterAsV2 } from '../reporters/reporterV2'; @@ -56,11 +57,19 @@ export async function createReporters(config: FullConfigInternal, mode: 'list' | const reporters: ReporterV2[] = []; descriptions ??= config.config.reporter; const reportOptions = reporterCommandOptions(config, mode, runOptions); + + function createBuiltInReporter(name: commonConfig.BuiltInReporter, options: CommonReporterOptions): ReporterV2 { + const reporter = new defaultReporters[name](options); + if ((reporter instanceof DotReporter || reporter instanceof LineReporter || reporter instanceof ListReporter) && reporter.options.onlyFailures) + return new OnlyFailuresTestReporter(reporter.options); + return reporter; + } + for (const r of descriptions) { const [name, arg] = r; - const options = { ...reportOptions, ...arg }; + const options = resolveReporterOptions(reportOptions, { ...reportOptions, ...arg }); if (name in defaultReporters) { - reporters.push(new defaultReporters[name as keyof typeof defaultReporters](options)); + reporters.push(createBuiltInReporter(name as commonConfig.BuiltInReporter, options)); } else { const reporterConstructor = await loadReporter(config, name); reporters.push(wrapReporterAsV2(new reporterConstructor(options))); @@ -68,11 +77,12 @@ export async function createReporters(config: FullConfigInternal, mode: 'list' | } if (process.env.PW_TEST_REPORTER) { const name = process.env.PW_TEST_REPORTER; + const options = resolveReporterOptions(reportOptions); if (name in defaultReporters) { - reporters.push(new defaultReporters[name as keyof typeof defaultReporters](reportOptions)); + reporters.push(createBuiltInReporter(name as commonConfig.BuiltInReporter, options)); } else { const reporterConstructor = await loadReporter(config, name); - reporters.push(wrapReporterAsV2(new reporterConstructor(reportOptions))); + reporters.push(wrapReporterAsV2(new reporterConstructor(options))); } } @@ -80,10 +90,12 @@ export async function createReporters(config: FullConfigInternal, mode: 'list' | if (reporters.length && !someReporterPrintsToStdio) { // Add a line/dot/list-mode reporter for convenience. // Important to put it first, just in case some other reporter stalls onEnd. - if (mode === 'list') + if (mode === 'list') { reporters.unshift(new ListModeReporter()); - else if (mode !== 'merge') - reporters.unshift(!process.env.CI ? new LineReporter() : new DotReporter()); + } else if (mode !== 'merge') { + const name = process.env.CI ? 'dot' : 'line'; + reporters.unshift(createBuiltInReporter(name, resolveReporterOptions(reportOptions))); + } } return reporters; } @@ -109,9 +121,19 @@ function reporterCommandOptions(config: FullConfigInternal, mode: 'list' | 'test configDir: config.configDir, _mode: mode, _commandHash: computeCommandHash(config, runOptions), + onlyFailures: mode !== 'list' && config.configCLIOverrides.reporterOnlyFailures, }; } +function resolveReporterOptions(commonOptions: CommonReporterOptions, options: CommonReporterOptions = commonOptions): CommonReporterOptions { + let onlyFailures = options.onlyFailures; + if (commonOptions._mode === 'list') + onlyFailures = false; + else if (commonOptions.onlyFailures) + onlyFailures = true; + return { ...options, onlyFailures }; +} + function computeCommandHash(config: FullConfigInternal, runOptions?: TestRunOptions) { const parts = []; // Include project names for readability. diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 7ef43b4815d58..09f07bea7f031 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -16,17 +16,18 @@ */ import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions, Page, LaunchOptions, ViewportSize, Geolocation, HTTPCredentials, Locator, APIResponse, PageScreenshotOptions } from 'playwright-core'; +import type { ReporterOptions } from './testReporter'; export * from 'playwright-core'; export type BlobReporterOptions = { outputDir?: string, fileName?: string }; -export type DotReporterOptions = { omitTags?: boolean }; -export type LineReporterOptions = { omitTags?: boolean }; -export type ListReporterOptions = { printSteps?: boolean, printFailuresInline?: boolean, omitTags?: boolean, printWorkerIndex?: boolean }; -export type GitHubReporterOptions = { omitTags?: boolean }; -export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean, omitTags?: boolean }; -export type JsonReporterOptions = { outputFile?: string }; -export type PerfettoReporterOptions = { outputFile?: string }; -export type HtmlReporterOptions = { +export type DotReporterOptions = ReporterOptions & { omitTags?: boolean }; +export type LineReporterOptions = ReporterOptions & { omitTags?: boolean }; +export type ListReporterOptions = ReporterOptions & { printSteps?: boolean, printFailuresInline?: boolean, omitTags?: boolean, printWorkerIndex?: boolean }; +export type GitHubReporterOptions = ReporterOptions & { omitTags?: boolean }; +export type JUnitReporterOptions = ReporterOptions & { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean, omitTags?: boolean }; +export type JsonReporterOptions = ReporterOptions & { outputFile?: string }; +export type PerfettoReporterOptions = ReporterOptions & { outputFile?: string }; +export type HtmlReporterOptions = ReporterOptions & { outputFolder?: string; open?: 'always' | 'never' | 'on-failure'; host?: string; diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index a7962739df19e..f45bf1fedecd4 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -52,11 +52,11 @@ export interface FullResult { * ```js * // my-awesome-reporter.ts * import type { - * Reporter, FullConfig, Suite, TestCase, TestResult, FullResult + * Reporter, ReporterOptions, FullConfig, Suite, TestCase, TestResult, FullResult * } from '@playwright/test/reporter'; * * class MyReporter implements Reporter { - * constructor(options: { customOption?: string } = {}) { + * constructor(options: ReporterOptions & { customOption?: string } = {}) { * console.log(`my-awesome-reporter setup with customOption set to ${options.customOption}`); * } * @@ -92,6 +92,18 @@ export interface FullResult { * }); * ``` * + * **Reporter options** + * + * The reporter constructor receives its configured options together with common options described by the + * `ReporterOptions` type: + * + * | Option | Description | + * |---|---| + * | `onlyFailures` | Whether to limit this reporter's output to failed, flaky, and interrupted tests. Can be configured for each reporter. `--reporter-only-failures` supplies `true`, overriding configuration. Each reporter constructor has the final say, including environment overrides. | + * + * This option does not filter or delay reporter callbacks. Reporters still receive the complete suite, every test + * result, and all timing information. + * * Here is a typical order of reporter calls: * - [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin) is called * once with a root suite that contains all other suites and tests. Learn more about @@ -264,6 +276,15 @@ export interface Reporter { printsToStdio?(): boolean; } +export type ReporterOptions = { + /** + * Whether to limit output to failed, flaky, and interrupted tests. + * Terminal reporters print failure details as soon as each attempt finishes. + * Reporter callbacks still receive all tests and results. + */ + onlyFailures?: boolean; +}; + export interface JSONReport { config: Omit & { projects: { diff --git a/tests/playwright-test/reporter-only-failures.spec.ts b/tests/playwright-test/reporter-only-failures.spec.ts new file mode 100644 index 0000000000000..b97a9efa5c8dd --- /dev/null +++ b/tests/playwright-test/reporter-only-failures.spec.ts @@ -0,0 +1,559 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; +import path from 'path'; +import xml2js from 'xml2js'; +import { test, expect, stripAnsi } from './playwright-test-fixtures'; +import { extractZip } from '../../packages/utils/third_party/extractZip'; + +import type { HTMLReport } from '../../packages/html-reporter/src/types'; +import type { JSONReport, JSONReportSuite } from '@playwright/test/reporter'; + +const passingTest = ` + import { test } from '@playwright/test'; + test('passes', async () => { + await test.step('passing step', async () => {}); + }); +`; + +const testFiles = { + 'passing.test.ts': ` + import fs from 'fs'; + import { test } from '@playwright/test'; + test('passes', async ({}, testInfo) => { + fs.writeFileSync('passing-ran.txt', 'ran'); + const attachmentPath = testInfo.outputPath('passing.txt'); + fs.writeFileSync(attachmentPath, 'passing attachment'); + await testInfo.attach('passing', { path: attachmentPath }); + }); + `, + 'failures.test.ts': ` + import fs from 'fs'; + import { test, expect } from '@playwright/test'; + test.describe('failures', () => { + test('fails', async ({}, testInfo) => { + const attachmentPath = testInfo.outputPath('failure.txt'); + fs.writeFileSync(attachmentPath, 'failure attachment'); + await testInfo.attach('failure', { path: attachmentPath }); + expect(1).toBe(2); + }); + test('flaky', ({}, testInfo) => { + if (!testInfo.retry) + throw new Error('flaky failure'); + }); + test('unexpectedly passes', () => { + test.fail(); + }); + }); + test.describe('ignored', () => { + test.skip('skipped', () => {}); + test('expected failure', () => { + test.fail(); + throw new Error('expected failure'); + }); + }); + `, +}; + +for (const reporter of ['list', 'line', 'dot']) { + test(`--reporter-only-failures overrides ${reporter} configuration`, async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + export default { reporter: [['${reporter}', { onlyFailures: false }]] }; + `, + 'a.test.ts': passingTest, + }, { 'reporter-only-failures': true }, { + [`PLAYWRIGHT_${reporter.toUpperCase()}_ONLY_FAILURES`]: undefined, + PLAYWRIGHT_FORCE_TTY: '1', + }); + expect(result.exitCode).toBe(0); + expect(result.output).toMatch(/^\n? 1 passed \([^)]+\)\n$/); + expect(result.rawOutput).not.toContain('\u001B[1A'); + expect(result.rawOutput).not.toContain('\u001B[2K'); + }); + + for (const onlyFailures of [false, true]) { + test(`${reporter} onlyFailures environment overrides ${onlyFailures ? 'configuration' : 'CLI'}`, async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + export default { reporter: [['${reporter}', { onlyFailures: ${!onlyFailures} }]] }; + `, + 'a.test.ts': passingTest, + }, onlyFailures ? {} : { 'reporter-only-failures': true }, { [`PLAYWRIGHT_${reporter.toUpperCase()}_ONLY_FAILURES`]: String(onlyFailures) }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.output.includes('Running 1 test using')).toBe(!onlyFailures); + expect(result.output.includes(reporter === 'dot' ? '·' : '› passes')).toBe(!onlyFailures); + }); + } + + for (const omitTags of [false, true]) { + test(`${reporter} onlyFailures respects omitTags environment override ${omitTags}`, async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + export default { reporter: [['${reporter}', { onlyFailures: true, omitTags: ${!omitTags} }]] }; + `, + 'a.test.ts': ` + import { test } from '@playwright/test'; + test('fails', { tag: '@tag' }, () => { + throw new Error('failure'); + }); + `, + }, {}, { [`PLAYWRIGHT_${reporter.toUpperCase()}_OMIT_TAGS`]: String(omitTags) }); + expect(result.exitCode).toBe(1); + const titleLines = result.output.split('\n').filter(line => line.includes('› fails')); + expect(titleLines.length).toBeGreaterThan(0); + for (const line of titleLines) + expect(line.includes('@tag')).toBe(!omitTags); + }); + } +} + +for (const fullyParallel of [false, true]) { + test(`onlyFailures preserves timing with fullyParallel=${fullyParallel}`, async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + export default { + reporter: [['list', { onlyFailures: true }]], + fullyParallel: ${fullyParallel}, + reportSlowTests: { max: 1, threshold: ${fullyParallel ? 1 : 300} }, + }; + `, + 'a.test.ts': ` + import { test } from '@playwright/test'; + test('passes', async () => { + await new Promise(resolve => setTimeout(resolve, 200)); + }); + test('fails', async () => { + await new Promise(resolve => setTimeout(resolve, 200)); + throw new Error('failure'); + }); + `, + }); + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(1); + expect(result.failed).toBe(1); + expect(result.output.includes('Slow test file: a.test.ts')).toBe(!fullyParallel); + }); +} + +for (const tty of ['0', '1']) { + test(`onlyFailures prints only the summary for successful tests with TTY=${tty}`, async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + export default { reporter: [['list', { onlyFailures: true }]] }; + `, + 'a.test.ts': passingTest + ` + test.skip('skipped', () => {}); + test('expected failure', () => { + test.fail(); + throw new Error('expected failure'); + }); + `, + }, {}, { PLAYWRIGHT_FORCE_TTY: tty }); + expect(result.exitCode).toBe(0); + expect(result.output).toMatch(/^ 1 skipped\n 2 passed \([^)]+\)\n$/); + expect(result.rawOutput).not.toContain('\u001B[1A'); + }); +} + +for (const useIntermediateMergeReport of [false, true]) { + test.describe(useIntermediateMergeReport ? 'merged stdio' : 'live stdio', () => { + test.use({ useIntermediateMergeReport }); + + for (const quiet of [false, true]) { + test(`onlyFailures preserves quiet=${quiet}`, async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + export default { reporter: [['list', { onlyFailures: true }]], quiet: ${quiet} }; + `, + 'a.test.ts': ` + import { test } from '@playwright/test'; + test('passes', () => { + process.stdout.write('test stdout'); + process.stderr.write('test stderr'); + }); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.output.includes('test stdout')).toBe(!quiet); + expect(result.output.includes('test stderr')).toBe(!quiet); + }); + } + }); +} + +test('onlyFailures preserves errors outside tests', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + export default { reporter: [['list', { onlyFailures: true }]], globalTeardown: './global-teardown.ts' }; + `, + 'global-teardown.ts': ` + export default () => { + throw new Error('global teardown failed'); + }; + `, + 'a.test.ts': passingTest, + }); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(0); + expect(result.passed).toBe(1); + expect(result.output).toContain('Error: global teardown failed'); + expect(result.output).toContain('1 error was not a part of any test'); + expect(result.output).not.toContain('Running 1 test using'); +}); + +test('onlyFailures prints timeout details for each attempt', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + export default { reporter: [['list', { onlyFailures: true }]], retries: 1 }; + `, + 'a.test.ts': ` + import { test } from '@playwright/test'; + test('times out', async () => { + test.setTimeout(500); + await new Promise(() => {}); + }); + `, + }); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + expect(result.output.match(/^ Test timeout of 500ms exceeded\./gm)).toHaveLength(2); + expect(result.output).toContain('Retry #1'); +}); + +for (const ci of [undefined, 'true']) { + test(`--reporter-only-failures works with the default reporter on CI=${ci}`, async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': passingTest, + }, { 'reporter-only-failures': true }, { CI: ci }); + expect(result.exitCode).toBe(0); + expect(result.output).toMatch(/^\n? 1 passed \([^)]+\)\n$/); + }); +} + +test('--reporter-only-failures applies to added and fallback reporters', async ({ runInlineTest }) => { + const files = { + 'playwright.config.ts': ` + export default { reporter: [['json', { outputFile: 'report.json' }]] }; + `, + 'a.test.ts': passingTest, + }; + const env = { PLAYWRIGHT_LINE_ONLY_FAILURES: undefined, PLAYWRIGHT_DOT_ONLY_FAILURES: undefined, PLAYWRIGHT_LIST_ONLY_FAILURES: undefined }; + const fallback = await runInlineTest(files, { 'reporter-only-failures': true }, env); + expect(fallback.exitCode).toBe(0); + expect(fallback.output).toMatch(/^ 1 passed \([^)]+\)\n$/); + const added = await runInlineTest(files, { 'reporter-only-failures': true, 'add-reporter': 'list' }, env); + expect(added.exitCode).toBe(0); + expect(added.output).toMatch(/^ 1 passed \([^)]+\)\n$/); +}); + +test('onlyFailures can be configured independently for each reporter', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...testFiles, + 'playwright.config.ts': ` + export default { + retries: 1, + reporter: [ + ['list', { onlyFailures: true }], + ['json', { onlyFailures: true, outputFile: 'failures.json' }], + ['json', { outputFile: 'complete.json' }], + ], + }; + `, + }, { workers: 1 }); + expect(result.exitCode).toBe(1); + expect(result.output).not.toContain('passing.test.ts'); + expect(result.passed).toBe(2); + const failures: JSONReport = JSON.parse(fs.readFileSync(testInfo.outputPath('failures.json'), 'utf8')); + const complete: JSONReport = JSON.parse(fs.readFileSync(testInfo.outputPath('complete.json'), 'utf8')); + expect(jsonTestTitles(failures.suites).sort()).toEqual(['fails', 'flaky', 'unexpectedly passes']); + expect(jsonTestTitles(complete.suites).sort()).toEqual(['expected failure', 'fails', 'flaky', 'passes', 'skipped', 'unexpectedly passes']); +}); + +for (const merge of [false, true]) { + test(`--reporter-only-failures filters final reports and preserves blobs when ${merge ? 'merging' : 'running'}`, async ({ runInlineTest, mergeReports }, testInfo) => { + const result = await runInlineTest({ + ...testFiles, + 'playwright.config.ts': ` + export default { + retries: 1, + reporter: [ + ['list'], + ['github'], + ['null'], + ['json', { outputFile: 'report.json' }], + ['junit', { outputFile: 'report.xml' }], + ['html', { open: 'never' }], + ['blob', { outputDir: 'complete-blob-report' }], + ['perfetto', { outputFile: 'perfetto.json' }], + ['./integrity-reporter.ts'], + ], + }; + `, + 'integrity-reporter.ts': ` + export default class { + onBegin(config, suite) { + this.suite = suite; + console.log('%%begin: ' + suite.allTests().length); + } + onEnd() { + console.log('%%end: ' + this.suite.allTests().length); + } + } + `, + }, merge ? { reporter: 'blob', workers: 1 } : { 'reporter-only-failures': true, 'workers': 1 }); + expect(result.exitCode).toBe(1); + let output = result.output; + if (merge) { + const merged = await mergeReports(testInfo.outputPath('blob-report'), {}, { + cwd: testInfo.outputPath(), + additionalArgs: ['--config', testInfo.outputPath('playwright.config.ts'), '--reporter-only-failures'], + }); + expect(merged.exitCode).toBe(0); + output = stripAnsi(merged.output); + } + expect(output).toContain('2 passed'); + expect(output).toContain('2 failed'); + expect(output).toContain('1 flaky'); + expect(output.match(/^ Error: expect\(received\)/gm)).toHaveLength(2); + expect(output.match(/^ Error: flaky failure/gm)).toHaveLength(1); + expect(output).toContain('Expected: 2'); + expect(output).toContain('Received: 1'); + expect(output).toMatch(/> +\d+ \| +expect\(1\)\.toBe\(2\);/); + expect(output).toMatch(/at .*failures\.test\.ts:\d+:\d+/); + expect(output).not.toContain('Running 6 tests'); + expect(output).not.toContain('passing.test.ts'); + expect(output).toContain('%%begin: 6'); + expect(output).toContain('%%end: 6'); + expect(fs.existsSync(testInfo.outputPath('passing-ran.txt'))).toBe(true); + + const titles = ['fails', 'flaky', 'unexpectedly passes']; + const json: JSONReport = JSON.parse(fs.readFileSync(testInfo.outputPath('report.json'), 'utf8')); + expect(jsonTestTitles(json.suites).sort()).toEqual(titles); + expect(json.stats).toMatchObject({ expected: 0, skipped: 0, unexpected: 2, flaky: 1 }); + + const xml = await xml2js.parseStringPromise(fs.readFileSync(testInfo.outputPath('report.xml'), 'utf8')); + expect(xml.testsuites.$.tests).toBe('3'); + expect(xml.testsuites.testsuite).toHaveLength(1); + expect(xml.testsuites.testsuite[0].testcase.map((entry: { $: { name: string } }) => entry.$.name).sort()).toEqual(titles.map(title => `failures › ${title}`)); + + const html = fs.readFileSync(testInfo.outputPath('playwright-report', 'index.html'), 'utf8'); + const data = html.match(/