From 487276c7c875232ed9636089cc45dfcb4a0f1d38 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Thu, 1 Oct 2026 13:03:44 -0700 Subject: [PATCH] =?UTF-8?q?Revert=20"feat(test-runner):=20read=20last=20fa?= =?UTF-8?q?iled=20tests=20from=20--last-failed=3D=20(=E2=80=A6"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 3456aa7d080dd9d60212998958a26e66ade918c2. --- docs/src/running-tests-js.md | 12 +-- docs/src/test-cli-js.md | 4 +- packages/playwright/src/cli/testActions.ts | 3 +- packages/playwright/src/program.ts | 4 +- packages/playwright/src/runner/lastRun.ts | 53 +++++------- packages/playwright/src/runner/tasks.ts | 3 +- packages/playwright/src/runner/testRunner.ts | 2 +- tests/playwright-test/runner.spec.ts | 90 +------------------- 8 files changed, 32 insertions(+), 139 deletions(-) diff --git a/docs/src/running-tests-js.md b/docs/src/running-tests-js.md index 1c563a568fee0..0925ef7db8001 100644 --- a/docs/src/running-tests-js.md +++ b/docs/src/running-tests-js.md @@ -91,20 +91,12 @@ To run only the tests that failed in the last test run, first run your tests and npx playwright test --last-failed ``` -Playwright stores the list of failed tests from the previous run in `/.last-run.json` (see [`property: TestConfig.outputDir`](./test-configuration.md)). To write it to a different file, pass `--last-run-output-file=` or set `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE`. +Playwright stores the list of failed tests from the previous run in `/.last-run.json` (see [`property: TestConfig.outputDir`](./test-configuration.md)). To use a different file path, pass `--last-failed-file=` or set `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE`. ```bash -npx playwright test --last-run-output-file=.cache/last-run-shard-1.json +npx playwright test --last-failed --last-failed-file=.cache/last-run-shard-1.json ``` -To re-run the failures recorded in a specific file, pass it to `--last-failed=`. The run fails if the file does not exist. The file is only read, the results of the re-run are written to the default location or to the `--last-run-output-file` path. Without a path, `--last-failed` always reads `/.last-run.json`. - -```bash -npx playwright test --last-failed=.cache/last-run-shard-1.json -``` - -Relative paths are resolved against the current working directory. - ### Run tests in VS Code Tests can be run right from VS Code using the [VS Code extension](https://marketplace.visualstudio.com/items?itemName=ms-playwright.playwright). Once installed you can simply click the green triangle next to the test you want to run or run all tests from the testing sidebar. Check out our [Getting Started with VS Code](./getting-started-vscode.md) guide for more details. diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.md index 67c8dbb5d5093..fadedf6b741dd 100644 --- a/docs/src/test-cli-js.md +++ b/docs/src/test-cli-js.md @@ -91,8 +91,8 @@ npx playwright test --ui | `--headed` | Run tests in headed browsers (default: headless). | | `--ignore-snapshots` | Ignore screenshot and snapshot expectations. | | `-j ` or `--workers ` | Number of concurrent workers or percentage of logical CPU cores, use 1 to run in a single worker (default: 50%). | -| `--last-failed [file]` | Only re-run the failures. Optionally takes a path to the last run file to read the failures from, use `--last-failed=` form (default: `/.last-run.json`). | -| `--last-run-output-file ` | Path to write the last run file to (default: `/.last-run.json`). Same as `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE` environment variable. | +| `--last-failed` | Only re-run the failures. | +| `--last-failed-file ` | Override the default last-run JSON path for `--last-failed` (default: `/.last-run.json`). Same as `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE` environment variable. | | `--list` | Collect all the tests and report them, but do not run. | | `--max-failures ` or `-x` | Stop after the first `N` failures. Passing `-x` stops after the first failure. | | `--no-deps` | Do not run project dependencies. | diff --git a/packages/playwright/src/cli/testActions.ts b/packages/playwright/src/cli/testActions.ts index f3aed24751d69..c03abcff8064f 100644 --- a/packages/playwright/src/cli/testActions.ts +++ b/packages/playwright/src/cli/testActions.ts @@ -40,8 +40,7 @@ export async function runTests(args: string[], opts: { [key: string]: any }) { projectFilter: opts.project || undefined, passWithNoTests: !!opts.passWithNoTests, lastFailed: !!opts.lastFailed, - lastRunInputFile: typeof opts.lastFailed === 'string' ? opts.lastFailed : undefined, - lastRunOutputFile: opts.lastRunOutputFile, + lastFailedFile: opts.lastFailedFile, testList: opts.testList ? path.resolve(process.cwd(), opts.testList) : undefined, testListInvert: opts.testListInvert ? path.resolve(process.cwd(), opts.testListInvert) : undefined, shardWeights: resolveShardWeightsOption(), diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index 505c05b668e04..9042f965224aa 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -218,8 +218,8 @@ const testOptions: [string, { description: string, choices?: string[], preset?: ['-G, --grep-invert ', { description: `Only run tests that do not match this regular expression` }], ['--headed', { description: `Run tests in headed browsers (default: headless)` }], ['--ignore-snapshots', { description: `Ignore screenshot and snapshot expectations` }], - ['--last-failed [file]', { description: `Only re-run the failures. Optionally takes a path to the last run file to read the failures from, use --last-failed= form (default: /.last-run.json)` }], - ['--last-run-output-file ', { description: `Path to write the last run file to (default: /.last-run.json). Same as PLAYWRIGHT_LAST_RUN_OUTPUT_FILE environment variable.` }], + ['--last-failed', { description: `Only re-run the failures` }], + ['--last-failed-file ', { description: `Override the default path for the last-run JSON file used with --last-failed (default: /.last-run.json). Same as PLAYWRIGHT_LAST_RUN_OUTPUT_FILE environment variable.` }], ['--list', { description: `Collect all the tests and report them, but do not run` }], ['--max-failures ', { description: `Stop after the first N failures` }], ['--no-deps', { description: `Do not run project dependencies` }], diff --git a/packages/playwright/src/runner/lastRun.ts b/packages/playwright/src/runner/lastRun.ts index 97bf5c8e4b842..8312da4471ea6 100644 --- a/packages/playwright/src/runner/lastRun.ts +++ b/packages/playwright/src/runner/lastRun.ts @@ -17,8 +17,6 @@ import fs from 'fs'; import path from 'path'; -import { errorWithFile } from '../util'; - import type { FullResult, Suite, TestCase } from '../../types/testReporter'; import type { config as commonConfig } from '../common'; import type { ReporterV2 } from '../reporters/reporterV2'; @@ -37,31 +35,30 @@ function didNotRun(test: TestCase): boolean { } export class LastRunReporter implements ReporterV2 { - private _inputFile: string | undefined; - private _isDefaultInputFile: boolean; - private _outputFile: string | undefined; + private _lastRunFile: string | undefined; private _suite: Suite | undefined; private _listMode: boolean; - constructor(filteredProjects: commonConfig.FullProjectInternal[], options: { listMode?: boolean, inputFile?: string, outputFile?: string }) { - this._listMode = !!options.listMode; - const [project] = filteredProjects; - const defaultFile = project ? path.join(project.project.outputDir, '.last-run.json') : undefined; - const outputFile = options.outputFile ?? process.env.PLAYWRIGHT_LAST_RUN_OUTPUT_FILE; - this._outputFile = outputFile ? path.resolve(process.cwd(), outputFile) : defaultFile; - this._inputFile = options.inputFile ? path.resolve(process.cwd(), options.inputFile) : defaultFile; - this._isDefaultInputFile = !options.inputFile; + constructor(filteredProjects: commonConfig.FullProjectInternal[], listMode?: boolean, lastFailedFileOverride?: string) { + this._listMode = !!listMode; + const override = lastFailedFileOverride ?? process.env.PLAYWRIGHT_LAST_RUN_OUTPUT_FILE; + if (override) { + this._lastRunFile = path.resolve(process.cwd(), override); + } else { + const [project] = filteredProjects; + if (project) + this._lastRunFile = path.join(project.project.outputDir, '.last-run.json'); + } } async filterLastFailed(): Promise { - if (!this._inputFile) + if (!this._lastRunFile) return undefined; try { - return await readFailedTests(this._inputFile); - } catch (e) { - if (this._isDefaultInputFile) - return undefined; - throw e; + const lastRunInfo = JSON.parse(await fs.promises.readFile(this._lastRunFile, 'utf8')) as LastRunInfo; + return lastRunInfo.failedTests; + } catch { + return undefined; } } @@ -78,25 +75,13 @@ export class LastRunReporter implements ReporterV2 { } async onEnd(result: FullResult) { - if (!this._outputFile || this._listMode) + if (!this._lastRunFile || this._listMode) return; const lastRunInfo: LastRunInfo = { status: result.status, failedTests: this._suite?.allTests().filter(t => !t.ok() || didNotRun(t)).map(t => t.id) || [], }; - await fs.promises.mkdir(path.dirname(this._outputFile), { recursive: true }); - await fs.promises.writeFile(this._outputFile, JSON.stringify(lastRunInfo, undefined, 2)); - } -} - -async function readFailedTests(file: string): Promise { - let lastRunInfo: LastRunInfo; - try { - lastRunInfo = JSON.parse(await fs.promises.readFile(file, 'utf8')); - } catch (e) { - throw errorWithFile(file, 'Cannot read last run file: ' + e.message); + await fs.promises.mkdir(path.dirname(this._lastRunFile), { recursive: true }); + await fs.promises.writeFile(this._lastRunFile, JSON.stringify(lastRunInfo, undefined, 2)); } - if (!Array.isArray(lastRunInfo?.failedTests)) - throw errorWithFile(file, 'Cannot read last run file: "failedTests" list is missing'); - return lastRunInfo.failedTests; } diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts index c9426b0c736fa..1361860eab42d 100644 --- a/packages/playwright/src/runner/tasks.ts +++ b/packages/playwright/src/runner/tasks.ts @@ -63,8 +63,7 @@ export type TestRunOptions = { listMode?: boolean; passWithNoTests?: boolean; lastFailed?: boolean; - lastRunInputFile?: string; - lastRunOutputFile?: string; + lastFailedFile?: string; testList?: string; testListInvert?: string; lastFailedTestIds?: string[]; diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.ts index e472b9c4df954..df44df98a18df 100644 --- a/packages/playwright/src/runner/testRunner.ts +++ b/packages/playwright/src/runner/testRunner.ts @@ -429,7 +429,7 @@ export async function runAllTestsWithConfig(config: FullConfigInternal, options: const filteredProjects = filterProjects(config.projects, options.projectFilter); const reporters = await createReporters(config, options.listMode ? 'list' : 'test', undefined, options); - const lastRun = new LastRunReporter(filteredProjects, { listMode: options.listMode, inputFile: options.lastRunInputFile, outputFile: options.lastRunOutputFile }); + const lastRun = new LastRunReporter(filteredProjects, options.listMode, options.lastFailedFile); if (options.lastFailed) { const lastFailedTestIds = await lastRun.filterLastFailed(); if (lastFailedTestIds) diff --git a/tests/playwright-test/runner.spec.ts b/tests/playwright-test/runner.spec.ts index 5914e2094a50e..812f6cade29ed 100644 --- a/tests/playwright-test/runner.spec.ts +++ b/tests/playwright-test/runner.spec.ts @@ -977,7 +977,7 @@ test('should run last failed tests in a shard with PLAYWRIGHT_LAST_RUN_OUTPUT_FI expect(result1.output).toContain('b.spec.js:3:11 › pass-b'); expect(result1.output).toContain('b.spec.js:4:11 › fail-b'); - const result2 = await runInlineTest(workspace, { shard: '2/2' }, env, { additionalArgs: [`--last-failed=${customRel}`] }); + const result2 = await runInlineTest(workspace, { shard: '2/2' }, env, { additionalArgs: ['--last-failed'] }); expect(result2.exitCode).toBe(1); expect(result2.passed).toBe(0); expect(result2.failed).toBe(1); @@ -985,11 +985,11 @@ test('should run last failed tests in a shard with PLAYWRIGHT_LAST_RUN_OUTPUT_FI expect(result2.output).toContain('b.spec.js:4:11 › fail-b'); }); -test('should run last failed tests in a shard with --last-run-output-file', async ({ runInlineTest }, testInfo) => { +test('should run last failed tests in a shard with --last-failed-file', async ({ runInlineTest }, testInfo) => { const customRel = '.cache/shard-2-cli-last-run.json'; const customAbs = path.join(testInfo.outputPath(), customRel); const defaultLastRun = path.join(testInfo.outputPath(), 'test-results', '.last-run.json'); - const lastRunArgs = [`--last-run-output-file=${customRel}`]; + const lastRunArgs = ['--last-failed', `--last-failed-file=${customRel}`]; const workspace = { 'a.spec.js': ` import { test, expect } from '@playwright/test'; @@ -1015,92 +1015,10 @@ test('should run last failed tests in a shard with --last-run-output-file', asyn expect(result1.output).toContain('b.spec.js:3:11 › pass-b'); expect(result1.output).toContain('b.spec.js:4:11 › fail-b'); - const result2 = await runInlineTest(workspace, { shard: '2/2' }, {}, { additionalArgs: [`--last-failed=${customRel}`, ...lastRunArgs] }); + const result2 = await runInlineTest(workspace, { shard: '2/2' }, {}, { additionalArgs: lastRunArgs }); expect(result2.exitCode).toBe(1); expect(result2.passed).toBe(0); expect(result2.failed).toBe(1); expect(result2.output).not.toContain('b.spec.js:3:11 › pass-b'); expect(result2.output).toContain('b.spec.js:4:11 › fail-b'); }); - -test('should read --last-failed= without overwriting it', async ({ runInlineTest }, testInfo) => { - const inputFile = testInfo.outputPath('.cache', 'input.json'); - const outputFile = testInfo.outputPath('.cache', 'output.json'); - const defaultLastRun = testInfo.outputPath('test-results', '.last-run.json'); - const spec = (expected: number) => ` - import { test, expect } from '@playwright/test'; - test('pass-a', async () => {}); - test('fail-a', async () => { - expect(1).toBe(${expected}); - }); - `; - - const result1 = await runInlineTest({ 'a.spec.js': spec(2) }, {}, {}, { additionalArgs: ['--last-run-output-file=.cache/input.json'] }); - expect(result1.exitCode).toBe(1); - expect(result1.passed).toBe(1); - expect(result1.failed).toBe(1); - expect(fs.existsSync(defaultLastRun)).toBe(false); - const input = fs.readFileSync(inputFile, 'utf8'); - expect(JSON.parse(input).failedTests).toHaveLength(1); - - const result2 = await runInlineTest({ 'a.spec.js': spec(1) }, {}, {}, { additionalArgs: ['--last-failed=.cache/input.json'] }); - expect(result2.exitCode).toBe(0); - expect(result2.passed).toBe(1); - expect(result2.output).not.toContain('pass-a'); - expect(result2.output).toContain('fail-a'); - expect(fs.readFileSync(inputFile, 'utf8')).toBe(input); - expect(JSON.parse(fs.readFileSync(defaultLastRun, 'utf8'))).toEqual({ status: 'passed', failedTests: [] }); - - const result3 = await runInlineTest({ 'a.spec.js': spec(1) }, {}, {}, { additionalArgs: ['--last-failed=.cache/input.json', '--last-run-output-file=.cache/output.json'] }); - expect(result3.exitCode).toBe(0); - expect(result3.passed).toBe(1); - expect(result3.output).not.toContain('pass-a'); - expect(fs.readFileSync(inputFile, 'utf8')).toBe(input); - expect(JSON.parse(fs.readFileSync(outputFile, 'utf8'))).toEqual({ status: 'passed', failedTests: [] }); - expect(fs.existsSync(defaultLastRun)).toBe(false); -}); - -test('should fail when --last-failed= is missing or malformed', async ({ runInlineTest }) => { - const workspace = { - 'a.spec.js': ` - import { test, expect } from '@playwright/test'; - test('pass-a', async () => {}); - `, - 'malformed.json': `{ "status": "failed" }`, - }; - - const result1 = await runInlineTest(workspace, {}, {}, { additionalArgs: ['--last-failed=missing.json'] }); - expect(result1.exitCode).toBe(1); - expect(result1.passed).toBe(0); - expect(result1.output).toContain('missing.json: Cannot read last run file: ENOENT'); - - const result2 = await runInlineTest(workspace, {}, {}, { additionalArgs: ['--last-failed=malformed.json'] }); - expect(result2.exitCode).toBe(1); - expect(result2.passed).toBe(0); - expect(result2.output).toContain('malformed.json: Cannot read last run file: "failedTests" list is missing'); -}); - -test('should resolve --last-failed and --last-run-output-file against cwd', async ({ runInlineTest }, testInfo) => { - const workspace = { - 'dir/playwright.config.ts': `module.exports = {};`, - 'dir/a.spec.js': ` - import { test, expect } from '@playwright/test'; - test('pass-a', async () => {}); - test('fail-a', async () => { - expect(1).toBe(2); - }); - `, - }; - - const result1 = await runInlineTest(workspace, { config: 'dir/playwright.config.ts' }, {}, { additionalArgs: ['--last-run-output-file=last-run.json'] }); - expect(result1.exitCode).toBe(1); - expect(result1.passed).toBe(1); - expect(result1.failed).toBe(1); - expect(fs.existsSync(testInfo.outputPath('last-run.json'))).toBe(true); - expect(fs.existsSync(testInfo.outputPath('dir', 'last-run.json'))).toBe(false); - - const result2 = await runInlineTest(workspace, { config: 'dir/playwright.config.ts' }, {}, { additionalArgs: ['--last-failed=last-run.json'] }); - expect(result2.exitCode).toBe(1); - expect(result2.passed).toBe(0); - expect(result2.failed).toBe(1); -});