diff --git a/docs/src/running-tests-js.md b/docs/src/running-tests-js.md index 0925ef7db8001..1c563a568fee0 100644 --- a/docs/src/running-tests-js.md +++ b/docs/src/running-tests-js.md @@ -91,12 +91,20 @@ To run only the tests that failed in the last test run, first run your tests and npx playwright test --last-failed ``` -Playwright stores the list of failed tests from the previous run in `/.last-run.json` (see [`property: TestConfig.outputDir`](./test-configuration.md)). To use a different file path, pass `--last-failed-file=` or set `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE`. +Playwright stores the list of failed tests from the previous run in `/.last-run.json` (see [`property: TestConfig.outputDir`](./test-configuration.md)). To write it to a different file, pass `--last-run-output-file=` or set `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE`. ```bash -npx playwright test --last-failed --last-failed-file=.cache/last-run-shard-1.json +npx playwright test --last-run-output-file=.cache/last-run-shard-1.json ``` +To re-run the failures recorded in a specific file, pass it to `--last-failed=`. The run fails if the file does not exist. The file is only read, the results of the re-run are written to the default location or to the `--last-run-output-file` path. Without a path, `--last-failed` always reads `/.last-run.json`. + +```bash +npx playwright test --last-failed=.cache/last-run-shard-1.json +``` + +Relative paths are resolved against the current working directory. + ### Run tests in VS Code Tests can be run right from VS Code using the [VS Code extension](https://marketplace.visualstudio.com/items?itemName=ms-playwright.playwright). Once installed you can simply click the green triangle next to the test you want to run or run all tests from the testing sidebar. Check out our [Getting Started with VS Code](./getting-started-vscode.md) guide for more details. diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.md index 02fa76f030573..23b34edfad395 100644 --- a/docs/src/test-cli-js.md +++ b/docs/src/test-cli-js.md @@ -91,8 +91,8 @@ npx playwright test --ui | `--headed` | Run tests in headed browsers (default: headless). | | `--ignore-snapshots` | Ignore screenshot and snapshot expectations. | | `-j ` or `--workers ` | Number of concurrent workers or percentage of logical CPU cores, use 1 to run in a single worker (default: 50%). | -| `--last-failed` | Only re-run the failures. | -| `--last-failed-file ` | Override the default last-run JSON path for `--last-failed` (default: `/.last-run.json`). Same as `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE` environment variable. | +| `--last-failed [file]` | Only re-run the failures. Optionally takes a path to the last run file to read the failures from, use `--last-failed=` form (default: `/.last-run.json`). | +| `--last-run-output-file ` | Path to write the last run file to (default: `/.last-run.json`). Same as `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE` environment variable. | | `--list` | Collect all the tests and report them, but do not run. | | `--max-failures ` or `-x` | Stop after the first `N` failures. Passing `-x` stops after the first failure. | | `--no-deps` | Do not run project dependencies. | diff --git a/packages/playwright/src/cli/testActions.ts b/packages/playwright/src/cli/testActions.ts index c03abcff8064f..f3aed24751d69 100644 --- a/packages/playwright/src/cli/testActions.ts +++ b/packages/playwright/src/cli/testActions.ts @@ -40,7 +40,8 @@ export async function runTests(args: string[], opts: { [key: string]: any }) { projectFilter: opts.project || undefined, passWithNoTests: !!opts.passWithNoTests, lastFailed: !!opts.lastFailed, - lastFailedFile: opts.lastFailedFile, + lastRunInputFile: typeof opts.lastFailed === 'string' ? opts.lastFailed : undefined, + lastRunOutputFile: opts.lastRunOutputFile, testList: opts.testList ? path.resolve(process.cwd(), opts.testList) : undefined, testListInvert: opts.testListInvert ? path.resolve(process.cwd(), opts.testListInvert) : undefined, shardWeights: resolveShardWeightsOption(), diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index 9042f965224aa..505c05b668e04 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -218,8 +218,8 @@ const testOptions: [string, { description: string, choices?: string[], preset?: ['-G, --grep-invert ', { description: `Only run tests that do not match this regular expression` }], ['--headed', { description: `Run tests in headed browsers (default: headless)` }], ['--ignore-snapshots', { description: `Ignore screenshot and snapshot expectations` }], - ['--last-failed', { description: `Only re-run the failures` }], - ['--last-failed-file ', { description: `Override the default path for the last-run JSON file used with --last-failed (default: /.last-run.json). Same as PLAYWRIGHT_LAST_RUN_OUTPUT_FILE environment variable.` }], + ['--last-failed [file]', { description: `Only re-run the failures. Optionally takes a path to the last run file to read the failures from, use --last-failed= form (default: /.last-run.json)` }], + ['--last-run-output-file ', { description: `Path to write the last run file to (default: /.last-run.json). Same as PLAYWRIGHT_LAST_RUN_OUTPUT_FILE environment variable.` }], ['--list', { description: `Collect all the tests and report them, but do not run` }], ['--max-failures ', { description: `Stop after the first N failures` }], ['--no-deps', { description: `Do not run project dependencies` }], diff --git a/packages/playwright/src/runner/lastRun.ts b/packages/playwright/src/runner/lastRun.ts index 8312da4471ea6..97bf5c8e4b842 100644 --- a/packages/playwright/src/runner/lastRun.ts +++ b/packages/playwright/src/runner/lastRun.ts @@ -17,6 +17,8 @@ import fs from 'fs'; import path from 'path'; +import { errorWithFile } from '../util'; + import type { FullResult, Suite, TestCase } from '../../types/testReporter'; import type { config as commonConfig } from '../common'; import type { ReporterV2 } from '../reporters/reporterV2'; @@ -35,30 +37,31 @@ function didNotRun(test: TestCase): boolean { } export class LastRunReporter implements ReporterV2 { - private _lastRunFile: string | undefined; + private _inputFile: string | undefined; + private _isDefaultInputFile: boolean; + private _outputFile: string | undefined; private _suite: Suite | undefined; private _listMode: boolean; - constructor(filteredProjects: commonConfig.FullProjectInternal[], listMode?: boolean, lastFailedFileOverride?: string) { - this._listMode = !!listMode; - const override = lastFailedFileOverride ?? process.env.PLAYWRIGHT_LAST_RUN_OUTPUT_FILE; - if (override) { - this._lastRunFile = path.resolve(process.cwd(), override); - } else { - const [project] = filteredProjects; - if (project) - this._lastRunFile = path.join(project.project.outputDir, '.last-run.json'); - } + constructor(filteredProjects: commonConfig.FullProjectInternal[], options: { listMode?: boolean, inputFile?: string, outputFile?: string }) { + this._listMode = !!options.listMode; + const [project] = filteredProjects; + const defaultFile = project ? path.join(project.project.outputDir, '.last-run.json') : undefined; + const outputFile = options.outputFile ?? process.env.PLAYWRIGHT_LAST_RUN_OUTPUT_FILE; + this._outputFile = outputFile ? path.resolve(process.cwd(), outputFile) : defaultFile; + this._inputFile = options.inputFile ? path.resolve(process.cwd(), options.inputFile) : defaultFile; + this._isDefaultInputFile = !options.inputFile; } async filterLastFailed(): Promise { - if (!this._lastRunFile) + if (!this._inputFile) return undefined; try { - const lastRunInfo = JSON.parse(await fs.promises.readFile(this._lastRunFile, 'utf8')) as LastRunInfo; - return lastRunInfo.failedTests; - } catch { - return undefined; + return await readFailedTests(this._inputFile); + } catch (e) { + if (this._isDefaultInputFile) + return undefined; + throw e; } } @@ -75,13 +78,25 @@ export class LastRunReporter implements ReporterV2 { } async onEnd(result: FullResult) { - if (!this._lastRunFile || this._listMode) + if (!this._outputFile || this._listMode) return; const lastRunInfo: LastRunInfo = { status: result.status, failedTests: this._suite?.allTests().filter(t => !t.ok() || didNotRun(t)).map(t => t.id) || [], }; - await fs.promises.mkdir(path.dirname(this._lastRunFile), { recursive: true }); - await fs.promises.writeFile(this._lastRunFile, JSON.stringify(lastRunInfo, undefined, 2)); + await fs.promises.mkdir(path.dirname(this._outputFile), { recursive: true }); + await fs.promises.writeFile(this._outputFile, JSON.stringify(lastRunInfo, undefined, 2)); + } +} + +async function readFailedTests(file: string): Promise { + let lastRunInfo: LastRunInfo; + try { + lastRunInfo = JSON.parse(await fs.promises.readFile(file, 'utf8')); + } catch (e) { + throw errorWithFile(file, 'Cannot read last run file: ' + e.message); } + if (!Array.isArray(lastRunInfo?.failedTests)) + throw errorWithFile(file, 'Cannot read last run file: "failedTests" list is missing'); + return lastRunInfo.failedTests; } diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts index 1361860eab42d..c9426b0c736fa 100644 --- a/packages/playwright/src/runner/tasks.ts +++ b/packages/playwright/src/runner/tasks.ts @@ -63,7 +63,8 @@ export type TestRunOptions = { listMode?: boolean; passWithNoTests?: boolean; lastFailed?: boolean; - lastFailedFile?: string; + lastRunInputFile?: string; + lastRunOutputFile?: string; testList?: string; testListInvert?: string; lastFailedTestIds?: string[]; diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.ts index df44df98a18df..e472b9c4df954 100644 --- a/packages/playwright/src/runner/testRunner.ts +++ b/packages/playwright/src/runner/testRunner.ts @@ -429,7 +429,7 @@ export async function runAllTestsWithConfig(config: FullConfigInternal, options: const filteredProjects = filterProjects(config.projects, options.projectFilter); const reporters = await createReporters(config, options.listMode ? 'list' : 'test', undefined, options); - const lastRun = new LastRunReporter(filteredProjects, options.listMode, options.lastFailedFile); + const lastRun = new LastRunReporter(filteredProjects, { listMode: options.listMode, inputFile: options.lastRunInputFile, outputFile: options.lastRunOutputFile }); if (options.lastFailed) { const lastFailedTestIds = await lastRun.filterLastFailed(); if (lastFailedTestIds) diff --git a/tests/playwright-test/runner.spec.ts b/tests/playwright-test/runner.spec.ts index 812f6cade29ed..5914e2094a50e 100644 --- a/tests/playwright-test/runner.spec.ts +++ b/tests/playwright-test/runner.spec.ts @@ -977,7 +977,7 @@ test('should run last failed tests in a shard with PLAYWRIGHT_LAST_RUN_OUTPUT_FI expect(result1.output).toContain('b.spec.js:3:11 › pass-b'); expect(result1.output).toContain('b.spec.js:4:11 › fail-b'); - const result2 = await runInlineTest(workspace, { shard: '2/2' }, env, { additionalArgs: ['--last-failed'] }); + const result2 = await runInlineTest(workspace, { shard: '2/2' }, env, { additionalArgs: [`--last-failed=${customRel}`] }); expect(result2.exitCode).toBe(1); expect(result2.passed).toBe(0); expect(result2.failed).toBe(1); @@ -985,11 +985,11 @@ test('should run last failed tests in a shard with PLAYWRIGHT_LAST_RUN_OUTPUT_FI expect(result2.output).toContain('b.spec.js:4:11 › fail-b'); }); -test('should run last failed tests in a shard with --last-failed-file', async ({ runInlineTest }, testInfo) => { +test('should run last failed tests in a shard with --last-run-output-file', async ({ runInlineTest }, testInfo) => { const customRel = '.cache/shard-2-cli-last-run.json'; const customAbs = path.join(testInfo.outputPath(), customRel); const defaultLastRun = path.join(testInfo.outputPath(), 'test-results', '.last-run.json'); - const lastRunArgs = ['--last-failed', `--last-failed-file=${customRel}`]; + const lastRunArgs = [`--last-run-output-file=${customRel}`]; const workspace = { 'a.spec.js': ` import { test, expect } from '@playwright/test'; @@ -1015,10 +1015,92 @@ test('should run last failed tests in a shard with --last-failed-file', async ({ expect(result1.output).toContain('b.spec.js:3:11 › pass-b'); expect(result1.output).toContain('b.spec.js:4:11 › fail-b'); - const result2 = await runInlineTest(workspace, { shard: '2/2' }, {}, { additionalArgs: lastRunArgs }); + const result2 = await runInlineTest(workspace, { shard: '2/2' }, {}, { additionalArgs: [`--last-failed=${customRel}`, ...lastRunArgs] }); expect(result2.exitCode).toBe(1); expect(result2.passed).toBe(0); expect(result2.failed).toBe(1); expect(result2.output).not.toContain('b.spec.js:3:11 › pass-b'); expect(result2.output).toContain('b.spec.js:4:11 › fail-b'); }); + +test('should read --last-failed= without overwriting it', async ({ runInlineTest }, testInfo) => { + const inputFile = testInfo.outputPath('.cache', 'input.json'); + const outputFile = testInfo.outputPath('.cache', 'output.json'); + const defaultLastRun = testInfo.outputPath('test-results', '.last-run.json'); + const spec = (expected: number) => ` + import { test, expect } from '@playwright/test'; + test('pass-a', async () => {}); + test('fail-a', async () => { + expect(1).toBe(${expected}); + }); + `; + + const result1 = await runInlineTest({ 'a.spec.js': spec(2) }, {}, {}, { additionalArgs: ['--last-run-output-file=.cache/input.json'] }); + expect(result1.exitCode).toBe(1); + expect(result1.passed).toBe(1); + expect(result1.failed).toBe(1); + expect(fs.existsSync(defaultLastRun)).toBe(false); + const input = fs.readFileSync(inputFile, 'utf8'); + expect(JSON.parse(input).failedTests).toHaveLength(1); + + const result2 = await runInlineTest({ 'a.spec.js': spec(1) }, {}, {}, { additionalArgs: ['--last-failed=.cache/input.json'] }); + expect(result2.exitCode).toBe(0); + expect(result2.passed).toBe(1); + expect(result2.output).not.toContain('pass-a'); + expect(result2.output).toContain('fail-a'); + expect(fs.readFileSync(inputFile, 'utf8')).toBe(input); + expect(JSON.parse(fs.readFileSync(defaultLastRun, 'utf8'))).toEqual({ status: 'passed', failedTests: [] }); + + const result3 = await runInlineTest({ 'a.spec.js': spec(1) }, {}, {}, { additionalArgs: ['--last-failed=.cache/input.json', '--last-run-output-file=.cache/output.json'] }); + expect(result3.exitCode).toBe(0); + expect(result3.passed).toBe(1); + expect(result3.output).not.toContain('pass-a'); + expect(fs.readFileSync(inputFile, 'utf8')).toBe(input); + expect(JSON.parse(fs.readFileSync(outputFile, 'utf8'))).toEqual({ status: 'passed', failedTests: [] }); + expect(fs.existsSync(defaultLastRun)).toBe(false); +}); + +test('should fail when --last-failed= is missing or malformed', async ({ runInlineTest }) => { + const workspace = { + 'a.spec.js': ` + import { test, expect } from '@playwright/test'; + test('pass-a', async () => {}); + `, + 'malformed.json': `{ "status": "failed" }`, + }; + + const result1 = await runInlineTest(workspace, {}, {}, { additionalArgs: ['--last-failed=missing.json'] }); + expect(result1.exitCode).toBe(1); + expect(result1.passed).toBe(0); + expect(result1.output).toContain('missing.json: Cannot read last run file: ENOENT'); + + const result2 = await runInlineTest(workspace, {}, {}, { additionalArgs: ['--last-failed=malformed.json'] }); + expect(result2.exitCode).toBe(1); + expect(result2.passed).toBe(0); + expect(result2.output).toContain('malformed.json: Cannot read last run file: "failedTests" list is missing'); +}); + +test('should resolve --last-failed and --last-run-output-file against cwd', async ({ runInlineTest }, testInfo) => { + const workspace = { + 'dir/playwright.config.ts': `module.exports = {};`, + 'dir/a.spec.js': ` + import { test, expect } from '@playwright/test'; + test('pass-a', async () => {}); + test('fail-a', async () => { + expect(1).toBe(2); + }); + `, + }; + + const result1 = await runInlineTest(workspace, { config: 'dir/playwright.config.ts' }, {}, { additionalArgs: ['--last-run-output-file=last-run.json'] }); + expect(result1.exitCode).toBe(1); + expect(result1.passed).toBe(1); + expect(result1.failed).toBe(1); + expect(fs.existsSync(testInfo.outputPath('last-run.json'))).toBe(true); + expect(fs.existsSync(testInfo.outputPath('dir', 'last-run.json'))).toBe(false); + + const result2 = await runInlineTest(workspace, { config: 'dir/playwright.config.ts' }, {}, { additionalArgs: ['--last-failed=last-run.json'] }); + expect(result2.exitCode).toBe(1); + expect(result2.passed).toBe(0); + expect(result2.failed).toBe(1); +});