feat(test-runner): read last failed tests from --last-failed=<file>

docs/src/running-tests-js.md

@@ -91,12 +91,20 @@ To run only the tests that failed in the last test run, first run your tests and
 npx playwright test --last-failed
 ```
 
-Playwright stores the list of failed tests from the previous run in `<outputDir>/.last-run.json` (see [`property: TestConfig.outputDir`](./test-configuration.md)). To use a different file path, pass `--last-failed-file=<path>` or set `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE`.
+Playwright stores the list of failed tests from the previous run in `<outputDir>/.last-run.json` (see [`property: TestConfig.outputDir`](./test-configuration.md)). To write it to a different file, pass `--last-run-output-file=<path>` or set `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE`.
 
 ```bash
-npx playwright test --last-failed --last-failed-file=.cache/last-run-shard-1.json
+npx playwright test --last-run-output-file=.cache/last-run-shard-1.json
 ```
 
+To re-run the failures recorded in a specific file, pass it to `--last-failed=<path>`. The run fails if the file does not exist. The file is only read, the results of the re-run are written to the default location or to the `--last-run-output-file` path. Without a path, `--last-failed` always reads `<outputDir>/.last-run.json`.
+
+```bash
+npx playwright test --last-failed=.cache/last-run-shard-1.json
+```
+
+Relative paths are resolved against the current working directory.
+
 ### Run tests in VS Code
 
 Tests can be run right from VS Code using the [VS Code extension](https://marketplace.visualstudio.com/items?itemName=ms-playwright.playwright). Once installed you can simply click the green triangle next to the test you want to run or run all tests from the testing sidebar. Check out our [Getting Started with VS Code](./getting-started-vscode.md) guide for more details.

docs/src/test-cli-js.md

@@ -91,8 +91,8 @@ npx playwright test --ui
 | `--headed` | Run tests in headed browsers (default: headless). |
 | `--ignore-snapshots` | Ignore screenshot and snapshot expectations. |
 | `-j <workers>` or `--workers <workers>` | Number of concurrent workers or percentage of logical CPU cores, use 1 to run in a single worker (default: 50%). |
-| `--last-failed` | Only re-run the failures. |
-| `--last-failed-file <file>` | Override the default last-run JSON path for `--last-failed` (default: `<outputDir>/.last-run.json`). Same as `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE` environment variable. |
+| `--last-failed [file]` | Only re-run the failures. Optionally takes a path to the last run file to read the failures from, use `--last-failed=<file>` form (default: `<outputDir>/.last-run.json`). |
+| `--last-run-output-file <file>` | Path to write the last run file to (default: `<outputDir>/.last-run.json`). Same as `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE` environment variable. |
 | `--list` | Collect all the tests and report them, but do not run. |
 | `--max-failures <N>` or `-x` | Stop after the first `N` failures. Passing `-x` stops after the first failure. |
 | `--no-deps` | Do not run project dependencies. |

packages/playwright/src/cli/testActions.ts

@@ -40,7 +40,8 @@ export async function runTests(args: string[], opts: { [key: string]: any }) {
     projectFilter: opts.project || undefined,
     passWithNoTests: !!opts.passWithNoTests,
     lastFailed: !!opts.lastFailed,
-    lastFailedFile: opts.lastFailedFile,
+    lastRunInputFile: typeof opts.lastFailed === 'string' ? opts.lastFailed : undefined,
+    lastRunOutputFile: opts.lastRunOutputFile,
     testList: opts.testList ? path.resolve(process.cwd(), opts.testList) : undefined,
     testListInvert: opts.testListInvert ? path.resolve(process.cwd(), opts.testListInvert) : undefined,
     shardWeights: resolveShardWeightsOption(),

packages/playwright/src/program.ts

@@ -218,8 +218,8 @@ const testOptions: [string, { description: string, choices?: string[], preset?:
   ['-G, --grep-invert <grep>', { description: `Only run tests that do not match this regular expression` }],
   ['--headed', { description: `Run tests in headed browsers (default: headless)` }],
   ['--ignore-snapshots', { description: `Ignore screenshot and snapshot expectations` }],
-  ['--last-failed', { description: `Only re-run the failures` }],
-  ['--last-failed-file <file>', { description: `Override the default path for the last-run JSON file used with --last-failed (default: <outputDir>/.last-run.json). Same as PLAYWRIGHT_LAST_RUN_OUTPUT_FILE environment variable.` }],
+  ['--last-failed [file]', { description: `Only re-run the failures. Optionally takes a path to the last run file to read the failures from, use --last-failed=<file> form (default: <outputDir>/.last-run.json)` }],
+  ['--last-run-output-file <file>', { description: `Path to write the last run file to (default: <outputDir>/.last-run.json). Same as PLAYWRIGHT_LAST_RUN_OUTPUT_FILE environment variable.` }],
   ['--list', { description: `Collect all the tests and report them, but do not run` }],
   ['--max-failures <N>', { description: `Stop after the first N failures` }],
   ['--no-deps', { description: `Do not run project dependencies` }],

packages/playwright/src/runner/lastRun.ts

@@ -17,6 +17,8 @@
 import fs from 'fs';
 import path from 'path';
 
+import { errorWithFile } from '../util';
+
 import type { FullResult, Suite, TestCase } from '../../types/testReporter';
 import type { config as commonConfig } from '../common';
 import type { ReporterV2 } from '../reporters/reporterV2';
@@ -35,30 +37,31 @@ function didNotRun(test: TestCase): boolean {
 }
 
 export class LastRunReporter implements ReporterV2 {
-  private _lastRunFile: string | undefined;
+  private _inputFile: string | undefined;
+  private _isDefaultInputFile: boolean;
+  private _outputFile: string | undefined;
   private _suite: Suite | undefined;
   private _listMode: boolean;
 
-  constructor(filteredProjects: commonConfig.FullProjectInternal[], listMode?: boolean, lastFailedFileOverride?: string) {
-    this._listMode = !!listMode;
-    const override = lastFailedFileOverride ?? process.env.PLAYWRIGHT_LAST_RUN_OUTPUT_FILE;
-    if (override) {
-      this._lastRunFile = path.resolve(process.cwd(), override);
-    } else {
-      const [project] = filteredProjects;
-      if (project)
-        this._lastRunFile = path.join(project.project.outputDir, '.last-run.json');
-    }
+  constructor(filteredProjects: commonConfig.FullProjectInternal[], options: { listMode?: boolean, inputFile?: string, outputFile?: string }) {
+    this._listMode = !!options.listMode;
+    const [project] = filteredProjects;
+    const defaultFile = project ? path.join(project.project.outputDir, '.last-run.json') : undefined;
+    const outputFile = options.outputFile ?? process.env.PLAYWRIGHT_LAST_RUN_OUTPUT_FILE;
+    this._outputFile = outputFile ? path.resolve(process.cwd(), outputFile) : defaultFile;
+    this._inputFile = options.inputFile ? path.resolve(process.cwd(), options.inputFile) : defaultFile;
+    this._isDefaultInputFile = !options.inputFile;
   }
 
   async filterLastFailed(): Promise<string[] | undefined> {
-    if (!this._lastRunFile)
+    if (!this._inputFile)
       return undefined;
     try {
-      const lastRunInfo = JSON.parse(await fs.promises.readFile(this._lastRunFile, 'utf8')) as LastRunInfo;
-      return lastRunInfo.failedTests;
-    } catch {
-      return undefined;
+      return await readFailedTests(this._inputFile);
+    } catch (e) {
+      if (this._isDefaultInputFile)
+        return undefined;
+      throw e;
     }
   }
 
@@ -75,13 +78,25 @@ export class LastRunReporter implements ReporterV2 {
   }
 
   async onEnd(result: FullResult) {
-    if (!this._lastRunFile || this._listMode)
+    if (!this._outputFile || this._listMode)
       return;
     const lastRunInfo: LastRunInfo = {
       status: result.status,
       failedTests: this._suite?.allTests().filter(t => !t.ok() || didNotRun(t)).map(t => t.id) || [],
     };
-    await fs.promises.mkdir(path.dirname(this._lastRunFile), { recursive: true });
-    await fs.promises.writeFile(this._lastRunFile, JSON.stringify(lastRunInfo, undefined, 2));
+    await fs.promises.mkdir(path.dirname(this._outputFile), { recursive: true });
+    await fs.promises.writeFile(this._outputFile, JSON.stringify(lastRunInfo, undefined, 2));
+  }
+}
+
+async function readFailedTests(file: string): Promise<string[]> {
+  let lastRunInfo: LastRunInfo;
+  try {
+    lastRunInfo = JSON.parse(await fs.promises.readFile(file, 'utf8'));
+  } catch (e) {
+    throw errorWithFile(file, 'Cannot read last run file: ' + e.message);
   }
+  if (!Array.isArray(lastRunInfo?.failedTests))
+    throw errorWithFile(file, 'Cannot read last run file: "failedTests" list is missing');
+  return lastRunInfo.failedTests;
 }

packages/playwright/src/runner/tasks.ts

@@ -63,7 +63,8 @@ export type TestRunOptions = {
   listMode?: boolean;
   passWithNoTests?: boolean;
   lastFailed?: boolean;
-  lastFailedFile?: string;
+  lastRunInputFile?: string;
+  lastRunOutputFile?: string;
   testList?: string;
   testListInvert?: string;
   lastFailedTestIds?: string[];

packages/playwright/src/runner/testRunner.ts

@@ -429,7 +429,7 @@ export async function runAllTestsWithConfig(config: FullConfigInternal, options:
 
   const filteredProjects = filterProjects(config.projects, options.projectFilter);
   const reporters = await createReporters(config, options.listMode ? 'list' : 'test', undefined, options);
-  const lastRun = new LastRunReporter(filteredProjects, options.listMode, options.lastFailedFile);
+  const lastRun = new LastRunReporter(filteredProjects, { listMode: options.listMode, inputFile: options.lastRunInputFile, outputFile: options.lastRunOutputFile });
   if (options.lastFailed) {
     const lastFailedTestIds = await lastRun.filterLastFailed();
     if (lastFailedTestIds)

tests/playwright-test/runner.spec.ts

@@ -977,19 +977,19 @@ test('should run last failed tests in a shard with PLAYWRIGHT_LAST_RUN_OUTPUT_FI
   expect(result1.output).toContain('b.spec.js:3:11 › pass-b');
   expect(result1.output).toContain('b.spec.js:4:11 › fail-b');
 
-  const result2 = await runInlineTest(workspace, { shard: '2/2' }, env, { additionalArgs: ['--last-failed'] });
+  const result2 = await runInlineTest(workspace, { shard: '2/2' }, env, { additionalArgs: [`--last-failed=${customRel}`] });
   expect(result2.exitCode).toBe(1);
   expect(result2.passed).toBe(0);
   expect(result2.failed).toBe(1);
   expect(result2.output).not.toContain('b.spec.js:3:11 › pass-b');
   expect(result2.output).toContain('b.spec.js:4:11 › fail-b');
 });
 
-test('should run last failed tests in a shard with --last-failed-file', async ({ runInlineTest }, testInfo) => {
+test('should run last failed tests in a shard with --last-run-output-file', async ({ runInlineTest }, testInfo) => {
   const customRel = '.cache/shard-2-cli-last-run.json';
   const customAbs = path.join(testInfo.outputPath(), customRel);
   const defaultLastRun = path.join(testInfo.outputPath(), 'test-results', '.last-run.json');
-  const lastRunArgs = ['--last-failed', `--last-failed-file=${customRel}`];
+  const lastRunArgs = [`--last-run-output-file=${customRel}`];
   const workspace = {
     'a.spec.js': `
       import { test, expect } from '@playwright/test';
@@ -1015,10 +1015,92 @@ test('should run last failed tests in a shard with --last-failed-file', async ({
   expect(result1.output).toContain('b.spec.js:3:11 › pass-b');
   expect(result1.output).toContain('b.spec.js:4:11 › fail-b');
 
-  const result2 = await runInlineTest(workspace, { shard: '2/2' }, {}, { additionalArgs: lastRunArgs });
+  const result2 = await runInlineTest(workspace, { shard: '2/2' }, {}, { additionalArgs: [`--last-failed=${customRel}`, ...lastRunArgs] });
   expect(result2.exitCode).toBe(1);
   expect(result2.passed).toBe(0);
   expect(result2.failed).toBe(1);
   expect(result2.output).not.toContain('b.spec.js:3:11 › pass-b');
   expect(result2.output).toContain('b.spec.js:4:11 › fail-b');
 });
+
+test('should read --last-failed=<file> without overwriting it', async ({ runInlineTest }, testInfo) => {
+  const inputFile = testInfo.outputPath('.cache', 'input.json');
+  const outputFile = testInfo.outputPath('.cache', 'output.json');
+  const defaultLastRun = testInfo.outputPath('test-results', '.last-run.json');
+  const spec = (expected: number) => `
+    import { test, expect } from '@playwright/test';
+    test('pass-a', async () => {});
+    test('fail-a', async () => {
+      expect(1).toBe(${expected});
+    });
+  `;
+
+  const result1 = await runInlineTest({ 'a.spec.js': spec(2) }, {}, {}, { additionalArgs: ['--last-run-output-file=.cache/input.json'] });
+  expect(result1.exitCode).toBe(1);
+  expect(result1.passed).toBe(1);
+  expect(result1.failed).toBe(1);
+  expect(fs.existsSync(defaultLastRun)).toBe(false);
+  const input = fs.readFileSync(inputFile, 'utf8');
+  expect(JSON.parse(input).failedTests).toHaveLength(1);
+
+  const result2 = await runInlineTest({ 'a.spec.js': spec(1) }, {}, {}, { additionalArgs: ['--last-failed=.cache/input.json'] });
+  expect(result2.exitCode).toBe(0);
+  expect(result2.passed).toBe(1);
+  expect(result2.output).not.toContain('pass-a');
+  expect(result2.output).toContain('fail-a');
+  expect(fs.readFileSync(inputFile, 'utf8')).toBe(input);
+  expect(JSON.parse(fs.readFileSync(defaultLastRun, 'utf8'))).toEqual({ status: 'passed', failedTests: [] });
+
+  const result3 = await runInlineTest({ 'a.spec.js': spec(1) }, {}, {}, { additionalArgs: ['--last-failed=.cache/input.json', '--last-run-output-file=.cache/output.json'] });
+  expect(result3.exitCode).toBe(0);
+  expect(result3.passed).toBe(1);
+  expect(result3.output).not.toContain('pass-a');
+  expect(fs.readFileSync(inputFile, 'utf8')).toBe(input);
+  expect(JSON.parse(fs.readFileSync(outputFile, 'utf8'))).toEqual({ status: 'passed', failedTests: [] });
+  expect(fs.existsSync(defaultLastRun)).toBe(false);
+});
+
+test('should fail when --last-failed=<file> is missing or malformed', async ({ runInlineTest }) => {
+  const workspace = {
+    'a.spec.js': `
+      import { test, expect } from '@playwright/test';
+      test('pass-a', async () => {});
+    `,
+    'malformed.json': `{ "status": "failed" }`,
+  };
+
+  const result1 = await runInlineTest(workspace, {}, {}, { additionalArgs: ['--last-failed=missing.json'] });
+  expect(result1.exitCode).toBe(1);
+  expect(result1.passed).toBe(0);
+  expect(result1.output).toContain('missing.json: Cannot read last run file: ENOENT');
+
+  const result2 = await runInlineTest(workspace, {}, {}, { additionalArgs: ['--last-failed=malformed.json'] });
+  expect(result2.exitCode).toBe(1);
+  expect(result2.passed).toBe(0);
+  expect(result2.output).toContain('malformed.json: Cannot read last run file: "failedTests" list is missing');
+});
+
+test('should resolve --last-failed and --last-run-output-file against cwd', async ({ runInlineTest }, testInfo) => {
+  const workspace = {
+    'dir/playwright.config.ts': `module.exports = {};`,
+    'dir/a.spec.js': `
+      import { test, expect } from '@playwright/test';
+      test('pass-a', async () => {});
+      test('fail-a', async () => {
+        expect(1).toBe(2);
+      });
+    `,
+  };
+
+  const result1 = await runInlineTest(workspace, { config: 'dir/playwright.config.ts' }, {}, { additionalArgs: ['--last-run-output-file=last-run.json'] });
+  expect(result1.exitCode).toBe(1);
+  expect(result1.passed).toBe(1);
+  expect(result1.failed).toBe(1);
+  expect(fs.existsSync(testInfo.outputPath('last-run.json'))).toBe(true);
+  expect(fs.existsSync(testInfo.outputPath('dir', 'last-run.json'))).toBe(false);
+
+  const result2 = await runInlineTest(workspace, { config: 'dir/playwright.config.ts' }, {}, { additionalArgs: ['--last-failed=last-run.json'] });
+  expect(result2.exitCode).toBe(1);
+  expect(result2.passed).toBe(0);
+  expect(result2.failed).toBe(1);
+});