feat(test-runner): add --shuffle to run tests in random order

docs/src/test-cli-js.md

@@ -105,6 +105,7 @@ npx playwright test --ui
 | `--reporter <reporter>` | Reporter to use, comma-separated, can be "dot", "line", "list", or others (default: "list" locally and "dot" on CI). You can also pass a path to a custom reporter file. |
 | `--retries <retries>` | Maximum retry count for flaky tests, zero for no retries (default: no retries). |
 | `--shard <shard>` | Shard tests and execute only the selected shard, specified in the form "current/all", 1-based, e.g., "3/5". |
+| `--shuffle [seed]` | Schedule tests in a random order: test files are shuffled, and so are individual tests in [parallel mode](./test-parallel.md). Tests that must run together, for example in a serial suite, keep their order. The seed is printed at the start of the run, pass it to reproduce the same order. |
 | `--test-list <file>` | Path to a file containing a list of tests to run. See [test list](#test-list) for details. |
 | `--test-list-invert <file>` | Path to a file containing a list of tests to skip. See [test list](#test-list) for details.  |
 | `--timeout <timeout>` | Specify test timeout threshold in milliseconds, zero for unlimited (default: 30 seconds). |

packages/playwright/src/cli/testActions.ts

@@ -128,6 +128,7 @@ function overridesFromOptions(options: { [key: string]: any }): ipc.ConfigCLIOve
     reporter: resolveReporterOption(options.reporter),
     additionalReporters: resolveReporterOption(options.addReporter),
     shard: resolveShardOption(options.shard),
+    shuffle: resolveShuffleOption(options.shuffle),
     timeout: options.timeout ? parseInt(options.timeout, 10) : undefined,
     tsconfig: options.tsconfig ? path.resolve(process.cwd(), options.tsconfig) : undefined,
     ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined,
@@ -170,6 +171,14 @@ function resolveReporterOption(reporter?: string): ReporterDescription[] | undef
   return reporter.split(',').map((r: string) => [resolveReporter(r)]);
 }
 
+function resolveShuffleOption(shuffle?: string | boolean): string | undefined {
+  if (!shuffle)
+    return undefined;
+  if (shuffle === true)
+    return String(Math.floor(Math.random() * 1e9));
+  return shuffle;
+}
+
 function resolveShardOption(shard?: string): ipc.ConfigCLIOverrides['shard'] {
   if (!shard)
     return undefined;

packages/playwright/src/common/ipc.ts

@@ -38,6 +38,7 @@ export type ConfigCLIOverrides = {
   reporter?: ReporterDescription[];
   additionalReporters?: ReporterDescription[];
   shard?: { current: number, total: number };
+  shuffle?: string;
   timeout?: number;
   tsconfig?: string;
   ignoreSnapshots?: boolean;

packages/playwright/src/program.ts

@@ -235,6 +235,7 @@ const testOptions: [string, { description: string, choices?: string[], preset?:
   ['--shard <shard>', { description: `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"` }],
   ['--test-list <file>', { description: `Path to a file containing a list of tests to run. See https://playwright.dev/docs/test-cli for more details.` }],
   ['--test-list-invert <file>', { description: `Path to a file containing a list of tests to skip. See https://playwright.dev/docs/test-cli for more details.` }],
+  ['--shuffle [seed]', { description: `Schedule test files, and tests in parallel mode, in a random order; pass a seed to reproduce a previous order` }],
   ['--timeout <timeout>', { description: `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${config.defaultTimeout})` }],
   ['--trace <mode>', { description: `Force tracing mode`, choices: kTraceModes as string[] }],
   ['--tsconfig <path>', { description: `Path to a single tsconfig applicable to all imported files (default: look up tsconfig for each imported file separately)` }],

packages/playwright/src/reporters/base.ts

@@ -242,9 +242,10 @@ export class TerminalReporter implements ReporterV2 {
   protected generateStartingMessage() {
     const jobs = this.config.metadata.actualWorkers ?? this.config.workers;
     const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : '';
+    const shuffleDetails = this.config.metadata.shuffleSeed ? `, shuffle seed ${this.config.metadata.shuffleSeed}` : '';
     if (!this.totalTestCount)
       return '';
-    return '\n' + this.screen.colors.dim('Running ') + this.totalTestCount + this.screen.colors.dim(` test${this.totalTestCount !== 1 ? 's' : ''} using `) + jobs + this.screen.colors.dim(` worker${jobs !== 1 ? 's' : ''}${shardDetails}`);
+    return '\n' + this.screen.colors.dim('Running ') + this.totalTestCount + this.screen.colors.dim(` test${this.totalTestCount !== 1 ? 's' : ''} using `) + jobs + this.screen.colors.dim(` worker${jobs !== 1 ? 's' : ''}${shardDetails}${shuffleDetails}`);
   }
 
   protected getSlowTests(): [string, number][] {

packages/playwright/src/runner/tasks.ts

@@ -30,7 +30,7 @@ import { applySuggestedRebaselines, clearSuggestedRebaselines } from './rebase';
 import { TaskRunner } from './taskRunner';
 import { detectChangedTestFiles } from './vcs';
 import { cc, config as commonConfig, FullConfigInternal, suiteUtils, test as testNs } from '../common';
-import { createTestGroups } from '../runner/testGroups';
+import { createTestGroups, shuffleTestGroups } from '../runner/testGroups';
 import { createTitleMatcher, forceRegExp, removeDirAndLogToConsole } from '../util';
 
 import type { TestGroup } from '../runner/testGroups';
@@ -418,6 +418,8 @@ function createPhasesTask(): Task<TestRun> {
       }
 
       testRun.config.config.metadata.actualWorkers = Math.min(testRun.config.config.workers, maxConcurrentTestGroups);
+      if (testRun.config.configCLIOverrides.shuffle)
+        testRun.config.config.metadata.shuffleSeed = testRun.config.configCLIOverrides.shuffle;
     },
   };
 }
@@ -449,6 +451,9 @@ function createRunTestsTask(): Task<TestRun> {
             phaseTestGroups.push(...testGroups);
         }
 
+        if (testRun.config.configCLIOverrides.shuffle)
+          shuffleTestGroups(phaseTestGroups, testRun.config.configCLIOverrides.shuffle);
+
         if (phaseTestGroups.length) {
           await dispatcher!.run(phaseTestGroups, extraEnvByProjectId);
           await dispatcher.stop();

packages/playwright/src/runner/testGroups.ts

@@ -14,6 +14,8 @@
  * limitations under the License.
  */
 
+import { calculateSha1 } from '@utils/crypto';
+
 import type { test } from '../common';
 
 export type TestGroup = {
@@ -141,6 +143,11 @@ export function createTestGroups(projectSuite: test.Suite, expectedParallelism:
   return result;
 }
 
+export function shuffleTestGroups(testGroups: TestGroup[], seed: string) {
+  const keys = new Map(testGroups.map(group => [group, calculateSha1(seed + '\x1e' + group.tests[0].id)]));
+  testGroups.sort((a, b) => keys.get(a)!.localeCompare(keys.get(b)!));
+}
+
 export function filterForShard(shard: { total: number, current: number }, weights: number[] | undefined, testGroups: TestGroup[]): Set<TestGroup> {
   weights ??= Array.from({ length: shard.total }, () => 1);
   if (weights.length !== shard.total)

tests/playwright-test/shuffle.spec.ts

@@ -0,0 +1,84 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+const count = 10;
+
+function testFile(name: string) {
+  return `
+    import { test } from '@playwright/test';
+    ${Array.from({ length: count }, (_, i) => `test('test${i}', () => console.log('\\n%%${name}-test${i}'));`).join('\n')}
+  `;
+}
+
+const fileNames = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'];
+const files = Object.fromEntries(fileNames.map(name => [`${name}.spec.ts`, testFile(name)]));
+const testsInFile = (name: string) => Array.from({ length: count }, (_, i) => `${name}-test${i}`);
+const naturalOrder = fileNames.flatMap(testsInFile);
+
+test('should shuffle files and keep tests within a file in order', async ({ runInlineTest }) => {
+  const result = await runInlineTest(files, { workers: 1, shuffle: '42' });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(naturalOrder.length);
+  expect(result.output).toContain('shuffle seed 42');
+  expect(result.outputLines).not.toEqual(naturalOrder);
+  const fileOrder = [...new Set(result.outputLines.map(line => line.split('-')[0]))];
+  expect(result.outputLines).toEqual(fileOrder.flatMap(testsInFile));
+
+  const result2 = await runInlineTest(files, { workers: 1, shuffle: '42' });
+  expect(result2.outputLines).toEqual(result.outputLines);
+
+  const result3 = await runInlineTest(files, { workers: 1, shuffle: '43' });
+  expect(result3.outputLines).not.toEqual(result.outputLines);
+});
+
+test('should shuffle individual tests in fully parallel mode', async ({ runInlineTest }) => {
+  const result = await runInlineTest(files, { 'workers': 1, 'shuffle': '42', 'fully-parallel': true });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(naturalOrder.length);
+  const aTests = result.outputLines.filter(line => line.startsWith('a-'));
+  expect(aTests).not.toEqual(testsInFile('a'));
+  expect([...aTests].sort()).toEqual([...testsInFile('a')].sort());
+});
+
+test('should pick and print a random seed', async ({ runInlineTest }) => {
+  const result = await runInlineTest(files, { workers: 1, shuffle: true });
+  expect(result.exitCode).toBe(0);
+  const seed = result.output.match(/shuffle seed (\d+)/)![1];
+  const result2 = await runInlineTest(files, { workers: 1, shuffle: seed });
+  expect(result2.outputLines).toEqual(result.outputLines);
+});
+
+test('should keep serial suites together in parallel mode', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      import { test, expect } from '@playwright/test';
+      test.describe.configure({ mode: 'parallel' });
+      ${Array.from({ length: count }, (_, i) => `test('test${i}', () => console.log('\\n%%test${i}'));`).join('\n')}
+      test.describe.serial('serial', () => {
+        let counter = 0;
+        ${Array.from({ length: count }, (_, i) => `test('serial${i}', () => { expect(counter++).toBe(${i}); console.log('\\n%%serial${i}'); });`).join('\n')}
+      });
+    `,
+  }, { workers: 1, shuffle: '7' });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(2 * count);
+  const lines = result.outputLines;
+  const serialStart = lines.indexOf('serial0');
+  expect(lines.slice(serialStart, serialStart + count)).toEqual(Array.from({ length: count }, (_, i) => `serial${i}`));
+  expect(lines.filter(line => line.startsWith('test'))).not.toEqual(Array.from({ length: count }, (_, i) => `test${i}`));
+});