diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.md index 5a2bb5ca52663..a38210032b80c 100644 --- a/docs/src/test-cli-js.md +++ b/docs/src/test-cli-js.md @@ -105,6 +105,7 @@ npx playwright test --ui | `--reporter ` | Reporter to use, comma-separated, can be "dot", "line", "list", or others (default: "list" locally and "dot" on CI). You can also pass a path to a custom reporter file. | | `--retries ` | Maximum retry count for flaky tests, zero for no retries (default: no retries). | | `--shard ` | Shard tests and execute only the selected shard, specified in the form "current/all", 1-based, e.g., "3/5". | +| `--shuffle [seed]` | Schedule tests in a random order: test files are shuffled, and so are individual tests in [parallel mode](./test-parallel.md). Tests that must run together, for example in a serial suite, keep their order. The seed is printed at the start of the run, pass it to reproduce the same order. | | `--test-list ` | Path to a file containing a list of tests to run. See [test list](#test-list) for details. | | `--test-list-invert ` | Path to a file containing a list of tests to skip. See [test list](#test-list) for details. | | `--timeout ` | Specify test timeout threshold in milliseconds, zero for unlimited (default: 30 seconds). | diff --git a/packages/playwright/src/cli/testActions.ts b/packages/playwright/src/cli/testActions.ts index f3cd08f907d63..c03abcff8064f 100644 --- a/packages/playwright/src/cli/testActions.ts +++ b/packages/playwright/src/cli/testActions.ts @@ -128,6 +128,7 @@ function overridesFromOptions(options: { [key: string]: any }): ipc.ConfigCLIOve reporter: resolveReporterOption(options.reporter), additionalReporters: resolveReporterOption(options.addReporter), shard: resolveShardOption(options.shard), + shuffle: resolveShuffleOption(options.shuffle), timeout: options.timeout ? parseInt(options.timeout, 10) : undefined, tsconfig: options.tsconfig ? path.resolve(process.cwd(), options.tsconfig) : undefined, ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined, @@ -170,6 +171,14 @@ function resolveReporterOption(reporter?: string): ReporterDescription[] | undef return reporter.split(',').map((r: string) => [resolveReporter(r)]); } +function resolveShuffleOption(shuffle?: string | boolean): string | undefined { + if (!shuffle) + return undefined; + if (shuffle === true) + return String(Math.floor(Math.random() * 1e9)); + return shuffle; +} + function resolveShardOption(shard?: string): ipc.ConfigCLIOverrides['shard'] { if (!shard) return undefined; diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts index 975fa56d0b55b..872e06b30eabc 100644 --- a/packages/playwright/src/common/ipc.ts +++ b/packages/playwright/src/common/ipc.ts @@ -38,6 +38,7 @@ export type ConfigCLIOverrides = { reporter?: ReporterDescription[]; additionalReporters?: ReporterDescription[]; shard?: { current: number, total: number }; + shuffle?: string; timeout?: number; tsconfig?: string; ignoreSnapshots?: boolean; diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index 2bedd91609413..9042f965224aa 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -235,6 +235,7 @@ const testOptions: [string, { description: string, choices?: string[], preset?: ['--shard ', { description: `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"` }], ['--test-list ', { description: `Path to a file containing a list of tests to run. See https://playwright.dev/docs/test-cli for more details.` }], ['--test-list-invert ', { description: `Path to a file containing a list of tests to skip. See https://playwright.dev/docs/test-cli for more details.` }], + ['--shuffle [seed]', { description: `Schedule test files, and tests in parallel mode, in a random order; pass a seed to reproduce a previous order` }], ['--timeout ', { description: `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${config.defaultTimeout})` }], ['--trace ', { description: `Force tracing mode`, choices: kTraceModes as string[] }], ['--tsconfig ', { description: `Path to a single tsconfig applicable to all imported files (default: look up tsconfig for each imported file separately)` }], diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts index 062584f54d1c6..00bbb0172bed9 100644 --- a/packages/playwright/src/reporters/base.ts +++ b/packages/playwright/src/reporters/base.ts @@ -242,9 +242,10 @@ export class TerminalReporter implements ReporterV2 { protected generateStartingMessage() { const jobs = this.config.metadata.actualWorkers ?? this.config.workers; const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : ''; + const shuffleDetails = this.config.metadata.shuffleSeed ? `, shuffle seed ${this.config.metadata.shuffleSeed}` : ''; if (!this.totalTestCount) return ''; - return '\n' + this.screen.colors.dim('Running ') + this.totalTestCount + this.screen.colors.dim(` test${this.totalTestCount !== 1 ? 's' : ''} using `) + jobs + this.screen.colors.dim(` worker${jobs !== 1 ? 's' : ''}${shardDetails}`); + return '\n' + this.screen.colors.dim('Running ') + this.totalTestCount + this.screen.colors.dim(` test${this.totalTestCount !== 1 ? 's' : ''} using `) + jobs + this.screen.colors.dim(` worker${jobs !== 1 ? 's' : ''}${shardDetails}${shuffleDetails}`); } protected getSlowTests(): [string, number][] { diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts index 17ea0785a4f61..1361860eab42d 100644 --- a/packages/playwright/src/runner/tasks.ts +++ b/packages/playwright/src/runner/tasks.ts @@ -30,7 +30,7 @@ import { applySuggestedRebaselines, clearSuggestedRebaselines } from './rebase'; import { TaskRunner } from './taskRunner'; import { detectChangedTestFiles } from './vcs'; import { cc, config as commonConfig, FullConfigInternal, suiteUtils, test as testNs } from '../common'; -import { createTestGroups } from '../runner/testGroups'; +import { createTestGroups, shuffleTestGroups } from '../runner/testGroups'; import { createTitleMatcher, forceRegExp, removeDirAndLogToConsole } from '../util'; import type { TestGroup } from '../runner/testGroups'; @@ -418,6 +418,8 @@ function createPhasesTask(): Task { } testRun.config.config.metadata.actualWorkers = Math.min(testRun.config.config.workers, maxConcurrentTestGroups); + if (testRun.config.configCLIOverrides.shuffle) + testRun.config.config.metadata.shuffleSeed = testRun.config.configCLIOverrides.shuffle; }, }; } @@ -449,6 +451,9 @@ function createRunTestsTask(): Task { phaseTestGroups.push(...testGroups); } + if (testRun.config.configCLIOverrides.shuffle) + shuffleTestGroups(phaseTestGroups, testRun.config.configCLIOverrides.shuffle); + if (phaseTestGroups.length) { await dispatcher!.run(phaseTestGroups, extraEnvByProjectId); await dispatcher.stop(); diff --git a/packages/playwright/src/runner/testGroups.ts b/packages/playwright/src/runner/testGroups.ts index 5c7bebc3405ce..937ad8ac562d5 100644 --- a/packages/playwright/src/runner/testGroups.ts +++ b/packages/playwright/src/runner/testGroups.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import { calculateSha1 } from '@utils/crypto'; + import type { test } from '../common'; export type TestGroup = { @@ -141,6 +143,11 @@ export function createTestGroups(projectSuite: test.Suite, expectedParallelism: return result; } +export function shuffleTestGroups(testGroups: TestGroup[], seed: string) { + const keys = new Map(testGroups.map(group => [group, calculateSha1(seed + '\x1e' + group.tests[0].id)])); + testGroups.sort((a, b) => keys.get(a)!.localeCompare(keys.get(b)!)); +} + export function filterForShard(shard: { total: number, current: number }, weights: number[] | undefined, testGroups: TestGroup[]): Set { weights ??= Array.from({ length: shard.total }, () => 1); if (weights.length !== shard.total) diff --git a/tests/playwright-test/shuffle.spec.ts b/tests/playwright-test/shuffle.spec.ts new file mode 100644 index 0000000000000..2f9af40c74d6b --- /dev/null +++ b/tests/playwright-test/shuffle.spec.ts @@ -0,0 +1,84 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect } from './playwright-test-fixtures'; + +const count = 10; + +function testFile(name: string) { + return ` + import { test } from '@playwright/test'; + ${Array.from({ length: count }, (_, i) => `test('test${i}', () => console.log('\\n%%${name}-test${i}'));`).join('\n')} + `; +} + +const fileNames = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h']; +const files = Object.fromEntries(fileNames.map(name => [`${name}.spec.ts`, testFile(name)])); +const testsInFile = (name: string) => Array.from({ length: count }, (_, i) => `${name}-test${i}`); +const naturalOrder = fileNames.flatMap(testsInFile); + +test('should shuffle files and keep tests within a file in order', async ({ runInlineTest }) => { + const result = await runInlineTest(files, { workers: 1, shuffle: '42' }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(naturalOrder.length); + expect(result.output).toContain('shuffle seed 42'); + expect(result.outputLines).not.toEqual(naturalOrder); + const fileOrder = [...new Set(result.outputLines.map(line => line.split('-')[0]))]; + expect(result.outputLines).toEqual(fileOrder.flatMap(testsInFile)); + + const result2 = await runInlineTest(files, { workers: 1, shuffle: '42' }); + expect(result2.outputLines).toEqual(result.outputLines); + + const result3 = await runInlineTest(files, { workers: 1, shuffle: '43' }); + expect(result3.outputLines).not.toEqual(result.outputLines); +}); + +test('should shuffle individual tests in fully parallel mode', async ({ runInlineTest }) => { + const result = await runInlineTest(files, { 'workers': 1, 'shuffle': '42', 'fully-parallel': true }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(naturalOrder.length); + const aTests = result.outputLines.filter(line => line.startsWith('a-')); + expect(aTests).not.toEqual(testsInFile('a')); + expect([...aTests].sort()).toEqual([...testsInFile('a')].sort()); +}); + +test('should pick and print a random seed', async ({ runInlineTest }) => { + const result = await runInlineTest(files, { workers: 1, shuffle: true }); + expect(result.exitCode).toBe(0); + const seed = result.output.match(/shuffle seed (\d+)/)![1]; + const result2 = await runInlineTest(files, { workers: 1, shuffle: seed }); + expect(result2.outputLines).toEqual(result.outputLines); +}); + +test('should keep serial suites together in parallel mode', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test.describe.configure({ mode: 'parallel' }); + ${Array.from({ length: count }, (_, i) => `test('test${i}', () => console.log('\\n%%test${i}'));`).join('\n')} + test.describe.serial('serial', () => { + let counter = 0; + ${Array.from({ length: count }, (_, i) => `test('serial${i}', () => { expect(counter++).toBe(${i}); console.log('\\n%%serial${i}'); });`).join('\n')} + }); + `, + }, { workers: 1, shuffle: '7' }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(2 * count); + const lines = result.outputLines; + const serialStart = lines.indexOf('serial0'); + expect(lines.slice(serialStart, serialStart + count)).toEqual(Array.from({ length: count }, (_, i) => `serial${i}`)); + expect(lines.filter(line => line.startsWith('test'))).not.toEqual(Array.from({ length: count }, (_, i) => `test${i}`)); +});