Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions docs/src/test-reporters-js.md
Original file line number Diff line number Diff line change
Expand Up @@ -117,13 +117,24 @@ export default defineConfig({
});
```

You can prefix every line of the output with the index of the worker that produced it:

```js title="playwright.config.ts"
import { defineConfig } from '@playwright/test';

export default defineConfig({
reporter: [['list', { printWorkerIndex: true }]],
});
```

List report supports the following configuration options and environment variables:

| Environment Variable Name | Reporter Config Option| Description | Default
|---|---|---|---|
| `PLAYWRIGHT_LIST_PRINT_STEPS` | `printSteps` | Whether to print each step on its own line. | `false`
| `PLAYWRIGHT_LIST_PRINT_FAILURES_INLINE` | `printFailuresInline` | Whether to print failure details immediately after a failed test instead of at the end. | `false`
| `PLAYWRIGHT_LIST_OMIT_TAGS` | `omitTags` | Whether to omit test tags that are automatically appended to test titles. | `false`
| `PLAYWRIGHT_LIST_PRINT_WORKER_INDEX` | `printWorkerIndex` | Whether to prefix every line of the output with the index of the worker that produced it. | `false`
| `PLAYWRIGHT_FORCE_TTY` | | Whether to produce output suitable for a live terminal. Supports `true`, `1`, `false`, `0`, `[WIDTH]`, and `[WIDTH]x[HEIGHT]`. `[WIDTH]` and `[WIDTH]x[HEIGHT]` specifies the TTY dimensions. | `true` when terminal is in TTY mode, `false` otherwise.
| `FORCE_COLOR` | | Whether to produce colored output. | `true` when terminal is in TTY mode, `false` otherwise.
| `NO_COLOR` | | Whether to disable colored output ([no-color.org](https://no-color.org/)). Any non-empty value disables colors. | unset
Expand Down
62 changes: 42 additions & 20 deletions packages/playwright/src/reporters/list.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ class ListReporter extends TerminalReporter {
private _needNewLine = false;
private _printSteps: boolean;
private _printFailuresInline: boolean;
private _printWorkerIndex: boolean;
private _failureIndex = new Map<TestCase, number>();
private _paused = new Set<TestResult>();

Expand All @@ -47,6 +48,7 @@ class ListReporter extends TerminalReporter {
super({ ...options, omitTags: getAsBooleanFromENV('PLAYWRIGHT_LIST_OMIT_TAGS', options?.omitTags), lastResult: printFailuresInline });
this._printSteps = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_STEPS', options?.printSteps);
this._printFailuresInline = printFailuresInline;
this._printWorkerIndex = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_WORKER_INDEX', options?.printWorkerIndex);
}

override onBegin(suite: Suite) {
Expand All @@ -66,19 +68,19 @@ class ListReporter extends TerminalReporter {
return;
this._maybeWriteNewLine();
this._testRows.set(test, this._lastRow);
const prefix = this._testPrefix(index, '');
const prefix = this._testPrefix(result, index, '');
const line = this.screen.colors.dim(this.formatTestTitle(test)) + this._retrySuffix(result);
this._appendLine(line, prefix);
}

override onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
super.onStdOut(chunk, test, result);
this._dumpToStdio(test, chunk, this.screen.stdout, 'out');
this._dumpToStdio(chunk, this.screen.stdout, result);
}

override onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
super.onStdErr(chunk, test, result);
this._dumpToStdio(test, chunk, this.screen.stderr, 'err');
this._dumpToStdio(chunk, this.screen.stderr, result);
}

private getStepIndex(testIndex: string, result: TestResult, step: TestStep): string {
Expand All @@ -103,11 +105,11 @@ class ListReporter extends TerminalReporter {
if (this._printSteps) {
this._maybeWriteNewLine();
this._stepRows.set(step, this._lastRow);
const prefix = this._testPrefix(this.getStepIndex(testIndex, result, step), '');
const prefix = this._testPrefix(result, this.getStepIndex(testIndex, result, step), '');
const line = test.title + this.screen.colors.dim(stepSuffix(step));
this._appendLine(line, prefix);
} else {
this._updateOrAppendLine(this._testRows, test, this.screen.colors.dim(this.formatTestTitle(test, step)) + this._retrySuffix(result), this._testPrefix(testIndex, ''));
this._updateOrAppendLine(this._testRows, test, this.screen.colors.dim(this.formatTestTitle(test, step)) + this._retrySuffix(result), this._testPrefix(result, testIndex, ''));
}
}

Expand All @@ -118,13 +120,13 @@ class ListReporter extends TerminalReporter {
const testIndex = this._resultIndex.get(result) || '';
if (!this._printSteps) {
if (this.screen.isTTY)
this._updateOrAppendLine(this._testRows, test, this.screen.colors.dim(this.formatTestTitle(test, step.parent)) + this._retrySuffix(result), this._testPrefix(testIndex, ''));
this._updateOrAppendLine(this._testRows, test, this.screen.colors.dim(this.formatTestTitle(test, step.parent)) + this._retrySuffix(result), this._testPrefix(result, testIndex, ''));
return;
}

const index = this.getStepIndex(testIndex, result, step);
const title = this.screen.isTTY ? test.title + this.screen.colors.dim(stepSuffix(step)) : this.formatTestTitle(test, step);
const prefix = this._testPrefix(index, '');
const prefix = this._testPrefix(result, index, '');
let text = '';
if (step.error)
text = this.screen.colors.red(title);
Expand Down Expand Up @@ -162,12 +164,25 @@ class ListReporter extends TerminalReporter {
}
}

private _dumpToStdio(test: TestCase | undefined, chunk: string | Buffer, stream: NodeJS.WriteStream, stdio: 'out' | 'err') {
private _dumpToStdio(chunk: string | Buffer, stream: NodeJS.WriteStream, result: TestResult | undefined) {
if (this.config.quiet)
return;
const text = chunk.toString('utf-8');
let text = chunk.toString('utf-8');
if (result)
text = this._prefixLines(text, this._workerPrefix(result), !this._needNewLine);
this._updateLineCountAndNewLineFlagForOutput(text);
stream.write(chunk);
stream.write(text);
}

private _prefixLines(text: string, prefix: string, atLineStart: boolean) {
if (!prefix)
return text;
const lines = text.split('\n');
return lines.map((line, i) => {
if (!line || (i === 0 && !atLineStart))
return line;
return prefix + line;
}).join('\n');
}

async onTestPaused(test: TestCase, result: TestResult) {
Expand All @@ -180,12 +195,12 @@ class ListReporter extends TerminalReporter {
this._updateTestLine(test, result);
this._maybeWriteNewLine();
if (test.outcome() === 'unexpected') {
const errors = this.formatResultErrors(test, result);
const errors = this._prefixLines(this.formatResultErrors(test, result), this._workerPrefix(result), true);
this.writeLine(errors);
this._updateLineCountAndNewLineFlagForOutput(errors);
markErrorsAsReported(result);
}
this._appendLine(this.screen.colors.yellow(`Paused ${test.outcome() === 'unexpected' ? 'on error' : 'at test end'}. Press Ctrl+C to end.`), this._testPrefix('', ''));
this._appendLine(this.screen.colors.yellow(`Paused ${test.outcome() === 'unexpected' ? 'on error' : 'at test end'}. Press Ctrl+C to end.`), this._testPrefix(result, '', ''));

await new Promise<void>(() => {});
}
Expand All @@ -197,18 +212,18 @@ class ListReporter extends TerminalReporter {
this._updateTestLine(test, result);
const isFailure = result.status !== 'skipped' && result.status !== test.expectedStatus;
if (!wasPaused && this._printFailuresInline && isFailure)
this._printFailure(test);
this._printFailure(test, result);
}

private _printFailure(test: TestCase) {
private _printFailure(test: TestCase, result: TestResult) {
this._maybeWriteNewLine();
// Retries of the same test share one failure index.
let index = this._failureIndex.get(test);
if (index === undefined) {
index = this._failureIndex.size + 1;
this._failureIndex.set(test, index);
}
const message = '\n' + this.formatFailure(test, index) + '\n';
const message = this._prefixLines('\n' + this.formatFailure(test, index) + '\n', this._workerPrefix(result), true);
this._updateLineCountAndNewLineFlagForOutput(message);
this.screen.stdout.write(message);
}
Expand All @@ -227,16 +242,16 @@ class ListReporter extends TerminalReporter {
}

if (result.status === 'skipped') {
prefix = this._testPrefix(index, this.screen.colors.green('-'));
prefix = this._testPrefix(result, index, this.screen.colors.green('-'));
// Do not show duration for skipped.
text = this.screen.colors.cyan(title) + this._retrySuffix(result);
} else {
const statusMark = result.status === 'passed' ? POSITIVE_STATUS_MARK : NEGATIVE_STATUS_MARK;
if (result.status === test.expectedStatus) {
prefix = this._testPrefix(index, this.screen.colors.green(statusMark));
prefix = this._testPrefix(result, index, this.screen.colors.green(statusMark));
text = title;
} else {
prefix = this._testPrefix(index, this.screen.colors.red(statusMark));
prefix = this._testPrefix(result, index, this.screen.colors.red(statusMark));
text = this.screen.colors.red(title);
}
text += this._retrySuffix(result) + this.screen.colors.dim(` (${msToString(result.duration)})`);
Expand Down Expand Up @@ -289,10 +304,17 @@ class ListReporter extends TerminalReporter {
this.screen.stdout.write(`\u001B[${this._lastRow - row}E`);
}

private _testPrefix(index: string, statusMark: string) {
private _workerPrefix(result: TestResult) {
if (!this._printWorkerIndex)
return '';
const workerIndex = result.workerIndex >= 0 ? String(result.workerIndex) : ' ';
return this.screen.colors.dim(`[${workerIndex}]`) + ' ';

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe add some padding?

}

private _testPrefix(result: TestResult, index: string, statusMark: string) {
const statusMarkLength = stripAnsiEscapes(statusMark).length;
const indexLength = Math.ceil(Math.log10(this.totalTestCount + 1));
return ' ' + statusMark + ' '.repeat(3 - statusMarkLength) + this.screen.colors.dim(index.padStart(indexLength) + ' ');
return this._workerPrefix(result) + ' ' + statusMark + ' '.repeat(3 - statusMarkLength) + this.screen.colors.dim(index.padStart(indexLength) + ' ');
}

private _retrySuffix(result: TestResult) {
Expand Down
2 changes: 1 addition & 1 deletion packages/playwright/types/test.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ export * from 'playwright-core';
export type BlobReporterOptions = { outputDir?: string, fileName?: string };
export type DotReporterOptions = { omitTags?: boolean };
export type LineReporterOptions = { omitTags?: boolean };
export type ListReporterOptions = { printSteps?: boolean, printFailuresInline?: boolean, omitTags?: boolean };
export type ListReporterOptions = { printSteps?: boolean, printFailuresInline?: boolean, omitTags?: boolean, printWorkerIndex?: boolean };
export type GitHubReporterOptions = { omitTags?: boolean };
export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean, omitTags?: boolean };
export type JsonReporterOptions = { outputFile?: string };
Expand Down
25 changes: 25 additions & 0 deletions tests/playwright-test/reporter-list.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,31 @@ for (const useIntermediateMergeReport of [false, true] as const) {
expect(result.exitCode).toBe(0);
});

test('print worker index', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { reporter: [['list', { printWorkerIndex: true, printFailuresInline: true }]] };
`,
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('fails', async ({}) => {
expect(1).toBe(2);
});
test('passes', async ({}) => {
console.log('line1\\nline2');
});
`,
}, { workers: 1 }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_FORCE_TTY: '80' });

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

run with 2 workers?

expect(result.exitCode).toBe(1);
const lines = result.output.split('\n').map(l => l.replace(/^#\d+ : /, ''));
expect(lines.some(l => l.startsWith(`[0] ${NEGATIVE_STATUS_MARK} 1 a.test.ts:3:15 › fails`))).toBe(true);
expect(lines).toContain('[0] Error: expect(received).toBe(expected) // Object.is equality');
expect(lines.some(l => /^\[0\] +at .*a.test.ts:4:23/.test(l))).toBe(true);
expect(lines).toContain('[1] line1');
expect(lines).toContain('[1] line2');
expect(lines.some(l => l.startsWith(`[1] ${POSITIVE_STATUS_MARK} 2 a.test.ts:6:15 › passes`))).toBe(true);
});

test('print stdio', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
Expand Down
2 changes: 1 addition & 1 deletion utils/generate_types/overrides-test.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ export * from 'playwright-core';
export type BlobReporterOptions = { outputDir?: string, fileName?: string };
export type DotReporterOptions = { omitTags?: boolean };
export type LineReporterOptions = { omitTags?: boolean };
export type ListReporterOptions = { printSteps?: boolean, printFailuresInline?: boolean, omitTags?: boolean };
export type ListReporterOptions = { printSteps?: boolean, printFailuresInline?: boolean, omitTags?: boolean, printWorkerIndex?: boolean };
export type GitHubReporterOptions = { omitTags?: boolean };
export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean, omitTags?: boolean };
export type JsonReporterOptions = { outputFile?: string };
Expand Down
Loading