From a288458674d74fd5dbbfc0db8d411b157ecad1da Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Thu, 17 Sep 2026 12:27:29 -0700 Subject: [PATCH 1/2] devops(bidi): publish reports to a git branch instead of Azure Azure blob storage is no longer available for report hosting. Publish the per-channel csv reports to the orphan bidi-reports branch instead, where GitHub renders them as tables and the commit history gives day-over-day diffs. Also emit a markdown copy of the report into the job summary, and drop the flakiness-results environment left dead by a02416290. --- .github/workflows/tests_bidi.yml | 60 +++++++++++++++++++++++--------- tests/bidi/csvReporter.ts | 33 +++++++++++++++--- 2 files changed, 72 insertions(+), 21 deletions(-) diff --git a/.github/workflows/tests_bidi.yml b/.github/workflows/tests_bidi.yml index 342207e7ab362..7c964d5dea4a6 100644 --- a/.github/workflows/tests_bidi.yml +++ b/.github/workflows/tests_bidi.yml @@ -24,11 +24,9 @@ env: jobs: test_bidi: name: BiDi - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} runs-on: ubuntu-24.04 permissions: - id-token: write # This is required for OIDC login (azure/login) to succeed - contents: read # This is required for actions/checkout to succeed + contents: write # This is required for actions/checkout and to push to the bidi-reports branch strategy: fail-fast: false matrix: @@ -76,19 +74,49 @@ jobs: with: job_name: ${{ matrix.channel }} - - name: Azure Login - if: ${{ !cancelled() && github.ref == 'refs/heads/main' }} - uses: azure/login@7ddb5af1ef8758cf1353cf3b42f940aee27ba21c # v3.0.2 - with: - client-id: ${{ secrets.AZURE_BLOB_REPORTS_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_BLOB_REPORTS_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_BLOB_REPORTS_SUBSCRIPTION_ID }} + - name: Add report to the job summary + if: ${{ !cancelled() && hashFiles('test-results/report.md') != '' }} + run: cat test-results/report.md >> "$GITHUB_STEP_SUMMARY" - - name: Upload report.csv to Azure - if: ${{ !cancelled() && github.ref == 'refs/heads/main' }} + - name: Publish report.csv to the bidi-reports branch + if: ${{ !cancelled() && github.ref == 'refs/heads/main' && github.repository == 'microsoft/playwright' }} run: | - REPORT_DIR='bidi-reports' - azcopy cp "./test-results/report.csv" "https://mspwblobreport.blob.core.windows.net/\$web/$REPORT_DIR/${{ matrix.channel }}.csv" - echo "Report url: https://mspwblobreport.z1.web.core.windows.net/$REPORT_DIR/${{ matrix.channel }}.csv" + cp test-results/report.csv "$RUNNER_TEMP/report.csv" + REPO_URL="https://x-access-token:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY.git" + # Seed the orphan branch on the first run, or if it ever gets deleted. + if ! git ls-remote --exit-code --heads "$REPO_URL" bidi-reports > /dev/null; then + git init -q "$RUNNER_TEMP/seed" + cd "$RUNNER_TEMP/seed" + echo 'Nightly BiDi test reports, published by .github/workflows/tests_bidi.yml.' > README.md + git add README.md + git commit -q -m 'chore: seed the bidi-reports branch' + git push "$REPO_URL" HEAD:refs/heads/bidi-reports || true + fi + git clone --depth=1 --branch bidi-reports "$REPO_URL" "$RUNNER_TEMP/bidi-reports" + cd "$RUNNER_TEMP/bidi-reports" + # Both matrix channels push to the same branch, so retry on a racing push. + for i in 1 2 3 4 5; do + git fetch --depth=1 origin bidi-reports + git reset --hard FETCH_HEAD + cp "$RUNNER_TEMP/report.csv" '${{ matrix.channel }}.csv' + git add '${{ matrix.channel }}.csv' + if git diff --cached --quiet; then + echo "No changes since the last run." + exit 0 + fi + git commit -m "bidi(${{ matrix.channel }}): results for ${GITHUB_SHA:0:9}" + if git push origin HEAD:bidi-reports; then + echo "Report url: https://github.com/$GITHUB_REPOSITORY/blob/bidi-reports/${{ matrix.channel }}.csv" + echo "Raw report url: https://raw.githubusercontent.com/$GITHUB_REPOSITORY/bidi-reports/${{ matrix.channel }}.csv" + exit 0 + fi + sleep $((RANDOM % 10 + 5)) + done + echo "Failed to push the report after 5 attempts." + exit 1 env: - AZCOPY_AUTO_LOGIN_TYPE: AZCLI + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GIT_AUTHOR_NAME: microsoft-playwright-automation[bot] + GIT_AUTHOR_EMAIL: 203992400+microsoft-playwright-automation[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: microsoft-playwright-automation[bot] + GIT_COMMITTER_EMAIL: 203992400+microsoft-playwright-automation[bot]@users.noreply.github.com diff --git a/tests/bidi/csvReporter.ts b/tests/bidi/csvReporter.ts index b84ca9e81d11f..219bbc758f126 100644 --- a/tests/bidi/csvReporter.ts +++ b/tests/bidi/csvReporter.ts @@ -51,7 +51,7 @@ class CsvReporter implements Reporter { continue; const row = []; const [, , , ...titles] = test.titlePath(); - row.push(csvEscape(`${file.title} › ${titles.join(' › ')}`)); + row.push(`${file.title} › ${titles.join(' › ')}`); row.push(test.expectedStatus); row.push(test.outcome()); if (fixme) { @@ -60,11 +60,11 @@ class CsvReporter implements Reporter { const result = test.results.find(r => r.error); if (result) { const errorMessage = stripAnsi(result.error?.message.replace(/\s+/g, ' ').trim().substring(0, 1024) ?? ''); - row.push(csvEscape(errorMessage)); + row.push(errorMessage); } else { const fail = test.annotations.find(a => a.type === 'fail'); if (fail) - row.push(csvEscape(`Should have failed: ${fail.description}`)); + row.push(`Should have failed: ${fail.description}`); else row.push(''); } @@ -73,11 +73,11 @@ class CsvReporter implements Reporter { } } } - const csv = rows.map(r => r.join(',')).join('\n'); const reportFile = path.resolve(this._options.configDir, this._options.outputFile || 'test-results.csv'); this._pendingWrite = (async () => { await fs.promises.mkdir(path.dirname(reportFile), { recursive: true }); - await fs.promises.writeFile(reportFile, csv); + await fs.promises.writeFile(reportFile, rows.map(r => r.map(csvEscape).join(',')).join('\n')); + await fs.promises.writeFile(reportFile.replace(/\.csv$/, '.md'), markdownTable(rows)); })(); } @@ -96,4 +96,27 @@ function csvEscape(str) { return str; } +// GitHub job summaries are capped at 1MiB, so keep the rendered table bounded. +const maxMarkdownRows = 500; +const maxMarkdownCell = 200; + +function markdownTable(rows: string[][]): string { + const [header, ...body] = rows; + const lines = [ + `### ${body.length} failing tests`, + '', + `| ${header.join(' | ')} |`, + `| ${header.map(() => '---').join(' | ')} |`, + ...body.slice(0, maxMarkdownRows).map(row => `| ${row.map(markdownEscape).join(' | ')} |`), + ]; + if (body.length > maxMarkdownRows) + lines.push('', `_...and ${body.length - maxMarkdownRows} more, see the csv report._`); + return lines.join('\n') + '\n'; +} + +function markdownEscape(str: string): string { + const truncated = str.length > maxMarkdownCell ? str.substring(0, maxMarkdownCell) + '…' : str; + return truncated.replace(/[\\|`<>]/g, c => '\\' + c); +} + export default CsvReporter; From 78548a90f9e747c3f34d8eb83655569521cbe9c5 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Thu, 17 Sep 2026 12:36:07 -0700 Subject: [PATCH 2/2] devops(bidi): publish the reports from a dedicated job Publishing from each matrix job meant two jobs racing to push to one branch, which needed a retry loop and gave the whole test job push rights. Publish once after the matrix instead, from the csv artifacts the test jobs already upload: no race, one commit per run, and the test job keeps contents:read. Also make the reporter's markdown output an explicit option rather than deriving it from the csv path, so local runs no longer write a stray report.md. --- .github/workflows/tests_bidi.yml | 72 +++++++++++++++++--------------- tests/bidi/csvReporter.ts | 27 +++++++----- tests/bidi/playwright.config.ts | 2 +- 3 files changed, 55 insertions(+), 46 deletions(-) diff --git a/.github/workflows/tests_bidi.yml b/.github/workflows/tests_bidi.yml index 7c964d5dea4a6..1e68db76efad9 100644 --- a/.github/workflows/tests_bidi.yml +++ b/.github/workflows/tests_bidi.yml @@ -26,7 +26,7 @@ jobs: name: BiDi runs-on: ubuntu-24.04 permissions: - contents: write # This is required for actions/checkout and to push to the bidi-reports branch + contents: read # This is required for actions/checkout to succeed strategy: fail-fast: false matrix: @@ -78,42 +78,46 @@ jobs: if: ${{ !cancelled() && hashFiles('test-results/report.md') != '' }} run: cat test-results/report.md >> "$GITHUB_STEP_SUMMARY" - - name: Publish report.csv to the bidi-reports branch - if: ${{ !cancelled() && github.ref == 'refs/heads/main' && github.repository == 'microsoft/playwright' }} + publish_reports: + name: Publish reports + needs: test_bidi + if: ${{ !cancelled() && github.ref == 'refs/heads/main' && github.repository == 'microsoft/playwright' }} + runs-on: ubuntu-24.04 + permissions: + contents: write # This is required to push to the bidi-reports branch + steps: + - name: Download csv reports + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: csv-report-* + path: csv-reports + - name: Publish the reports to the bidi-reports branch run: | - cp test-results/report.csv "$RUNNER_TEMP/report.csv" - REPO_URL="https://x-access-token:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY.git" - # Seed the orphan branch on the first run, or if it ever gets deleted. - if ! git ls-remote --exit-code --heads "$REPO_URL" bidi-reports > /dev/null; then - git init -q "$RUNNER_TEMP/seed" - cd "$RUNNER_TEMP/seed" - echo 'Nightly BiDi test reports, published by .github/workflows/tests_bidi.yml.' > README.md - git add README.md - git commit -q -m 'chore: seed the bidi-reports branch' - git push "$REPO_URL" HEAD:refs/heads/bidi-reports || true + git init -q "$RUNNER_TEMP/reports" + cd "$RUNNER_TEMP/reports" + git remote add origin "https://x-access-token:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY.git" + # Continue the branch history, or start it on the very first run. An + # initial commit with no fetched parent is what makes the branch orphan. + git fetch --depth=1 origin bidi-reports && git reset --hard FETCH_HEAD || true + echo 'Nightly BiDi test reports, published by .github/workflows/tests_bidi.yml.' > README.md + channels=() + for dir in "$GITHUB_WORKSPACE"/csv-reports/csv-report-*/; do + channel=${dir%/} + channel=${channel##*/csv-report-} + cp "$dir/report.csv" "$channel.csv" + channels+=("$channel") + done + git add -A + if git diff --cached --quiet; then + echo "No changes since the last run." + exit 0 fi - git clone --depth=1 --branch bidi-reports "$REPO_URL" "$RUNNER_TEMP/bidi-reports" - cd "$RUNNER_TEMP/bidi-reports" - # Both matrix channels push to the same branch, so retry on a racing push. - for i in 1 2 3 4 5; do - git fetch --depth=1 origin bidi-reports - git reset --hard FETCH_HEAD - cp "$RUNNER_TEMP/report.csv" '${{ matrix.channel }}.csv' - git add '${{ matrix.channel }}.csv' - if git diff --cached --quiet; then - echo "No changes since the last run." - exit 0 - fi - git commit -m "bidi(${{ matrix.channel }}): results for ${GITHUB_SHA:0:9}" - if git push origin HEAD:bidi-reports; then - echo "Report url: https://github.com/$GITHUB_REPOSITORY/blob/bidi-reports/${{ matrix.channel }}.csv" - echo "Raw report url: https://raw.githubusercontent.com/$GITHUB_REPOSITORY/bidi-reports/${{ matrix.channel }}.csv" - exit 0 - fi - sleep $((RANDOM % 10 + 5)) + git commit -q -m "bidi: results for ${GITHUB_SHA:0:9}" + git push origin HEAD:bidi-reports + for channel in "${channels[@]}"; do + echo "Report url: https://github.com/$GITHUB_REPOSITORY/blob/bidi-reports/$channel.csv" + echo "Raw report url: https://raw.githubusercontent.com/$GITHUB_REPOSITORY/bidi-reports/$channel.csv" done - echo "Failed to push the report after 5 attempts." - exit 1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GIT_AUTHOR_NAME: microsoft-playwright-automation[bot] diff --git a/tests/bidi/csvReporter.ts b/tests/bidi/csvReporter.ts index 219bbc758f126..d5ef17cdf0131 100644 --- a/tests/bidi/csvReporter.ts +++ b/tests/bidi/csvReporter.ts @@ -24,9 +24,12 @@ import path from 'path'; type ReporterOptions = { outputFile?: string, + markdownFile?: string, configDir: string, }; +const header = ['Test Name', 'Expected Status', 'Status', 'Error Message']; + class CsvReporter implements Reporter { private _suite: Suite; private _options: ReporterOptions; @@ -41,7 +44,7 @@ class CsvReporter implements Reporter { } onEnd(result: FullResult) { - const rows = [['Test Name', 'Expected Status', 'Status', 'Error Message']]; + const rows: string[][] = []; for (const project of this._suite.suites) { for (const file of project.suites) { for (const test of file.allTests()) { @@ -74,10 +77,15 @@ class CsvReporter implements Reporter { } } const reportFile = path.resolve(this._options.configDir, this._options.outputFile || 'test-results.csv'); + const markdownFile = this._options.markdownFile && path.resolve(this._options.configDir, this._options.markdownFile); this._pendingWrite = (async () => { await fs.promises.mkdir(path.dirname(reportFile), { recursive: true }); - await fs.promises.writeFile(reportFile, rows.map(r => r.map(csvEscape).join(',')).join('\n')); - await fs.promises.writeFile(reportFile.replace(/\.csv$/, '.md'), markdownTable(rows)); + const csv = [header, ...rows].map(r => r.map(csvEscape).join(',')).join('\n'); + await fs.promises.writeFile(reportFile, csv); + if (markdownFile) { + await fs.promises.mkdir(path.dirname(markdownFile), { recursive: true }); + await fs.promises.writeFile(markdownFile, markdownTable(rows)); + } })(); } @@ -98,25 +106,22 @@ function csvEscape(str) { // GitHub job summaries are capped at 1MiB, so keep the rendered table bounded. const maxMarkdownRows = 500; -const maxMarkdownCell = 200; function markdownTable(rows: string[][]): string { - const [header, ...body] = rows; const lines = [ - `### ${body.length} failing tests`, + `### ${rows.length} failing tests`, '', `| ${header.join(' | ')} |`, `| ${header.map(() => '---').join(' | ')} |`, - ...body.slice(0, maxMarkdownRows).map(row => `| ${row.map(markdownEscape).join(' | ')} |`), + ...rows.slice(0, maxMarkdownRows).map(row => `| ${row.map(markdownEscape).join(' | ')} |`), ]; - if (body.length > maxMarkdownRows) - lines.push('', `_...and ${body.length - maxMarkdownRows} more, see the csv report._`); + if (rows.length > maxMarkdownRows) + lines.push('', `_...and ${rows.length - maxMarkdownRows} more, see the csv report._`); return lines.join('\n') + '\n'; } function markdownEscape(str: string): string { - const truncated = str.length > maxMarkdownCell ? str.substring(0, maxMarkdownCell) + '…' : str; - return truncated.replace(/[\\|`<>]/g, c => '\\' + c); + return str.replace(/[\\|`<>]/g, c => '\\' + c); } export default CsvReporter; diff --git a/tests/bidi/playwright.config.ts b/tests/bidi/playwright.config.ts index afa43752e13da..83996f2bde224 100644 --- a/tests/bidi/playwright.config.ts +++ b/tests/bidi/playwright.config.ts @@ -46,7 +46,7 @@ const reporters = () => { hasDebugOutput ? ['list'] : ['dot'], ['blob'], ['../config/parquetReporter.ts'], - ['./csvReporter', { outputFile: path.join(outputDir, 'report.csv') }], + ['./csvReporter', { outputFile: path.join(outputDir, 'report.csv'), markdownFile: path.join(outputDir, 'report.md') }], ['./expectationReporter', { rebase: false }], ] : [ ['html', { open: 'on-failure' }],