devops(bidi): publish BiDi reports to a git branch instead of Azure

.github/workflows/tests_bidi.yml

@@ -24,10 +24,8 @@ env:
 jobs:
   test_bidi:
     name: BiDi
-    environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }}
     runs-on: ubuntu-24.04
     permissions:
-      id-token: write   # This is required for OIDC login (azure/login) to succeed
       contents: read    # This is required for actions/checkout to succeed
     strategy:
       fail-fast: false
@@ -76,19 +74,53 @@ jobs:
       with:
         job_name: ${{ matrix.channel }}
 
-    - name: Azure Login
-      if: ${{ !cancelled() && github.ref == 'refs/heads/main' }}
-      uses: azure/login@7ddb5af1ef8758cf1353cf3b42f940aee27ba21c # v3.0.2
-      with:
-        client-id: ${{ secrets.AZURE_BLOB_REPORTS_CLIENT_ID }}
-        tenant-id: ${{ secrets.AZURE_BLOB_REPORTS_TENANT_ID }}
-        subscription-id: ${{ secrets.AZURE_BLOB_REPORTS_SUBSCRIPTION_ID }}
+    - name: Add report to the job summary
+      if: ${{ !cancelled() && hashFiles('test-results/report.md') != '' }}
+      run: cat test-results/report.md >> "$GITHUB_STEP_SUMMARY"
 
-    - name: Upload report.csv to Azure
-      if: ${{ !cancelled() && github.ref == 'refs/heads/main' }}
+  publish_reports:
+    name: Publish reports
+    needs: test_bidi
+    if: ${{ !cancelled() && github.ref == 'refs/heads/main' && github.repository == 'microsoft/playwright' }}
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: write   # This is required to push to the bidi-reports branch
+    steps:
+    - name: Download csv reports
+      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+      with:
+        pattern: csv-report-*
+        path: csv-reports
+    - name: Publish the reports to the bidi-reports branch
       run: |
-        REPORT_DIR='bidi-reports'
-        azcopy cp "./test-results/report.csv" "https://mspwblobreport.blob.core.windows.net/\$web/$REPORT_DIR/${{ matrix.channel }}.csv"
-        echo "Report url: https://mspwblobreport.z1.web.core.windows.net/$REPORT_DIR/${{ matrix.channel }}.csv"
+        git init -q "$RUNNER_TEMP/reports"
+        cd "$RUNNER_TEMP/reports"
+        git remote add origin "https://x-access-token:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY.git"
+        # Continue the branch history, or start it on the very first run. An
+        # initial commit with no fetched parent is what makes the branch orphan.
+        git fetch --depth=1 origin bidi-reports && git reset --hard FETCH_HEAD || true
+        echo 'Nightly BiDi test reports, published by .github/workflows/tests_bidi.yml.' > README.md
+        channels=()
+        for dir in "$GITHUB_WORKSPACE"/csv-reports/csv-report-*/; do
+          channel=${dir%/}
+          channel=${channel##*/csv-report-}
+          cp "$dir/report.csv" "$channel.csv"
+          channels+=("$channel")
+        done
+        git add -A
+        if git diff --cached --quiet; then
+          echo "No changes since the last run."
+          exit 0
+        fi
+        git commit -q -m "bidi: results for ${GITHUB_SHA:0:9}"
+        git push origin HEAD:bidi-reports
+        for channel in "${channels[@]}"; do
+          echo "Report url: https://github.com/$GITHUB_REPOSITORY/blob/bidi-reports/$channel.csv"
+          echo "Raw report url: https://raw.githubusercontent.com/$GITHUB_REPOSITORY/bidi-reports/$channel.csv"
+        done
       env:
-        AZCOPY_AUTO_LOGIN_TYPE: AZCLI
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        GIT_AUTHOR_NAME: microsoft-playwright-automation[bot]
+        GIT_AUTHOR_EMAIL: 203992400+microsoft-playwright-automation[bot]@users.noreply.github.com
+        GIT_COMMITTER_NAME: microsoft-playwright-automation[bot]
+        GIT_COMMITTER_EMAIL: 203992400+microsoft-playwright-automation[bot]@users.noreply.github.com

tests/bidi/csvReporter.ts

@@ -24,9 +24,12 @@ import path from 'path';
 
 type ReporterOptions = {
   outputFile?: string,
+  markdownFile?: string,
   configDir: string,
 };
 
+const header = ['Test Name', 'Expected Status', 'Status', 'Error Message'];
+
 class CsvReporter implements Reporter {
   private _suite: Suite;
   private _options: ReporterOptions;
@@ -41,7 +44,7 @@ class CsvReporter implements Reporter {
   }
 
   onEnd(result: FullResult) {
-    const rows = [['Test Name', 'Expected Status', 'Status', 'Error Message']];
+    const rows: string[][] = [];
     for (const project of this._suite.suites) {
       for (const file of project.suites) {
         for (const test of file.allTests()) {
@@ -51,7 +54,7 @@ class CsvReporter implements Reporter {
             continue;
           const row = [];
           const [, , , ...titles] = test.titlePath();
-          row.push(csvEscape(`${file.title} › ${titles.join(' › ')}`));
+          row.push(`${file.title} › ${titles.join(' › ')}`);
           row.push(test.expectedStatus);
           row.push(test.outcome());
           if (fixme) {
@@ -60,11 +63,11 @@ class CsvReporter implements Reporter {
             const result = test.results.find(r => r.error);
             if (result) {
               const errorMessage = stripAnsi(result.error?.message.replace(/\s+/g, ' ').trim().substring(0, 1024) ?? '');
-              row.push(csvEscape(errorMessage));
+              row.push(errorMessage);
             } else {
               const fail = test.annotations.find(a => a.type === 'fail');
               if (fail)
-                row.push(csvEscape(`Should have failed: ${fail.description}`));
+                row.push(`Should have failed: ${fail.description}`);
               else
                 row.push('');
             }
@@ -73,11 +76,16 @@ class CsvReporter implements Reporter {
         }
       }
     }
-    const csv = rows.map(r => r.join(',')).join('\n');
     const reportFile = path.resolve(this._options.configDir, this._options.outputFile || 'test-results.csv');
+    const markdownFile = this._options.markdownFile && path.resolve(this._options.configDir, this._options.markdownFile);
     this._pendingWrite = (async () => {
       await fs.promises.mkdir(path.dirname(reportFile), { recursive: true });
+      const csv = [header, ...rows].map(r => r.map(csvEscape).join(',')).join('\n');
       await fs.promises.writeFile(reportFile, csv);
+      if (markdownFile) {
+        await fs.promises.mkdir(path.dirname(markdownFile), { recursive: true });
+        await fs.promises.writeFile(markdownFile, markdownTable(rows));
+      }
     })();
   }
 
@@ -96,4 +104,24 @@ function csvEscape(str) {
   return str;
 }
 
+// GitHub job summaries are capped at 1MiB, so keep the rendered table bounded.
+const maxMarkdownRows = 500;
+
+function markdownTable(rows: string[][]): string {
+  const lines = [
+    `### ${rows.length} failing tests`,
+    '',
+    `| ${header.join(' | ')} |`,
+    `| ${header.map(() => '---').join(' | ')} |`,
+    ...rows.slice(0, maxMarkdownRows).map(row => `| ${row.map(markdownEscape).join(' | ')} |`),
+  ];
+  if (rows.length > maxMarkdownRows)
+    lines.push('', `_...and ${rows.length - maxMarkdownRows} more, see the csv report._`);
+  return lines.join('\n') + '\n';
+}
+
+function markdownEscape(str: string): string {
+  return str.replace(/[\\|`<>]/g, c => '\\' + c);
+}
+
 export default CsvReporter;

tests/bidi/playwright.config.ts

@@ -46,7 +46,7 @@ const reporters = () => {
     hasDebugOutput ? ['list'] : ['dot'],
     ['blob'],
     ['../config/parquetReporter.ts'],
-    ['./csvReporter', { outputFile: path.join(outputDir, 'report.csv') }],
+    ['./csvReporter', { outputFile: path.join(outputDir, 'report.csv'), markdownFile: path.join(outputDir, 'report.md') }],
     ['./expectationReporter', { rebase: false }],
   ] : [
     ['html', { open: 'on-failure' }],