diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db020085..3a475430 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -317,9 +317,8 @@ jobs: # (docs/testing.md#browsers). When one fails, the job's summary names the # step it failed in, and its artifact holds the trace to replay it step # by step (`npx playwright show-trace`), the report, a video, and the - # server's log. On main the traces are kept always: the deploy publishes - # Chromium's report, trace viewer and all, at /journeys/ - # (docs/deployment.md). + # server's log. On main the traces are kept always, and each job keeps + # its run as a blob, for journeys-report to merge. journeys: name: journeys (${{ matrix.name }}) runs-on: ubuntu-latest @@ -337,6 +336,8 @@ jobs: browsers: chromium-headless-shell firefox webkit env: JOURNEY_TRACE: ${{ (inputs.trace || github.ref == 'refs/heads/main') && 'on' || '' }} + # The blob's name, so the two jobs' blobs sit side by side when merged. + PLAYWRIGHT_BLOB_OUTPUT_NAME: report-${{ matrix.name }}.zip steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -386,12 +387,61 @@ jobs: e2e/journeys/test-results/ e2e/journeys/logs/ retention-days: 7 - - if: always() && github.ref == 'refs/heads/main' && matrix.name == 'chromium' + # A job run again replaces its blob from the attempt before; kept as + # long as the report, so that re-running one job still finds the + # other's. + - if: always() && github.ref == 'refs/heads/main' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: journeys-blob-${{ matrix.name }} + path: e2e/journeys/blob-report/ + retention-days: 7 + overwrite: true + + # On main, the two journeys jobs' runs merged into one report, every + # browser's journeys and their traces in it, even when a journey failed: + # the deploy publishes it at /journeys/ (docs/deployment.md). When both + # jobs passed, both blobs must be there, so a run the deploy may publish + # never has a report missing a browser. ci-ok doesn't need it, as it + # runs on main alone; its failure still fails the run, and so holds + # back release and deploy. + journeys-report: + if: ${{ !cancelled() && github.ref == 'refs/heads/main' }} + needs: journeys + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: .nvmrc + cache: npm + - run: npm ci + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: journeys-blob-* + merge-multiple: true + path: e2e/journeys/blob-reports/ + - working-directory: e2e/journeys + env: + JOURNEYS: ${{ needs.journeys.result }} + run: | + if [ "$JOURNEYS" = success ]; then + test -f blob-reports/report-chromium.zip + test -f blob-reports/report-engines.zip + fi + if compgen -G 'blob-reports/*.zip' > /dev/null; then + npx playwright merge-reports --reporter html blob-reports + fi + - if: hashFiles('e2e/journeys/playwright-report/index.html') != '' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: journeys-report path: e2e/journeys/playwright-report/ retention-days: 7 + overwrite: true # The docs (docs/testing.md#documentation): every link to a file or a # heading in the repository resolves (lychee, offline: the web's diff --git a/docs/deployment.md b/docs/deployment.md index f5138ea4..e8ef4a24 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -120,7 +120,7 @@ data it reads: | `/ns/vocab/*.ttl`, `/ns/shapes//v.ttl` | [`ns/`](../ns/) | The vocabulary and the shapes ([vocab.md](vocab.md), [shapes.md](shapes.md)). | | `/decks/index.ttl`, `/decks//v.ttl` | [`decks/`](../decks/) | The deck library ([deck-library.md](deck-library.md)). | | `/vendor/…` | [`packages/vocab/vendor/`](../packages/vocab/vendor/) | The vendored DCAT-AP and SKOS shapes. | -| `/journeys/` | the CI run's `journeys-report` artifact | The user journeys' report ([testing.md](testing.md#user-journeys)), with each journey's trace. The report of the CI run on the release commit; not there after a deploy by hand. | +| `/journeys/` | the CI run's `journeys-report` artifact | The user journeys' report ([testing.md](testing.md#user-journeys)), every browser's in one, with each journey's trace. The report of the CI run on the release commit; not there after a deploy by hand. | Vite copies `ns/`, `decks/` and `vendor/` as they are (`turtleDirectoryPlugin` in diff --git a/docs/testing.md b/docs/testing.md index c9816c60..92fccf7f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -402,8 +402,10 @@ error, and the browser it failed in. The job's - `logs/css.log`: the server's whole log. **The latest report is online** at , -published with each deploy: the Chromium job's report of the CI run on -the release commit the site runs (none after a deploy by hand). It shows every +published with each deploy: the report of the CI run on the release +commit the site runs (none after a deploy by hand), every browser's +journeys in one, as the `journeys-report` job merges them from both +journeys jobs (Playwright's blob reports). It shows every journey's steps and screenshots, and its **Trace** link opens the step-by-step timeline in the browser (the trace viewer is part of the report and needs it served, as it is there, not opened as a file). On `main` the @@ -516,11 +518,12 @@ jobs, side by side: | `pod ()` | the end-to-end tests, one job per blocking server (above) | | `contract ()` | that each advisory server starts and meets the contract | | `journeys ()` | the user journeys: every one in Chromium, and those that differ between engines in Firefox, WebKit and on a phone ([Browsers](#browsers)) | +| `journeys-report` | nothing: on `main`, after both `journeys` jobs, even when one failed, it merges their runs into one report, the `journeys-report` artifact the deploy publishes at `/journeys/` ([User journeys](#user-journeys)). When both passed, it fails without both their reports; its failure fails the run, so it holds back the release and the deploy, but not `ci-ok` | | `docs` | the docs' links, every file's license, and `publiccode.yml` ([Documentation](#documentation)); their style and spelling are `build`'s | | `workflows` | the workflows and local actions, by actionlint and zizmor (`npm run lint:workflows`, [The workflows](#the-workflows)) | | `commits` | on a pull request, that each commit it adds follows Conventional Commits, by commitlint ([CONTRIBUTING.md](../CONTRIBUTING.md#commit-messages)), and that its body or a commit closes its issue, or the body says it is one step of it (`npm run check:closes`; not for Renovate's); in the merge queue or on a push to `main`, nothing | | `caches` | nothing: on `main`, after `build`, it deletes `main`'s older Turbo caches, the newest holding what they do. The one job whose token can write (`actions: write`, to delete caches); it runs none of the repository's code, and its failure fails neither the run nor `ci-ok` | -| `ci-ok` | that every job above passed, `caches` aside | +| `ci-ok` | that every job above passed, `journeys-report` and `caches` aside | **`ci-ok` is the one check a merge to `main` waits for**: the ruleset on `main` (a setting of the repository, Settings › Rules) is to require diff --git a/e2e/journeys/pages/SomethingWentWrong.ts b/e2e/journeys/pages/SomethingWentWrong.ts index 371e87b2..e871be41 100644 --- a/e2e/journeys/pages/SomethingWentWrong.ts +++ b/e2e/journeys/pages/SomethingWentWrong.ts @@ -86,7 +86,8 @@ export class SomethingWentWrong extends Screen { async goToStart(): Promise { await this.intent("Go to the start", async () => { await this.page.getByRole("button", { name: this.t("crash.start"), exact: true }).click(); - await expect(this.page).toHaveURL(/#\/$/); + // Not the URL: the start, #/, is replaced at once by where it leads. + await expect(this.heading).toBeHidden(); }); } } diff --git a/e2e/journeys/playwright.config.ts b/e2e/journeys/playwright.config.ts index f893109c..43df2645 100644 --- a/e2e/journeys/playwright.config.ts +++ b/e2e/journeys/playwright.config.ts @@ -46,7 +46,10 @@ export default defineConfig({ ["junit", { outputFile: "test-results/junit.xml" }], // For the CI job's summary (harness/summary.ts). ["json", { outputFile: "test-results/results.json" }], - ...(CI ? ([["github"]] as const) : []), + // In CI, also each job's run as a blob, to merge into one report of + // every browser's journeys: the one published at /journeys/ + // (docs/testing.md#user-journeys). + ...(CI ? ([["github"], ["blob"]] as const) : []), ], use: { baseURL: APP,