From 23423fede77e607bd0052c5f503502cbf296f7cb Mon Sep 17 00:00:00 2001 From: Anton Wiklund Date: Mon, 19 Oct 2026 13:46:41 +0200 Subject: [PATCH 1/2] ci(journeys): publish every browser's journeys in one report The report at /journeys/ held only the Chromium job's journeys, so it said nothing of Firefox, WebKit or the phone, where the login's redirects, a guest's storage and the session's restore differ. Their report was a CI artifact alone, gone after a week. Each journeys job now also writes its run as a blob, and on main a journeys-report job merges both with `playwright merge-reports` into the journeys-report artifact the deploy already publishes. The blobs are kept as long as the report, so re-running one failed job a few days later still finds the other's; when both jobs passed, the merge fails without both blobs, so a run the deploy may publish never has a report missing a browser. Closes: #192 --- .github/workflows/ci.yml | 58 ++++++++++++++++++++++++++++--- docs/deployment.md | 2 +- docs/testing.md | 9 +++-- e2e/journeys/playwright.config.ts | 5 ++- 4 files changed, 65 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db020085..3a475430 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -317,9 +317,8 @@ jobs: # (docs/testing.md#browsers). When one fails, the job's summary names the # step it failed in, and its artifact holds the trace to replay it step # by step (`npx playwright show-trace`), the report, a video, and the - # server's log. On main the traces are kept always: the deploy publishes - # Chromium's report, trace viewer and all, at /journeys/ - # (docs/deployment.md). + # server's log. On main the traces are kept always, and each job keeps + # its run as a blob, for journeys-report to merge. journeys: name: journeys (${{ matrix.name }}) runs-on: ubuntu-latest @@ -337,6 +336,8 @@ jobs: browsers: chromium-headless-shell firefox webkit env: JOURNEY_TRACE: ${{ (inputs.trace || github.ref == 'refs/heads/main') && 'on' || '' }} + # The blob's name, so the two jobs' blobs sit side by side when merged. + PLAYWRIGHT_BLOB_OUTPUT_NAME: report-${{ matrix.name }}.zip steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -386,12 +387,61 @@ jobs: e2e/journeys/test-results/ e2e/journeys/logs/ retention-days: 7 - - if: always() && github.ref == 'refs/heads/main' && matrix.name == 'chromium' + # A job run again replaces its blob from the attempt before; kept as + # long as the report, so that re-running one job still finds the + # other's. + - if: always() && github.ref == 'refs/heads/main' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: journeys-blob-${{ matrix.name }} + path: e2e/journeys/blob-report/ + retention-days: 7 + overwrite: true + + # On main, the two journeys jobs' runs merged into one report, every + # browser's journeys and their traces in it, even when a journey failed: + # the deploy publishes it at /journeys/ (docs/deployment.md). When both + # jobs passed, both blobs must be there, so a run the deploy may publish + # never has a report missing a browser. ci-ok doesn't need it, as it + # runs on main alone; its failure still fails the run, and so holds + # back release and deploy. + journeys-report: + if: ${{ !cancelled() && github.ref == 'refs/heads/main' }} + needs: journeys + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: .nvmrc + cache: npm + - run: npm ci + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: journeys-blob-* + merge-multiple: true + path: e2e/journeys/blob-reports/ + - working-directory: e2e/journeys + env: + JOURNEYS: ${{ needs.journeys.result }} + run: | + if [ "$JOURNEYS" = success ]; then + test -f blob-reports/report-chromium.zip + test -f blob-reports/report-engines.zip + fi + if compgen -G 'blob-reports/*.zip' > /dev/null; then + npx playwright merge-reports --reporter html blob-reports + fi + - if: hashFiles('e2e/journeys/playwright-report/index.html') != '' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: journeys-report path: e2e/journeys/playwright-report/ retention-days: 7 + overwrite: true # The docs (docs/testing.md#documentation): every link to a file or a # heading in the repository resolves (lychee, offline: the web's diff --git a/docs/deployment.md b/docs/deployment.md index f5138ea4..e8ef4a24 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -120,7 +120,7 @@ data it reads: | `/ns/vocab/*.ttl`, `/ns/shapes//v.ttl` | [`ns/`](../ns/) | The vocabulary and the shapes ([vocab.md](vocab.md), [shapes.md](shapes.md)). | | `/decks/index.ttl`, `/decks//v.ttl` | [`decks/`](../decks/) | The deck library ([deck-library.md](deck-library.md)). | | `/vendor/…` | [`packages/vocab/vendor/`](../packages/vocab/vendor/) | The vendored DCAT-AP and SKOS shapes. | -| `/journeys/` | the CI run's `journeys-report` artifact | The user journeys' report ([testing.md](testing.md#user-journeys)), with each journey's trace. The report of the CI run on the release commit; not there after a deploy by hand. | +| `/journeys/` | the CI run's `journeys-report` artifact | The user journeys' report ([testing.md](testing.md#user-journeys)), every browser's in one, with each journey's trace. The report of the CI run on the release commit; not there after a deploy by hand. | Vite copies `ns/`, `decks/` and `vendor/` as they are (`turtleDirectoryPlugin` in diff --git a/docs/testing.md b/docs/testing.md index c9816c60..92fccf7f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -402,8 +402,10 @@ error, and the browser it failed in. The job's - `logs/css.log`: the server's whole log. **The latest report is online** at , -published with each deploy: the Chromium job's report of the CI run on -the release commit the site runs (none after a deploy by hand). It shows every +published with each deploy: the report of the CI run on the release +commit the site runs (none after a deploy by hand), every browser's +journeys in one, as the `journeys-report` job merges them from both +journeys jobs (Playwright's blob reports). It shows every journey's steps and screenshots, and its **Trace** link opens the step-by-step timeline in the browser (the trace viewer is part of the report and needs it served, as it is there, not opened as a file). On `main` the @@ -516,11 +518,12 @@ jobs, side by side: | `pod ()` | the end-to-end tests, one job per blocking server (above) | | `contract ()` | that each advisory server starts and meets the contract | | `journeys ()` | the user journeys: every one in Chromium, and those that differ between engines in Firefox, WebKit and on a phone ([Browsers](#browsers)) | +| `journeys-report` | nothing: on `main`, after both `journeys` jobs, even when one failed, it merges their runs into one report, the `journeys-report` artifact the deploy publishes at `/journeys/` ([User journeys](#user-journeys)). When both passed, it fails without both their reports; its failure fails the run, so it holds back the release and the deploy, but not `ci-ok` | | `docs` | the docs' links, every file's license, and `publiccode.yml` ([Documentation](#documentation)); their style and spelling are `build`'s | | `workflows` | the workflows and local actions, by actionlint and zizmor (`npm run lint:workflows`, [The workflows](#the-workflows)) | | `commits` | on a pull request, that each commit it adds follows Conventional Commits, by commitlint ([CONTRIBUTING.md](../CONTRIBUTING.md#commit-messages)), and that its body or a commit closes its issue, or the body says it is one step of it (`npm run check:closes`; not for Renovate's); in the merge queue or on a push to `main`, nothing | | `caches` | nothing: on `main`, after `build`, it deletes `main`'s older Turbo caches, the newest holding what they do. The one job whose token can write (`actions: write`, to delete caches); it runs none of the repository's code, and its failure fails neither the run nor `ci-ok` | -| `ci-ok` | that every job above passed, `caches` aside | +| `ci-ok` | that every job above passed, `journeys-report` and `caches` aside | **`ci-ok` is the one check a merge to `main` waits for**: the ruleset on `main` (a setting of the repository, Settings › Rules) is to require diff --git a/e2e/journeys/playwright.config.ts b/e2e/journeys/playwright.config.ts index f893109c..43df2645 100644 --- a/e2e/journeys/playwright.config.ts +++ b/e2e/journeys/playwright.config.ts @@ -46,7 +46,10 @@ export default defineConfig({ ["junit", { outputFile: "test-results/junit.xml" }], // For the CI job's summary (harness/summary.ts). ["json", { outputFile: "test-results/results.json" }], - ...(CI ? ([["github"]] as const) : []), + // In CI, also each job's run as a blob, to merge into one report of + // every browser's journeys: the one published at /journeys/ + // (docs/testing.md#user-journeys). + ...(CI ? ([["github"], ["blob"]] as const) : []), ], use: { baseURL: APP, From 0b2e07efdf8c34c1e0a8bea328b382c45ffc4552 Mon Sep 17 00:00:00 2001 From: Anton Wiklund Date: Mon, 19 Oct 2026 14:12:22 +0200 Subject: [PATCH 2/2] test(journeys): wait for the crash screen to go, not for #/ "Go to the start" sets the address to #/, which the app at once replaces with the deck list of the one instance it knows. Waiting for the URL to end in #/ passed only when a poll caught that moment: on this branch's CI it came too late, and the journey failed though the app was right. The page object now waits for the crash screen to go, and the journey's next step checks the deck list it leads to. Refs: #147, #192 --- e2e/journeys/pages/SomethingWentWrong.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e/journeys/pages/SomethingWentWrong.ts b/e2e/journeys/pages/SomethingWentWrong.ts index 371e87b2..e871be41 100644 --- a/e2e/journeys/pages/SomethingWentWrong.ts +++ b/e2e/journeys/pages/SomethingWentWrong.ts @@ -86,7 +86,8 @@ export class SomethingWentWrong extends Screen { async goToStart(): Promise { await this.intent("Go to the start", async () => { await this.page.getByRole("button", { name: this.t("crash.start"), exact: true }).click(); - await expect(this.page).toHaveURL(/#\/$/); + // Not the URL: the start, #/, is replaced at once by where it leads. + await expect(this.heading).toBeHidden(); }); } }