diff --git a/.github/workflows/benchmark-round.yml b/.github/workflows/benchmark-round.yml index 7adb589..c0ebe1b 100644 --- a/.github/workflows/benchmark-round.yml +++ b/.github/workflows/benchmark-round.yml @@ -8,6 +8,10 @@ on: type: choice default: "" options: ["", smoke, standard, stress, P, M, G, GG] + quality: + description: Run the untimed quality pass (spec 009); with a suite, only in cell nf50-cs5 + type: boolean + default: false number_of_files: description: Number of posts per build (-nf) type: string @@ -102,11 +106,14 @@ jobs: NF: ${{ matrix.number_of_files }} CS: ${{ matrix.content_size }} MR: ${{ matrix.min_runs }} + QUALITY: ${{ inputs.quality }} run: | + Q="" + if [ "$QUALITY" = "true" ]; then Q="--quality"; fi if [ -n "$SUITE" ]; then - ./ssgberk --test "$TEST" --suite "$SUITE" --cell "$CELL" + ./ssgberk --test "$TEST" --suite "$SUITE" --cell "$CELL" $Q else - ./ssgberk --test "$TEST" -nf "$NF" -cs "$CS" -mr "$MR" + ./ssgberk --test "$TEST" -nf "$NF" -cs "$CS" -mr "$MR" $Q fi - name: Record runner environment if: always() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c37f39..e72d87d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,3 +32,47 @@ jobs: - run: | R=$(ls -td results/*/ | head -1) jq -e '.rawData.datarate.hugo[0].mean > 0' "$R/results.json" + + quality-image: + needs: toolset + strategy: + matrix: + runner: [ubuntu-24.04, ubuntu-24.04-arm] + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + - run: docker build -t ssgberk/quality quality + - name: Collector on the fixture site + run: | + docker run --rm --network none \ + -v "$PWD/tests/fixtures/quality/site:/fixture:ro" -v "$PWD/quality/test:/test:ro" \ + ssgberk/quality sh -c 'node /quality/run.mjs --site /fixture --pages "$(cat /test/pages.json)" --out /tmp/out \ + && node /test/check-shape.mjs /tmp/out/raw.json /test/expected-shape.json \ + && node /test/bad-url.mjs' + + quality: + needs: quality-image + strategy: + matrix: + runner: [ubuntu-24.04, ubuntu-24.04-arm] + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - run: ./ssgberk --test mdbook -nf 50 -cs 5 -mr 1 --quality + - run: | + R=$(ls -td results/*/ | head -1) + jq -e '.rawData.datarate.mdbook[0].mean > 0' "$R/results.json" + jq -e '.quality.mdbook.status == "ok"' "$R/results.json" + for s in lighthouse seo html a11y links weight js extra; do + jq -e --arg s "$s" '.quality.mdbook[$s].status == "ok"' "$R/results.json" + done + jq -e '[.quality.mdbook.lighthouse.mobile[], .quality.mdbook.lighthouse.desktop[]] + | length == 6 and all(.status == "ok")' "$R/results.json" + jq -e '[.quality.mdbook.lighthouse.mobile[], .quality.mdbook.lighthouse.desktop[] + | .scores[] | .median | select(. < 0 or . > 1)] | length == 0' "$R/results.json" + jq -e '.environment.quality.lighthouse and .environment.quality.chromium' "$R/results.json" + test -s "$R/quality-summary.csv" + grep -q '^## Qualidade' "$R/summary.md" diff --git a/README.md b/README.md index 04b5cfc..d42a3fa 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,22 @@ Both summary files are regenerated by `--parse`. After a `--suite` run, `results//` also holds `suite.json` (cells, generator order, outcome counts), `suite-summary.csv`/`suite-summary.md` (all cells in one table) and `scaling.csv` (scaling exponent per generator). Each cell has its own `results.json` and summaries under `/nf-cs/`. +## Quality pass + +`--quality` audits the output of the last timed build of each generator, without timing it (spec `docs/specs/009-output-quality`): + + $ ./ssgberk --test mdbook -nf 50 -cs 5 --quality + +With `--suite`, the pass runs only in the cell `nf50-cs5` (suite P, cell 0). The toolset's tests need Python 3.12 (the toolset image and CI use it). It builds the `ssgberk/quality` image from `quality/` once. That image pins Lighthouse, Playwright's Chromium, axe-core, html-validate and lychee. The image runs with no network and with the site copied in, and it audits the index, the first post and `404.html`: +- Lighthouse, mobile and desktop presets, median of 3 runs; +- SEO signals (title, lang, description, viewport, canonical, Open Graph, Twitter card, JSON-LD, robots, hreflang, headings, `alt`, plus `robots.txt`, `sitemap.xml`, feed and favicon); +- HTML validity, axe-core violations and broken internal links; +- weight on disk, gzip and brotli; +- JS bytes, and whether the post works without JS; +- files outside the reference page set. + +Results go to `results//quality//quality.json`, to `quality` in `results.json`, to the "Qualidade" section of `summary.md` and to `quality-summary.csv`. They have no combined score and no ranking. A failing check is a result and never fails the run. For generators that do not use the SF 005 index layout yet, the audited post is the first file matching `output_glob` (`postSource: "glob"` in `quality.json`). + ## Content size `-cs` is the content size per post, in KB: diff --git a/docs/specs/009-output-quality/plan.md b/docs/specs/009-output-quality/plan.md new file mode 100644 index 0000000..6ac67f9 --- /dev/null +++ b/docs/specs/009-output-quality/plan.md @@ -0,0 +1,194 @@ +# 009 Output Quality: Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. The steps are in `tasks.md` and use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** an untimed quality pass that audits the output of one build per generator (Lighthouse, SEO signals, HTML validity, accessibility, links, weight, JS, extra output) and writes `quality.json`, `results.json` `quality`, a "Qualidade" section in `summary.md` and `quality-summary.csv`. + +**Architecture:** +1. **Export.** `DockerHelper.benchmark` copies the build output out of the generator container before removing it. +2. **Collection.** A pinned BT image (`quality/`) runs a Node collector (`quality/run.mjs`) with no network. The collector serves the site, runs Lighthouse, axe-core, html-validate and lychee, and measures file sizes. It writes raw data only (`raw.json`). +3. **Reduction.** Pure Python in `toolset/quality/` reduces `raw.json` into `quality.json` and computes the SEO signals, the pages, the JS check and the extra output from the exported files. Most of the logic is therefore testable with pytest and no Docker. + +**Tech stack:** +- **Toolset:** Python 3.12 stdlib (`html.parser`, `tarfile`, `xml.etree`, `statistics`, `fnmatch`) and docker-py 7.1.0, as in the existing toolset. +- **Collector:** Node 24 with the `lighthouse`, `chrome-launcher`, `playwright` (Chromium only), `axe-core` and `html-validate` npm packages, plus the `lychee` binary. + +**Spec:** `docs/specs/009-output-quality/spec.md`. Read it before any task; requirement ids (R-n) below refer to it. + +## Global Constraints + +- The pass never changes a timing, a status, `succeeded`/`failed` or a ranking (R-2, goal 1). +- The quality container runs with `network_mode="none"` and has no volumes or mounts. The site goes in with `put_archive`; `raw.json` comes out with `get_archive` (R-4, goal 2). +- Every tool version is pinned exactly: + - npm: exact versions in `quality/package.json` and a committed `quality/package-lock.json`; + - Node, lychee: `ARG` with a full version in `quality/Dockerfile`; + - Chromium: whatever the pinned `playwright` version installs (R-4, R-6). +- With `--suite`, the pass runs only in the cell `numberOfFiles=50, contentSize=5` (R-1). +- A failing check is a section with `status: "error"` and `error: ""`, never an exception out of the pass (R-16). +- Three audited URLs: `/`, the first `a[href]` inside `.post-item` on the index, and `/404.html` (R-7). +- Lighthouse: mobile and desktop presets, 3 runs per URL and preset, median and range per value, failed audit ids of the median run (R-8). +- No combined score and no ranking in any output (R-18). +- Toolset code is stdlib plus `docker`, and passes `ruff check toolset tests`. Tests run with `pytest -q` and need no Docker. +- Commits: author and committer `Matheus Breguêz `, GPG-signed (key `B6FA8458D5176E83`), trailer `Co-Authored-By: Claude Opus 5.5 (1M context) `. Never use `--no-gpg-sign` or `--author`. + +## Review Focus + +1. **The post link is relative, has a query or fragment, or ends without `/`.** Examples: `post/a/`, `./post/a/index.html`, `/post/a?x=1#t`, `/post/a`. `resolve_pages` must still find the file on disk. Pinned in Task 1 (`test_resolve_pages_variants`). +2. **A generator writes non-UTF-8 bytes or a malformed HTML page.** SEO extraction and `works_without_js` must not raise; they read with `errors="replace"` and `html.parser` tolerates bad markup. Pinned in Task 2 (`test_page_signals_malformed_html`). +3. **The collector fails for one page or one check.** For example, Lighthouse times out on the 404 page, or lychee is missing. The other pages and sections must still be reported, and only the affected entry is an `error`. Pinned in Task 3 (`test_build_quality_partial_errors`). +4. **The exported archive is missing,** because the export failed or the noise re-run timed out. The pass records `quality..status = "error"` and the build result is unchanged. Pinned in Task 6 (`test_quality_pass_missing_archive_keeps_result`). +5. **`--parse` on a run that had a quality pass.** `quality` must survive the re-parse, because raw.txt cannot rebuild it. Pinned in Task 6 (`test_reparse_keeps_quality`). + +## File structure + +| File | Responsibility | Task | +|---|---|---| +| `toolset/quality/__init__.py` | package marker | 1 | +| `toolset/quality/site.py` | generator config (`output_folder`, `output_glob`), archive extraction, page resolution (R-7), URL to file, extra output (R-15), `works_without_js` (R-14) | 1 | +| `toolset/quality/seo.py` | per-page and site-wide SEO signals (R-9) | 2 | +| `toolset/quality/reduce.py` | `raw.json` to `quality.json`: Lighthouse, axe, html-validate, lychee, weight, JS reductions and section assembly (R-8, R-10..R-14, R-16) | 3 | +| `quality/Dockerfile`, `quality/package.json`, `quality/package-lock.json`, `quality/run.mjs` | the pinned collector image (R-4, R-5) | 4 | +| `quality/test/fixture/**`, `quality/test/pages.json`, `quality/test/expected-shape.json`, `quality/test/check-shape.mjs` | collector test inside the image | 4 | +| `toolset/utils/docker_helper.py` | output export in `benchmark`, `build_quality_image`, `run_quality` | 5 | +| `toolset/quality/runner.py` | `eligible` (R-1), `archive_path`, `run_pass` (R-3, R-16) | 6 | +| `toolset/run-tests.py`, `toolset/utils/benchmark_config.py` | `--quality` flag | 6 | +| `toolset/benchmark/benchmarker.py` | export arguments and the pass hook (R-2) | 6 | +| `toolset/utils/results.py` | `add_quality`, `quality` and `environment.quality` in `results.json`, kept by `--parse` (R-6, R-17) | 6 | +| `toolset/utils/summary.py` | `quality_rows`, `quality_csv`, `quality_markdown`; the section in `to_markdown` (R-18, R-19) | 7 | +| `.github/workflows/ci.yml`, `.github/workflows/benchmark-round.yml`, `README.md` | CI jobs, round input, docs (goal 4) | 8 | +| `tests/test_quality_site.py`, `tests/test_quality_seo.py`, `tests/test_quality_reduce.py`, `tests/test_quality_docker.py`, `tests/test_quality_runner.py`, `tests/test_quality_summary.py`, `tests/fixtures/quality/**` | tests | 1–7 | + +## Contracts + +### Collector CLI (`quality/run.mjs`) + +``` +node /quality/run.mjs --site --pages '' --out +``` + +`--pages` is the object returned by `site.resolve_pages`, for example: + +```json +{"index": {"url": "/", "file": "index.html"}, + "post": {"url": "/post/hello/", "file": "post/hello/index.html"}, + "404": {"url": "/404.html", "file": "404.html"}} +``` + +The collector writes `/raw.json` and exits 0, even when checks fail. It exits non-zero only if it cannot write `raw.json`. + +### `raw.json` (collector output, input of `reduce.build_quality`) + +Any top-level section may instead be `{"error": ""}`, and any page entry may be `{"error": ""}`. + +```json +{ + "tools": {"node": "v24.x", "lighthouse": "x.y.z", "chromium": "x.y", "axeCore": "x.y.z", + "htmlValidate": "x.y.z", "lychee": "x.y.z", "playwright": "x.y.z"}, + "pages": {"index": {"url": "/", "file": "index.html"}, "post": {...}, "404": {...}}, + "lighthouse": { + "mobile": {"index": [RUN, RUN, RUN], "post": [...], "404": [...]}, + "desktop": {"index": [...], "post": [...], "404": [...]} + }, + "axe": {"index": {"violations": [{"id": "color-contrast", "impact": "serious", "nodes": 3}]}, "post": {...}, "404": {...}}, + "rendered": {"post": {"title": "Post title", "firstParagraph": "First paragraph text"}}, + "html": {"index": {"messages": [{"ruleId": "void-style", "severity": 2}]}, "post": {...}, "404": {...}}, + "links": {"report": }, + "files": [{"path": "index.html", "bytes": 1234, "gzip": 600, "br": 500}] +} +``` + +`RUN` is either `{"error": ""}` or: + +```json +{"scores": {"performance": 0.98, "accessibility": 1, "best-practices": 1, "seo": 0.82}, + "metrics": {"fcp": 812.3, "lcp": 812.3, "tbt": 0, "cls": 0, "si": 812.3}, + "resources": {"totalBytes": 5321, "jsBytes": 0, "cssBytes": 1210, "requests": 3, "domSize": 54}, + "failedAudits": ["meta-description"]} +``` + +Metric units are Lighthouse's `numericValue`: milliseconds for FCP, LCP, TBT and SI, unitless for CLS. + +### `quality.json` (R-16) + +```json +{ + "status": "ok", + "pages": {"index": "/", "post": "/post/hello/", "404": "/404.html"}, + "postSource": "index", + "tools": {...copied from raw.json...}, + "lighthouse": {"status": "ok", + "mobile": {"post": {"status": "ok", "runs": 3, + "scores": {"performance": {"median": 0.98, "min": 0.97, "max": 0.99}, ...}, + "metrics": {"lcp": {"median": 812.3, "min": ..., "max": ...}, ...}, + "resources": {"jsBytes": {"median": 0, ...}, ...}, + "failedAudits": ["meta-description"]}, "index": {...}, "404": {...}}, + "desktop": {...}}, + "seo": {"status": "ok", "total": 12, + "pages": {"index": {...signals...}, "post": {...}, "404": {...}}, + "present": {"index": 2, "post": 2, "404": 2}, + "site": {"robotsTxt": false, "sitemap": {"present": false, "valid": null, "urls": null}, + "feed": false, "favicon": false}}, + "html": {"status": "ok", "pages": {"index": {"errors": 0, "warnings": 1, "rules": ["..."]}, ...}}, + "a11y": {"status": "ok", "pages": {"index": {"violations": {"critical": 0, "serious": 1, "moderate": 0, "minor": 0}, + "total": 1, "rules": ["color-contrast"]}, ...}}, + "links": {"status": "ok", "broken": 0, "byKind": {"link": 0, "anchor": 0, "asset": 0}, "items": []}, + "weight": {"status": "ok", "bytes": 0, "gzipBytes": 0, "brotliBytes": 0, "bytesPerPost": 0, + "generatorAddedBytes": 0}, + "js": {"status": "ok", "pages": {"index": 0, "post": 0, "404": 0}, "diskBytes": 0, + "jsClass": "none", "worksWithoutJs": true}, + "extra": {"status": "ok", "files": 0, "bytes": 0, "byExtension": {}} +} +``` + +When the whole pass fails, `quality.json` is `{"status": "error", "error": ": "}`. + +### Python interfaces + +- **`toolset/quality/site.py`:** + - `REFERENCE_ASSETS = ('assets/ssgberk.css', 'assets/ssgberk.png')` + - `generator_config(test_directory) -> (output_folder: str, output_glob: str)` + - `extract(tar_path, dest) -> str`: extracts the archive and strips its top directory; returns `dest`. + - `url_to_file(site_dir, url) -> str | None` + - `resolve_pages(site_dir) -> dict` (shape above); raises `ValueError`. + - `list_files(site_dir) -> list[str]`: sorted POSIX relative paths. + - `extra_output(site_dir, output_glob) -> {"files", "bytes", "byExtension"}` + - `works_without_js(site_dir, post_file, rendered) -> bool | None` +- **`toolset/quality/seo.py`:** + - `PAGE_SIGNALS` (tuple of 12 keys) + - `page_signals(html: str) -> dict` + - `site_signals(site_dir, page_signals_list) -> dict` + - `seo_section(site_dir, pages) -> {"total", "pages", "present", "site"}` +- **`toolset/quality/reduce.py`:** + - `SECTIONS`, `spread(values)` + - `reduce_lighthouse(raw_lh)`, `reduce_axe(raw_axe)`, `reduce_html(raw_html)`, `reduce_links(raw_links)` + - `weight(files, output_glob)`, `js_section(lighthouse, files, works)` + - `build_quality(raw, site_dir, pages, output_glob) -> dict` +- **`toolset/quality/runner.py`:** + - `QUALITY_CELL = (50, 5.0)` + - `eligible(config) -> bool` + - `skipped(config) -> bool`: `--quality` was given but this cell is not eligible. + - `archive_path(results_dir, name) -> str` + - `run_pass(docker_helper, test, results_dir) -> dict`, which never raises. +- **`toolset/utils/docker_helper.py`:** + - `QUALITY_IMAGE = 'ssgberk/quality'`, `QUALITY_TIMEOUT_SECONDS = 1800` + - `DockerHelper.benchmark(..., export=None)`, where `export` is `(output_folder, dest_tar_path)` + - `DockerHelper.build_quality_image() -> str` (image id) + - `DockerHelper.run_quality(tar_path, site_name, pages, timeout_seconds=QUALITY_TIMEOUT_SECONDS) -> dict` (raw.json) +- **`toolset/utils/results.py`:** `Results.add_quality(name, quality, image_id=None)` +- **`toolset/utils/summary.py`:** + - `QUALITY_COLUMNS` + - `quality_rows(results) -> list[dict]` + - `quality_csv(rows) -> str` + - `quality_markdown(rows) -> list[str]` + +## Decisions + +1. **The collector writes raw data, and Python reduces it.** Medians, counts, the SEO signals and the page set live in Python so they are covered by pytest without Docker or Chrome (spec "Testing"). The collector only does what needs a browser or a binary. +2. **One Chromium, two drivers.** Lighthouse drives Playwright's Chromium through `chrome-launcher` (`chromePath = chromium.executablePath()`). axe-core and the rendered-text check use Playwright on the same binary. Chrome runs with `--headless=new --no-sandbox --disable-gpu --disable-dev-shm-usage`, because the container runs as root and `/dev/shm` is small. +3. **The static server is a few lines of Node `http`.** It has gzip, fixed headers and directory index, and it serves `404.html` with status 404, so no server package version can change the results. +4. **The summary uses the post page.** The 4 scores and LCP in the `summary.md` row come from the mobile preset on the post page, the page type that scales with the site. `quality.json` keeps every page and both presets. +5. **The image builds outside the time logger.** `build_quality_image` streams its own `APIClient.build`, so the generator's build-time log lines and `imageBuild` are not touched. It builds once per process, the first time a pass runs. +6. **No new dependency in the toolset image.** brotli is computed by the collector (Node `zlib`), not by Python. +7. **The export is skipped when the build did not end `ok` with exit code 0.** For a noisy re-run, the export of the re-run overwrites the first one, so the audited output is always the last build. +8. **The sitemap is parsed with `xml.etree.ElementTree`, not `defusedxml`.** The toolset stays stdlib plus `docker`. `sitemap.xml` comes from the generator under test, not from the network. ElementTree never fetches external entities, and the expat bundled with Python 3.12 limits entity expansion (billion laughs). A parse error is recorded as `valid: false`. +9. **What happens to the exported files.** The extracted site stays in `results//quality//site/` for inspection. `site.tar` is deleted after the pass. diff --git a/docs/specs/009-output-quality/spec.md b/docs/specs/009-output-quality/spec.md new file mode 100644 index 0000000..c64f96c --- /dev/null +++ b/docs/specs/009-output-quality/spec.md @@ -0,0 +1,102 @@ +# 009 Output Quality + +- **Date:** 2026-10-10 +- **Status:** implemented +- **Repo:** `ssgberk/benchmark-tool` (BT). No change to `ssgberk/ssg-frameworks` (SF). +- **Issue:** #22 +- **Related:** BT `008-benchmark-methodology` (suites, `results.json`, environment fingerprint), BT `006-results-summary` (`summary.md`), SF `005-reference-site-design` (what is built, R-14 page set), SF `006-layout-conformance` (post page resolved from the index). + +## Context + +The toolset measures how long a cold build takes and what it costs: wall time, CPU, peak RSS, input and output bytes, image build time (BT 008). SF 001 and SF 006 only prove that the output is the reference site: same pages, same selectors, same counts. SF 006 leaves out visual comparison, CSS rendering and accessibility audits. + +Every generator renders the same reference site, with the same content, templates and stylesheet, so the output is directly comparable. The differences come from the generator itself: hydration and client-side JS, extra requests, minification, extra pages, markup validity and accessibility, and the SEO signals it injects. None of this is measured today. + +This spec adds an **untimed quality pass** over the output of one build per generator. It adds no build metric and does not change how builds are timed. + +## Requirements + +### Running the pass + +- **R-1** `--quality` enables the pass. Without `--suite`, it runs in the cell given by `-nf`/`-cs`. With `--suite`, it runs only in the cell `numberOfFiles=50, contentSize=5` (suite P, cell 0), and in every other cell it only logs that the pass was skipped. +- **R-2** The pass runs after the timed runs of a generator succeed (status `ok`). It never runs for `failed`, `timeout`, `oom`, `nonconformant` or `unsupported` results, and it never changes a timing, a status or a ranking. +- **R-3** Before the build container is removed, `DockerHelper` copies `/` out of it with `container.get_archive()` into `results//quality//site/`. `WorkingDir` comes from the container config, and `output_folder` comes from `benchmark_config.json`. That folder holds the output of the last timed build. +- **R-4** The checks run in a BT-owned image built from `quality/Dockerfile`, with pinned versions of Node, Playwright's Chromium build, Lighthouse, axe-core, html-validate and lychee. Chrome for Testing has no Linux arm64 build, but Playwright ships Chromium for both architectures. Its container runs with `network_mode="none"`. The site goes in with `put_archive` and `raw.json` comes out with `get_archive`. Nothing is bind-mounted, because the toolset itself runs in a container and host paths differ. +- **R-5** Inside that container, one static server serves the site on `127.0.0.1` with gzip on and fixed headers. `quality/run.mjs` runs every check against it and writes `quality.json`. +- **R-6** The pinned tool versions (Lighthouse, Chromium, axe-core, html-validate, lychee, Node) and the quality image id are recorded in `results.json` `environment.quality`. + +### URLs + +- **R-7** Up to three URLs are audited: the index `/`, one post page and `/404.html` when that file exists. The post page is the first `a[href]` inside `.post-item`, the same link SF 006 resolves. When the index has no such link (generators that do not use the SF 005 layout yet), it is the first file, in sorted order, that matches `output_glob`. `quality.json` records which rule chose it in `postSource` (`index` or `glob`). + +### Checks + +- **R-8 Lighthouse:** the mobile preset (simulated throttling) and the desktop preset, with 3 runs per URL and preset. Each run records: + - the scores for performance, accessibility, best-practices and SEO; + - FCP, LCP, TBT, CLS and Speed Index; + - total byte weight, JS bytes, CSS bytes, request count and DOM size; + - the ids of failed audits. + + `quality.json` stores, for each numeric value, the median and range of the 3 runs, and stores the failed audit ids of the median run. +- **R-9 SEO signals:** reported per page, not scored. + - `` (present, length), `html[lang]`, `meta[name=description]` (present, length), `meta[name=viewport]`, `link[rel=canonical]`. + - Open Graph `og:title`, `og:description`, `og:type` and `og:url`, and `twitter:card`. + - JSON-LD: blocks present and whether each one parses. + - The `meta[name=robots]` value and `link[rel=alternate][hreflang]` count. + - The `h1` count, whether the heading levels skip, `img` without `alt`, and `a` with no text or accessible name. + + Site-wide: `robots.txt` present, `sitemap.xml` present and valid (and its URL count), RSS or Atom feed present, and favicon present. For a page, `seo.present` counts the per-page signals found, out of a fixed total that `plan.md` lists. +- **R-10 HTML validity:** `html-validate` error and warning counts per page, plus the rule ids. +- **R-11 Accessibility:** axe-core violations per page, counted by impact (critical, serious, moderate, minor), plus the rule ids. +- **R-12 Links:** `lychee --offline` over the whole site. Count the broken internal links, anchors and asset references, and list up to 20 of them. +- **R-13 Weight:** + - total output bytes, gzip bytes and brotli bytes; + - bytes per post page; + - generator-added bytes, which is the total minus the SF 005 reference assets (`assets/ssgberk.css`, `assets/ssgberk.png`). Every generator ships those assets unchanged. +- **R-14 JavaScript:** + - JS bytes requested per audited page (from Lighthouse) and JS bytes on disk; + - `worksWithoutJs`, true when the post page's static HTML already contains the post title and the first body paragraph. + + `jsClass` is `none` (0 bytes of JS on every audited page) or `hydrated` (any JS). +- **R-15 Extra output:** files outside the SF 005 page set (index, post pages, `404.html`, reference assets), counted and sized and grouped by extension. + +### Results + +- **R-16** `quality.json` has one section per check (`lighthouse`, `seo`, `html`, `a11y`, `links`, `weight`, `js`, `extra`). Each section carries `status` (`ok` or `error`) and, on error, `error` with the message. A failing check is a result, not an error: it never fails the generator, the cell or the run. +- **R-17** `results.json` gets `quality.<generator>`: the parsed `quality.json`, or `{"status": "error", "error": ...}` when the pass could not run. +- **R-18** `summary.md` gets a "Qualidade" section with one row per generator. The columns are: + - the 4 mobile Lighthouse scores and mobile LCP; + - the post page's JS bytes, `jsClass` and gzip bytes; + - the HTML error, axe violation and broken link counts; + - SEO signals present, as `n/total`. + + Rows are sorted by name, with no combined score and no ranking. `quality-summary.csv` holds the same columns plus every numeric field. The rounds page reads that file; the page itself is not in this repository. +- **R-19** The section notes that the SF 005 templates carry no meta description and no viewport, so every generator loses the same SEO points. That constant isn't a difference between generators. + +## Goals and success criteria + +1. **One command audits the output.** *Measured by:* `./ssgberk --test hugo -nf 50 -cs 5 --quality` writes `results/<ts>/quality/hugo/quality.json` with all 8 sections and `status: ok`. A unit test asserts that `rawData`, `succeeded`, `failed` and the rankings in `summary.md` are identical with and without a `quality` section for the same parsed build output. +2. **The pass is isolated.** *Measured by:* a unit test asserts that the quality container runs with `network_mode="none"` and no volumes or mounts. Another asserts that a raised error in the pass leaves the build result unchanged and records `quality.<fw>.status = "error"`. +3. **Checks are pinned and recorded.** *Measured by:* `environment.quality` lists every tool version from R-6, and `quality/Dockerfile` names an exact version for each one. +4. **The matrix runs in CI.** *Measured by:* a BT CI job runs the pass on hugo for `ubuntu-24.04` and `ubuntu-24.04-arm`, and checks that every section is present and every score is between 0 and 1. A `workflow_dispatch` input runs it on every generator, one job each, and the job succeeds even when a check reports failures. + +## Testing + +- **pytest (no Docker):** + - SEO signal extraction from the SF 005 reference page and from variants (with description, with OG tags, with a broken JSON-LD block, a skipped heading level, `img` without `alt`); + - the Lighthouse median and range reduction; + - the axe and html-validate count reduction; + - extra output classification; + - weight arithmetic; + - the `quality.json` to `results.json` and `summary.md` merge; + - the R-1 skip rules. +- **Inside the quality image:** `quality/run.mjs` runs against a 3-page fixture site (index, one post, 404) and its `quality.json` matches a checked-in expected shape (keys and types, not Lighthouse values). + +## Out of scope + +- New build metrics: install time, image size breakdown, incremental builds, startup cost. +- Determinism (building twice and comparing the trees), because it needs a second build. +- A combined quality score or ranking. +- Changes to the SF 005 reference site, the SF 006 conformance rules or any generator. +- Field data, RUM and real-user Core Web Vitals. +- Tuning generators to score better: the pass audits the reference configuration as built. diff --git a/docs/specs/009-output-quality/tasks.md b/docs/specs/009-output-quality/tasks.md new file mode 100644 index 0000000..ef3557e --- /dev/null +++ b/docs/specs/009-output-quality/tasks.md @@ -0,0 +1,2647 @@ +# 009 Output Quality: Tasks + +Read `spec.md` and `plan.md` (Global Constraints, Contracts, Decisions) first. Every task's requirements include the Global Constraints. Run commands from the repository root. Commit with `git commit -S` as configured; never pass `--no-gpg-sign` or `--author`. End every commit message with: + +``` +Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> +``` + +Test command (no Docker): `python3 -m pytest -q` and `ruff check toolset tests`. + +--- + +### Task 1: Site helpers (`toolset/quality/site.py`) + +**Files:** +- Create: `toolset/quality/__init__.py` (empty), `toolset/quality/site.py` +- Create: `tests/fixtures/quality/site/index.html`, `tests/fixtures/quality/site/post/hello/index.html`, `tests/fixtures/quality/site/404.html`, `tests/fixtures/quality/site/assets/ssgberk.css`, `tests/fixtures/quality/site/assets/ssgberk.png`, `tests/fixtures/quality/site/tags/a/index.html`, `tests/fixtures/quality/site/feed.xml` +- Test: `tests/test_quality_site.py` + +**Interfaces:** +- Consumes: nothing. +- Produces: + - `REFERENCE_ASSETS` + - `generator_config(test_directory) -> (str, str)` + - `extract(tar_path, dest) -> str` + - `url_to_file(site_dir, url) -> str | None` + - `resolve_pages(site_dir) -> dict` + - `list_files(site_dir) -> list[str]` + - `extra_output(site_dir, output_glob) -> dict` (no `status` key) + - `works_without_js(site_dir, post_file, rendered) -> bool | None` + - `text_of(html) -> str`, which collapses the visible text of an HTML string. + +- [ ] **Step 1: Write the fixture site** + +`tests/fixtures/quality/site/index.html`: + +```html +<!doctype html> +<html lang="en"> +<head><meta charset="utf-8"><title>SSGBerk Reference + + +
SSGBerk Reference +
+
+

Built for the SSGBerk build-time benchmark.

+ + +``` + +`tests/fixtures/quality/site/post/hello/index.html`: + +```html + + +Hello + + +
SSGBerk Reference +
+
+

Hello

+

SSGBerk

+
  • a
  • b
  • c
+

First paragraph of the post.

Second paragraph.

+
+

Built for the SSGBerk build-time benchmark.

+ + +``` + +`tests/fixtures/quality/site/404.html`: + +```html + + +Not found + + +
SSGBerk Reference +
+

Page not found

+

Built for the SSGBerk build-time benchmark.

+ + +``` + +`tests/fixtures/quality/site/assets/ssgberk.css`: + +```css +body { font-family: sans-serif; margin: 0 auto; max-width: 40rem; } +``` + +`tests/fixtures/quality/site/tags/a/index.html` (extra output): + +```html + +Tag a

a

+``` + +`tests/fixtures/quality/site/feed.xml` (extra output): + +```xml + +SSGBerk Reference +``` + +Write the 1×1 PNG with Python: + +```bash +python3 - <<'EOF' +import struct, zlib +def chunk(t, d): + return struct.pack('>I', len(d)) + t + d + struct.pack('>I', zlib.crc32(t + d) & 0xffffffff) +png = (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 1, 1, 8, 2, 0, 0, 0)) + + chunk(b'IDAT', zlib.compress(b'\x00\xff\xff\xff')) + chunk(b'IEND', b'')) +open('tests/fixtures/quality/site/assets/ssgberk.png', 'wb').write(png) +EOF +``` + +- [ ] **Step 2: Write the failing tests** + +`tests/test_quality_site.py`: + +```python +import io +import json +import pathlib +import tarfile + +import pytest + +from toolset.quality import site + +SITE = pathlib.Path(__file__).parent / "fixtures" / "quality" / "site" + + +def test_generator_config(tmp_path): + (tmp_path / "benchmark_config.json").write_text(json.dumps( + {"framework": "hugo", "config": [{"output_folder": "public", "output_glob": "post/*/index.html"}]})) + assert site.generator_config(str(tmp_path)) == ("public", "post/*/index.html") + + +def _tar(entries): + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w") as tar: + for name, data in entries: + info = tarfile.TarInfo(name) + if data is None: + info.type = tarfile.DIRTYPE + tar.addfile(info) + else: + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + return buf.getvalue() + + +def test_extract_strips_top_dir(tmp_path): + tar_path = tmp_path / "site.tar" + tar_path.write_bytes(_tar([("public", None), ("public/index.html", b"

i

"), + ("public/post/a/index.html", b"

a

")])) + dest = tmp_path / "site" + assert site.extract(str(tar_path), str(dest)) == str(dest) + assert (dest / "index.html").read_bytes() == b"

i

" + assert (dest / "post" / "a" / "index.html").exists() + + +def test_extract_rejects_path_traversal(tmp_path): + tar_path = tmp_path / "site.tar" + tar_path.write_bytes(_tar([("public/../../evil.html", b"x")])) + with pytest.raises(tarfile.TarError): + site.extract(str(tar_path), str(tmp_path / "site")) + assert not (tmp_path.parent / "evil.html").exists() + + +def test_resolve_pages_fixture(): + assert site.resolve_pages(str(SITE)) == { + "index": {"url": "/", "file": "index.html"}, + "post": {"url": "/post/hello/", "file": "post/hello/index.html"}, + "404": {"url": "/404.html", "file": "404.html"}, + } + + +@pytest.mark.parametrize("href,url,file", [ + ("/post/a/", "/post/a/", "post/a/index.html"), + ("post/a/", "/post/a/", "post/a/index.html"), + ("./post/a/index.html", "/post/a/index.html", "post/a/index.html"), + ("/post/a?x=1#t", "/post/a", "post/a/index.html"), + ("/post/a", "/post/a", "post/a/index.html"), + ("/post/b", "/post/b", "post/b.html"), +]) +def test_resolve_pages_variants(tmp_path, href, url, file): + (tmp_path / "post" / "a").mkdir(parents=True) + (tmp_path / "post" / "a" / "index.html").write_text("a") + (tmp_path / "post" / "b.html").write_text("b") + (tmp_path / "index.html").write_text( + '' % href) + pages = site.resolve_pages(str(tmp_path)) + assert pages["post"] == {"url": url, "file": file} + + +def test_resolve_pages_without_post_link(tmp_path): + (tmp_path / "index.html").write_text('x') + with pytest.raises(ValueError, match="post-item"): + site.resolve_pages(str(tmp_path)) + + +def test_resolve_pages_link_to_missing_file(tmp_path): + (tmp_path / "index.html").write_text('
  • g
    • ') + with pytest.raises(ValueError, match="no file"): + site.resolve_pages(str(tmp_path)) + + +def test_list_files_is_sorted_posix(): + files = site.list_files(str(SITE)) + assert files == sorted(files) and "post/hello/index.html" in files + + +def test_extra_output_fixture(): + extra = site.extra_output(str(SITE), "post/*/index.html") + assert extra["files"] == 2 + assert set(extra["byExtension"]) == {".html", ".xml"} + assert extra["byExtension"][".html"]["files"] == 1 + assert extra["bytes"] == sum(e["bytes"] for e in extra["byExtension"].values()) + + +def test_works_without_js(): + post = "post/hello/index.html" + rendered = {"title": "Hello", "firstParagraph": "First paragraph of the post."} + assert site.works_without_js(str(SITE), post, rendered) is True + assert site.works_without_js(str(SITE), post, {"title": "Hello", "firstParagraph": "Only after JS"}) is False + assert site.works_without_js(str(SITE), post, None) is None + assert site.works_without_js(str(SITE), post, {"error": "boom"}) is None + + +def test_text_of_collapses_whitespace(): + assert site.text_of("

    a\n b

    ") == "a b" +``` + +- [ ] **Step 3: Run them and see them fail** + +Run: `python3 -m pytest -q tests/test_quality_site.py` +Expected: FAIL with `ModuleNotFoundError: No module named 'toolset.quality'`. + +- [ ] **Step 4: Implement `toolset/quality/site.py`** + +```python +''' +Pages and files of a generated site (spec 009 R-3, R-7, R-14, R-15). +''' +import fnmatch +import json +import os +import posixpath +import re +import tarfile +from html.parser import HTMLParser +from urllib.parse import unquote, urlsplit + +# SF 005 reference assets: every generator ships them unchanged (R-13, R-15) +REFERENCE_ASSETS = ('assets/ssgberk.css', 'assets/ssgberk.png') + + +def generator_config(test_directory): + '''(output_folder, output_glob) of benchmark_config.json config[0].''' + with open(os.path.join(test_directory, 'benchmark_config.json')) as f: + config = json.load(f)['config'][0] + return config['output_folder'], config['output_glob'] + + +def extract(tar_path, dest): + ''' + Extracts the archive DockerHelper.benchmark exported (a single top + directory named after output_folder) into dest, without that directory. + ''' + os.makedirs(dest, exist_ok=True) + with tarfile.open(tar_path) as tar: + members = [] + for member in tar.getmembers(): + parts = member.name.split('/', 1) + if len(parts) < 2 or not parts[1]: + continue + member.name = parts[1] + members.append(member) + # filter='data' rejects absolute paths, '..' and links out of dest + tar.extractall(dest, members=members, filter='data') + return dest + + +def url_to_file(site_dir, url): + '''The file a static server would return for url, or None.''' + p = unquote(urlsplit(url).path).lstrip('/') + if p == '' or p.endswith('/'): + candidates = [posixpath.join(p, 'index.html')] + else: + candidates = [p, p + '/index.html', p + '.html'] + for c in candidates: + if os.path.isfile(os.path.join(site_dir, c)): + return posixpath.normpath(c) + return None + + +class _PostLink(HTMLParser): + '''First a[href] after the first element whose class list has post-item.''' + + def __init__(self): + super().__init__(convert_charrefs=True) + self.in_item = False + self.href = None + + def handle_starttag(self, tag, attrs): + a = {k: (v or '') for k, v in attrs} + if 'post-item' in a.get('class', '').split(): + self.in_item = True + if tag == 'a' and self.in_item and self.href is None and a.get('href'): + self.href = a['href'] + + +def _read(path): + with open(path, encoding='utf-8', errors='replace') as f: + return f.read() + + +def resolve_pages(site_dir): + '''The three audited pages (R-7): index, first post link, 404.''' + parser = _PostLink() + parser.feed(_read(os.path.join(site_dir, 'index.html'))) + if not parser.href: + raise ValueError('no a[href] inside .post-item on the index') + path = urlsplit(parser.href).path + url = posixpath.normpath(posixpath.join('/', path)) + if path.endswith('/') and url != '/': + url += '/' + post_file = url_to_file(site_dir, url) + if post_file is None: + raise ValueError('post link %s matches no file' % parser.href) + return { + 'index': {'url': '/', 'file': 'index.html'}, + 'post': {'url': url, 'file': post_file}, + '404': {'url': '/404.html', 'file': '404.html'}, + } + + +def list_files(site_dir): + out = [] + for root, _dirs, files in os.walk(site_dir): + for name in files: + rel = os.path.relpath(os.path.join(root, name), site_dir) + out.append(rel.replace(os.sep, '/')) + return sorted(out) + + +def extra_output(site_dir, output_glob): + '''R-15: files outside the SF 005 page set, by extension.''' + by_ext = {} + total_files = total_bytes = 0 + for f in list_files(site_dir): + if f in ('index.html', '404.html') or f in REFERENCE_ASSETS \ + or fnmatch.fnmatch(f, output_glob): + continue + size = os.path.getsize(os.path.join(site_dir, f)) + ext = os.path.splitext(f)[1].lower() or '(none)' + entry = by_ext.setdefault(ext, {'files': 0, 'bytes': 0}) + entry['files'] += 1 + entry['bytes'] += size + total_files += 1 + total_bytes += size + return {'files': total_files, 'bytes': total_bytes, + 'byExtension': dict(sorted(by_ext.items()))} + + +class _Text(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.parts = [] + self.skip = 0 + + def handle_starttag(self, tag, attrs): + if tag in ('script', 'style', 'template'): + self.skip += 1 + + def handle_endtag(self, tag): + if tag in ('script', 'style', 'template') and self.skip: + self.skip -= 1 + + def handle_data(self, data): + if not self.skip: + self.parts.append(data) + + +def text_of(html): + parser = _Text() + parser.feed(html) + parser.close() + return re.sub(r'\s+', ' ', ' '.join(parser.parts)).strip() + + +def works_without_js(site_dir, post_file, rendered): + ''' + R-14: True when the static post HTML already holds the title and first + body paragraph that the browser shows after scripts ran. + ''' + if not isinstance(rendered, dict) or 'error' in rendered: + return None + title = re.sub(r'\s+', ' ', rendered.get('title') or '').strip() + first = re.sub(r'\s+', ' ', rendered.get('firstParagraph') or '').strip() + if not title or not first: + return None + static = text_of(_read(os.path.join(site_dir, post_file))) + return title in static and first in static +``` + +- [ ] **Step 5: Run the tests** + +Run: `python3 -m pytest -q tests/test_quality_site.py && ruff check toolset tests` +Expected: all pass, no ruff findings. + +- [ ] **Step 6: Commit** + +```bash +git add toolset/quality/__init__.py toolset/quality/site.py tests/test_quality_site.py tests/fixtures/quality/site +git commit -m "feat(quality): site helpers for the output quality pass (spec 009) + +Co-Authored-By: Claude Opus 5.5 (1M context) " +``` + +--- + +### Task 2: SEO signals (`toolset/quality/seo.py`) + +**Files:** +- Create: `toolset/quality/seo.py` +- Test: `tests/test_quality_seo.py` + +**Interfaces:** +- Consumes: `site.list_files`, `site._read` (Task 1); the fixture site. +- Produces: + - `PAGE_SIGNALS` + - `page_signals(html) -> dict` + - `site_signals(site_dir, page_signals_list) -> dict` + - `seo_section(site_dir, pages) -> {"total", "pages", "present", "site"}` (no `status` key) + +- [ ] **Step 1: Write the failing tests** + +`tests/test_quality_seo.py`: + +```python +import pathlib + +from toolset.quality import seo, site + +SITE = pathlib.Path(__file__).parent / "fixtures" / "quality" / "site" + +RICH = """ +Post title + + + + + + + + + + +

    T

    S

    a""" + + +def test_reference_page_has_only_title_and_lang(): + s = seo.page_signals((SITE / "post" / "hello" / "index.html").read_text()) + assert s["title"] is True and s["titleLength"] == 5 and s["lang"] == "en" + assert s["description"] is False and s["viewport"] is False and s["canonical"] is False + assert s["jsonLd"] is False and s["jsonLdValid"] is None and s["robotsMeta"] is None + assert s["h1"] == 1 and s["headingSkip"] is False and s["imgWithoutAlt"] == 0 + assert s["present"] == 2 + + +def test_rich_page_has_every_signal(): + s = seo.page_signals(RICH) + assert s["present"] == len(seo.PAGE_SIGNALS) == 12 + assert s["descriptionLength"] == len("A post about things") + assert s["robotsMeta"] == "index,follow" and s["hreflang"] == 2 + assert s["jsonLdBlocks"] == 1 and s["jsonLdValid"] is True + assert s["faviconLink"] is True and s["imgWithoutAlt"] == 0 + + +def test_broken_json_ld(): + s = seo.page_signals('') + assert s["jsonLd"] is True and s["jsonLdValid"] is False + + +def test_heading_skip_images_and_empty_links(): + html = ('

    a

    b

    ' + '' + 'logo no href') + s = seo.page_signals(html) + assert s["headingSkip"] is True + assert s["imgWithoutAlt"] == 1 + assert s["emptyLinks"] == 2 + + +def test_page_signals_malformed_html(): + html = b"x

    unclosed \xff\xfe".decode("utf-8", "replace") + s = seo.page_signals(html) + assert s["title"] is True and s["lang"] == "en" + + +def test_site_signals(tmp_path): + (tmp_path / "robots.txt").write_text("User-agent: *\n") + (tmp_path / "sitemap.xml").write_text( + '' + '/a//b/') + (tmp_path / "feed.xml").write_text("") + (tmp_path / "favicon.ico").write_bytes(b"\x00") + s = seo.site_signals(str(tmp_path), []) + assert s == {"robotsTxt": True, "sitemap": {"present": True, "valid": True, "urls": 2}, + "feed": True, "favicon": True} + + +def test_site_signals_invalid_sitemap_and_link_favicon(tmp_path): + (tmp_path / "sitemap.xml").write_text("") + s = seo.site_signals(str(tmp_path), [{"faviconLink": True}]) + assert s["sitemap"] == {"present": True, "valid": False, "urls": None} + assert s["favicon"] is True and s["robotsTxt"] is False and s["feed"] is False + + +def test_seo_section_fixture(): + pages = site.resolve_pages(str(SITE)) + section = seo.seo_section(str(SITE), pages) + assert section["total"] == 12 + assert section["present"] == {"index": 2, "post": 2, "404": 2} + assert section["site"]["feed"] is True and section["site"]["sitemap"]["present"] is False + assert set(section["pages"]) == {"index", "post", "404"} +``` + +- [ ] **Step 2: Run them and see them fail** + +Run: `python3 -m pytest -q tests/test_quality_seo.py` +Expected: FAIL with `ImportError: cannot import name 'seo'`. + +- [ ] **Step 3: Implement `toolset/quality/seo.py`** + +```python +''' +SEO signals of a generated page and site (spec 009 R-9). Reported, not scored. +''' +import json +import os +import re +import xml.etree.ElementTree as ET +from html.parser import HTMLParser + +from toolset.quality import site + +# The per-page signals counted in seo.present (out of len(PAGE_SIGNALS)) +PAGE_SIGNALS = ('title', 'lang', 'description', 'viewport', 'canonical', 'ogTitle', + 'ogDescription', 'ogType', 'ogUrl', 'twitterCard', 'jsonLd', 'robotsMeta') +META = {('name', 'description'): 'description', ('name', 'viewport'): 'viewport', + ('property', 'og:title'): 'ogTitle', ('property', 'og:description'): 'ogDescription', + ('property', 'og:type'): 'ogType', ('property', 'og:url'): 'ogUrl', + ('name', 'twitter:card'): 'twitterCard', ('name', 'robots'): 'robotsMeta'} +FEEDS = ('feed.xml', 'atom.xml', 'rss.xml', 'index.xml', 'feed/index.xml', 'rss/index.xml') +FAVICONS = ('favicon.ico', 'favicon.svg', 'favicon.png') + + +class _Collector(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.lang = None + self.title = None + self.meta = {} + self.canonical = None + self.hreflang = 0 + self.favicon_link = False + self.jsonld = [] + self.headings = [] + self.img_without_alt = 0 + self.empty_links = 0 + self._capture = None # 'title' or 'jsonld' + self._buf = [] + self._links = [] # one bool per open : has an accessible name + + def handle_starttag(self, tag, attrs): + a = {k: (v or '') for k, v in attrs} + if tag == 'html' and a.get('lang'): + self.lang = a['lang'] + elif tag == 'title' and self.title is None: + self._capture, self._buf = 'title', [] + elif tag == 'meta': + for (attr, value), key in META.items(): + if a.get(attr, '').lower() == value: + self.meta[key] = a.get('content', '') + elif tag == 'link': + rel = a.get('rel', '').lower().split() + if 'canonical' in rel and a.get('href'): + self.canonical = a['href'] + if 'alternate' in rel and 'hreflang' in a: + self.hreflang += 1 + if 'icon' in rel: + self.favicon_link = True + elif tag == 'script' and a.get('type', '').lower() == 'application/ld+json': + self._capture, self._buf = 'jsonld', [] + elif re.fullmatch(r'h[1-6]', tag): + self.headings.append(int(tag[1])) + elif tag == 'img': + if 'alt' not in a: + self.img_without_alt += 1 + elif a['alt'].strip() and self._links: + self._links[-1] = True + elif tag == 'a' and 'href' in a: + self._links.append(bool(a.get('aria-label', '').strip() or a.get('title', '').strip())) + + def handle_endtag(self, tag): + if tag == 'title' and self._capture == 'title': + self.title = ''.join(self._buf).strip() + self._capture = None + elif tag == 'script' and self._capture == 'jsonld': + self.jsonld.append(''.join(self._buf)) + self._capture = None + elif tag == 'a' and self._links: + if not self._links.pop(): + self.empty_links += 1 + + def handle_data(self, data): + if self._capture: + self._buf.append(data) + if self._links and data.strip(): + self._links[-1] = True + + +def _skips(levels): + return any(b > a + 1 for a, b in zip(levels, levels[1:])) + + +def page_signals(html): + c = _Collector() + c.feed(html) + c.close() + valid = [] + for block in c.jsonld: + try: + json.loads(block) + valid.append(True) + except ValueError: + valid.append(False) + s = { + 'title': bool(c.title), + 'titleLength': len(c.title or ''), + 'lang': c.lang, + 'description': bool(c.meta.get('description')), + 'descriptionLength': len(c.meta.get('description', '')), + 'viewport': 'viewport' in c.meta, + 'canonical': bool(c.canonical), + 'ogTitle': 'ogTitle' in c.meta, + 'ogDescription': 'ogDescription' in c.meta, + 'ogType': 'ogType' in c.meta, + 'ogUrl': 'ogUrl' in c.meta, + 'twitterCard': 'twitterCard' in c.meta, + 'jsonLd': bool(c.jsonld), + 'jsonLdBlocks': len(c.jsonld), + 'jsonLdValid': all(valid) if valid else None, + 'robotsMeta': c.meta.get('robotsMeta'), + 'hreflang': c.hreflang, + 'h1': c.headings.count(1), + 'headingSkip': _skips(c.headings), + 'imgWithoutAlt': c.img_without_alt, + 'emptyLinks': c.empty_links, + 'faviconLink': c.favicon_link, + } + s['present'] = sum(1 for k in PAGE_SIGNALS if s[k] not in (False, None, '')) + return s + + +def _sitemap(site_dir): + path = os.path.join(site_dir, 'sitemap.xml') + if not os.path.isfile(path): + return {'present': False, 'valid': None, 'urls': None} + try: + root = ET.parse(path).getroot() + except ET.ParseError: + return {'present': True, 'valid': False, 'urls': None} + if not root.tag.endswith(('urlset', 'sitemapindex')): + return {'present': True, 'valid': False, 'urls': None} + return {'present': True, 'valid': True, + 'urls': sum(1 for el in root.iter() if el.tag.endswith('loc'))} + + +def site_signals(site_dir, page_signals_list): + exists = lambda rel: os.path.isfile(os.path.join(site_dir, rel)) # noqa: E731 + return { + 'robotsTxt': exists('robots.txt'), + 'sitemap': _sitemap(site_dir), + 'feed': any(exists(f) for f in FEEDS), + 'favicon': any(exists(f) for f in FAVICONS) + or any(s.get('faviconLink') for s in page_signals_list), + } + + +def seo_section(site_dir, pages): + signals = {key: page_signals(site._read(os.path.join(site_dir, page['file']))) + for key, page in pages.items()} + return { + 'total': len(PAGE_SIGNALS), + 'pages': signals, + 'present': {key: s['present'] for key, s in signals.items()}, + 'site': site_signals(site_dir, list(signals.values())), + } +``` + +- [ ] **Step 4: Run the tests** + +Run: `python3 -m pytest -q tests/test_quality_seo.py && ruff check toolset tests` +Expected: all pass. If ruff flags E731 despite the `noqa`, turn `exists` into a nested `def`. + +- [ ] **Step 5: Commit** + +```bash +git add toolset/quality/seo.py tests/test_quality_seo.py +git commit -m "feat(quality): SEO signals per page and site (spec 009 R-9) + +Co-Authored-By: Claude Opus 5.5 (1M context) " +``` + +--- + +### Task 3: Reduce the collector output (`toolset/quality/reduce.py`) + +**Files:** +- Create: `toolset/quality/reduce.py` +- Create: `tests/fixtures/quality/raw.json` (a full collector output for the fixture site) +- Test: `tests/test_quality_reduce.py` + +**Interfaces:** +- Consumes: + - `site.extra_output`, `site.works_without_js`, `site.REFERENCE_ASSETS` (Task 1); + - `seo.seo_section` (Task 2); + - the `raw.json` contract in `plan.md`. +- Produces: + - `SECTIONS`, `spread` + - `reduce_lighthouse`, `reduce_axe`, `reduce_html`, `reduce_links`, `weight`, `js_section` + - `build_quality(raw, site_dir, pages, output_glob) -> dict` (the `quality.json` contract) + +- [ ] **Step 1: Write the raw fixture** + +Generate `tests/fixtures/quality/raw.json`, so it matches the fixture site byte for byte: + +```bash +python3 - <<'EOF' +import gzip, json, os, pathlib +site = pathlib.Path('tests/fixtures/quality/site') +def run(perf, lcp, failed): + return {"scores": {"performance": perf, "accessibility": 1, "best-practices": 1, "seo": 0.82}, + "metrics": {"fcp": lcp, "lcp": lcp, "tbt": 0, "cls": 0, "si": lcp}, + "resources": {"totalBytes": 2400, "jsBytes": 0, "cssBytes": 300, "requests": 2, "domSize": 40}, + "failedAudits": failed} +pages = {"index": {"url": "/", "file": "index.html"}, + "post": {"url": "/post/hello/", "file": "post/hello/index.html"}, + "404": {"url": "/404.html", "file": "404.html"}} +lh = {p: [run(0.97, 900, ["a"]), run(0.99, 700, ["c"]), run(0.98, 800, ["b"])] for p in pages} +files = [] +for f in sorted(str(p.relative_to(site)).replace(os.sep, '/') for p in site.rglob('*') if p.is_file()): + data = (site / f).read_bytes() + files.append({"path": f, "bytes": len(data), "gzip": len(gzip.compress(data, 9)), "br": len(gzip.compress(data, 9)) - 1}) +raw = {"tools": {"node": "v24.21.0", "lighthouse": "1.0.0", "playwright": "1.0.0", "chromium": "1.0", + "axeCore": "1.0.0", "htmlValidate": "1.0.0", "chromeLauncher": "1.0.0", "lychee": "1.0.0"}, + "pages": pages, + "lighthouse": {"mobile": lh, "desktop": lh}, + "axe": {"index": {"violations": []}, + "post": {"violations": [{"id": "color-contrast", "impact": "serious", "nodes": 3}, + {"id": "region", "impact": "moderate", "nodes": 1}]}, + "404": {"violations": []}}, + "rendered": {"post": {"title": "Hello", "firstParagraph": "First paragraph of the post."}}, + "html": {"index": {"messages": []}, + "post": {"messages": [{"ruleId": "no-inline-style", "severity": 2}, + {"ruleId": "prefer-native-element", "severity": 1}]}, + "404": {"messages": []}}, + "links": {"report": {"fail_map": {str(site / "index.html"): [ + {"url": "file:///x/missing.css", "status": {"text": "Cannot find file"}}, + {"url": "file:///x/post/hello/#nope", "status": {"text": "Cannot find fragment"}}]}}}, + "files": files} +pathlib.Path('tests/fixtures/quality/raw.json').write_text(json.dumps(raw, indent=2) + "\n") +EOF +``` + +- [ ] **Step 2: Write the failing tests** + +`tests/test_quality_reduce.py`: + +```python +import json +import pathlib + +from toolset.quality import reduce, site + +FIX = pathlib.Path(__file__).parent / "fixtures" / "quality" +SITE = FIX / "site" +GLOB = "post/*/index.html" + + +def _raw(): + return json.loads((FIX / "raw.json").read_text()) + + +def test_spread(): + assert reduce.spread([3, 1, 2]) == {"median": 2, "min": 1, "max": 3} + assert reduce.spread([None, 4]) == {"median": 4, "min": 4, "max": 4} + assert reduce.spread([None]) is None + + +def test_reduce_lighthouse_median_and_median_run_audits(): + lh = reduce.reduce_lighthouse(_raw()["lighthouse"]) + post = lh["mobile"]["post"] + assert post["status"] == "ok" and post["runs"] == 3 + assert post["scores"]["performance"] == {"median": 0.98, "min": 0.97, "max": 0.99} + assert post["metrics"]["lcp"] == {"median": 800, "min": 700, "max": 900} + # the run whose performance is the median (0.98) failed audit "b" + assert post["failedAudits"] == ["b"] + assert set(lh) == {"mobile", "desktop"} + + +def test_reduce_lighthouse_page_errors(): + raw = {"mobile": {"index": {"error": "chrome died"}, + "post": [{"error": "timeout"}, {"error": "timeout"}], + "404": [{"error": "x"}, _raw()["lighthouse"]["mobile"]["post"][0]]}, + "desktop": {}} + lh = reduce.reduce_lighthouse(raw) + assert lh["mobile"]["index"] == {"status": "error", "error": "chrome died"} + assert lh["mobile"]["post"] == {"status": "error", "error": "timeout"} + assert lh["mobile"]["404"]["status"] == "ok" and lh["mobile"]["404"]["runs"] == 1 + + +def test_reduce_axe_counts_by_impact(): + a = reduce.reduce_axe(_raw()["axe"]) + assert a["post"] == {"violations": {"critical": 0, "serious": 1, "moderate": 1, "minor": 0}, + "total": 2, "rules": ["color-contrast", "region"]} + assert a["index"]["total"] == 0 + assert reduce.reduce_axe({"index": {"error": "boom"}})["index"] == {"status": "error", "error": "boom"} + + +def test_reduce_html_counts(): + h = reduce.reduce_html(_raw()["html"]) + assert h["post"] == {"errors": 1, "warnings": 1, "rules": ["no-inline-style", "prefer-native-element"]} + + +def test_reduce_links_kinds(): + links = reduce.reduce_links(_raw()["links"]) + assert links["broken"] == 2 + assert links["byKind"] == {"link": 0, "anchor": 1, "asset": 1} + assert links["items"][0]["status"] == "Cannot find file" + + +def test_reduce_links_caps_items(): + fail = {"s": [{"url": "/p%d/" % i, "status": {"text": "x"}} for i in range(30)]} + links = reduce.reduce_links({"report": {"fail_map": fail}}) + assert links["broken"] == 30 and len(links["items"]) == reduce.MAX_BROKEN_LISTED == 20 + + +def test_weight(): + files = [{"path": "index.html", "bytes": 100, "gzip": 50, "br": 40}, + {"path": "post/a/index.html", "bytes": 300, "gzip": 100, "br": 90}, + {"path": "post/b/index.html", "bytes": 100, "gzip": 60, "br": 50}, + {"path": "assets/ssgberk.css", "bytes": 1000, "gzip": 400, "br": 300}] + assert reduce.weight(files, GLOB) == {"bytes": 1500, "gzipBytes": 610, "brotliBytes": 480, + "bytesPerPost": 200, "generatorAddedBytes": 500} + + +def test_js_section_classes(): + lh = {"mobile": {"index": {"status": "ok", "resources": {"jsBytes": {"median": 0}}}, + "post": {"status": "ok", "resources": {"jsBytes": {"median": 0}}}}} + files = [{"path": "a.js", "bytes": 10, "gzip": 5, "br": 4}] + js = reduce.js_section(lh, files, True) + assert js == {"pages": {"index": 0, "post": 0}, "diskBytes": 10, "jsClass": "none", "worksWithoutJs": True} + lh["mobile"]["post"]["resources"]["jsBytes"]["median"] = 5000 + assert reduce.js_section(lh, files, False)["jsClass"] == "hydrated" + assert reduce.js_section({"mobile": {}}, [], None)["jsClass"] is None + + +def test_build_quality_fixture_all_ok(): + pages = site.resolve_pages(str(SITE)) + q = reduce.build_quality(_raw(), str(SITE), pages, GLOB) + assert q["status"] == "ok" + assert q["pages"] == {"index": "/", "post": "/post/hello/", "404": "/404.html"} + for section in reduce.SECTIONS: + assert q[section]["status"] == "ok", (section, q[section]) + assert q["js"]["worksWithoutJs"] is True and q["js"]["jsClass"] == "none" + assert q["extra"]["files"] == 2 and q["seo"]["present"]["post"] == 2 + + +def test_build_quality_partial_errors(): + raw = _raw() + raw["lighthouse"] = {"error": "lighthouse crashed"} + del raw["links"] + raw["files"] = {"error": "EACCES"} + pages = site.resolve_pages(str(SITE)) + q = reduce.build_quality(raw, str(SITE), pages, GLOB) + assert q["lighthouse"] == {"status": "error", "error": "RuntimeError: lighthouse crashed"} + assert q["links"]["status"] == "error" and "missing" in q["links"]["error"] + assert q["weight"]["status"] == "error" and q["js"]["status"] == "error" + for section in ("seo", "html", "a11y", "extra"): + assert q[section]["status"] == "ok" + assert q["status"] == "ok" +``` + +- [ ] **Step 3: Run them and see them fail** + +Run: `python3 -m pytest -q tests/test_quality_reduce.py` +Expected: FAIL with `ImportError: cannot import name 'reduce'`. + +- [ ] **Step 4: Implement `toolset/quality/reduce.py`** + +```python +''' +Reduces the collector output (quality/run.mjs raw.json) into quality.json +(spec 009 R-8, R-10..R-16). Pure functions; the contracts are in plan.md. +''' +import fnmatch +import posixpath +import statistics + +from toolset.quality import seo, site + +SECTIONS = ('lighthouse', 'seo', 'html', 'a11y', 'links', 'weight', 'js', 'extra') +PRESETS = ('mobile', 'desktop') +SCORES = ('performance', 'accessibility', 'best-practices', 'seo') +METRICS = ('fcp', 'lcp', 'tbt', 'cls', 'si') +RESOURCES = ('totalBytes', 'jsBytes', 'cssBytes', 'requests', 'domSize') +IMPACTS = ('critical', 'serious', 'moderate', 'minor') +MAX_BROKEN_LISTED = 20 + + +def spread(values): + vs = [v for v in values if v is not None] + if not vs: + return None + return {'median': statistics.median(vs), 'min': min(vs), 'max': max(vs)} + + +def _is_error(part): + return isinstance(part, dict) and 'error' in part + + +def _error(part): + return {'status': 'error', 'error': part['error']} + + +def _median_run(runs): + ranked = sorted(runs, key=lambda r: (r['scores'].get('performance') is None, + r['scores'].get('performance') or 0)) + return ranked[(len(ranked) - 1) // 2] + + +def _lighthouse_page(runs): + if _is_error(runs): + return _error(runs) + good = [r for r in runs if not _is_error(r)] + if not good: + return {'status': 'error', 'error': runs[0]['error'] if runs else 'no runs'} + return { + 'status': 'ok', + 'runs': len(good), + 'scores': {k: spread([r['scores'].get(k) for r in good]) for k in SCORES}, + 'metrics': {k: spread([r['metrics'].get(k) for r in good]) for k in METRICS}, + 'resources': {k: spread([r['resources'].get(k) for r in good]) for k in RESOURCES}, + 'failedAudits': _median_run(good)['failedAudits'], + } + + +def reduce_lighthouse(raw_lh): + return {preset: {page: _lighthouse_page(runs) + for page, runs in (raw_lh.get(preset) or {}).items()} + for preset in PRESETS} + + +def reduce_axe(raw_axe): + out = {} + for page, data in raw_axe.items(): + if _is_error(data): + out[page] = _error(data) + continue + counts = {impact: 0 for impact in IMPACTS} + for v in data.get('violations') or []: + if v.get('impact') in counts: + counts[v['impact']] += 1 + out[page] = {'violations': counts, 'total': len(data.get('violations') or []), + 'rules': sorted({v['id'] for v in data.get('violations') or []})} + return out + + +def reduce_html(raw_html): + out = {} + for page, data in raw_html.items(): + if _is_error(data): + out[page] = _error(data) + continue + messages = data.get('messages') or [] + out[page] = {'errors': sum(1 for m in messages if m.get('severity') == 2), + 'warnings': sum(1 for m in messages if m.get('severity') == 1), + 'rules': sorted({m['ruleId'] for m in messages if m.get('ruleId')})} + return out + + +def _kind(url): + if '#' in url: + return 'anchor' + path = url.split('?', 1)[0] + if path.endswith('/') or path.endswith('.html') or not posixpath.splitext(path)[1]: + return 'link' + return 'asset' + + +def reduce_links(raw_links): + report = raw_links.get('report') or {} + fail_map = report.get('fail_map') or report.get('error_map') or {} + items = [] + for source, failures in sorted(fail_map.items()): + for f in failures: + status = f.get('status') + items.append({'source': source, 'url': f.get('url'), + 'status': status.get('text') if isinstance(status, dict) else status}) + by_kind = {'link': 0, 'anchor': 0, 'asset': 0} + for item in items: + by_kind[_kind(item['url'] or '')] += 1 + return {'broken': len(items), 'byKind': by_kind, 'items': items[:MAX_BROKEN_LISTED]} + + +def weight(files, output_glob): + posts = [f['bytes'] for f in files if fnmatch.fnmatch(f['path'], output_glob)] + total = sum(f['bytes'] for f in files) + reference = sum(f['bytes'] for f in files if f['path'] in site.REFERENCE_ASSETS) + return {'bytes': total, + 'gzipBytes': sum(f['gzip'] for f in files), + 'brotliBytes': sum(f['br'] for f in files), + 'bytesPerPost': round(sum(posts) / len(posts)) if posts else None, + 'generatorAddedBytes': total - reference} + + +def js_section(lighthouse, files, works): + pages = {} + for page, data in (lighthouse.get('mobile') or {}).items(): + js = (data.get('resources') or {}).get('jsBytes') if data.get('status') == 'ok' else None + pages[page] = js['median'] if js else None + known = [v for v in pages.values() if v is not None] + js_class = None if not known else ('none' if all(v == 0 for v in known) else 'hydrated') + return {'pages': pages, + 'diskBytes': sum(f['bytes'] for f in files if f['path'].endswith(('.js', '.mjs'))), + 'jsClass': js_class, + 'worksWithoutJs': works} + + +def _raw(raw, key): + part = raw.get(key) + if part is None: + raise RuntimeError('%s missing in collector output' % key) + if _is_error(part): + raise RuntimeError(part['error']) + return part + + +def _section(fn): + try: + return dict({'status': 'ok'}, **fn()) + except Exception as e: + return {'status': 'error', 'error': '%s: %s' % (type(e).__name__, e)} + + +def build_quality(raw, site_dir, pages, output_glob): + '''quality.json (R-16): one section per check, each ok or error.''' + q = {'status': 'ok', 'pages': {k: v['url'] for k, v in pages.items()}, + 'tools': raw.get('tools')} + q['lighthouse'] = _section(lambda: reduce_lighthouse(_raw(raw, 'lighthouse'))) + q['seo'] = _section(lambda: seo.seo_section(site_dir, pages)) + q['html'] = _section(lambda: {'pages': reduce_html(_raw(raw, 'html'))}) + q['a11y'] = _section(lambda: {'pages': reduce_axe(_raw(raw, 'axe'))}) + q['links'] = _section(lambda: reduce_links(_raw(raw, 'links'))) + q['weight'] = _section(lambda: weight(_raw(raw, 'files'), output_glob)) + + def js(): + if q['lighthouse']['status'] != 'ok': + raise RuntimeError('lighthouse section failed') + rendered = (raw.get('rendered') or {}).get('post') if not _is_error(raw.get('rendered')) else None + works = site.works_without_js(site_dir, pages['post']['file'], rendered) + return js_section(q['lighthouse'], _raw(raw, 'files'), works) + + q['js'] = _section(js) + q['extra'] = _section(lambda: site.extra_output(site_dir, output_glob)) + return q +``` + +- [ ] **Step 5: Run the tests** + +Run: `python3 -m pytest -q tests/test_quality_reduce.py && ruff check toolset tests` +Expected: all pass. + +- [ ] **Step 6: Commit** + +```bash +git add toolset/quality/reduce.py tests/test_quality_reduce.py tests/fixtures/quality/raw.json +git commit -m "feat(quality): reduce collector output into quality.json (spec 009) + +Co-Authored-By: Claude Opus 5.5 (1M context) " +``` + +--- + +### Task 4: Collector image (`quality/`) + +**Files:** +- Create: `quality/Dockerfile`, `quality/package.json`, `quality/package-lock.json`, `quality/run.mjs`, `quality/.dockerignore` +- Create: `quality/test/pages.json`, `quality/test/expected-shape.json`, `quality/test/check-shape.mjs` +- Test: the image itself (Step 7). It needs Docker; take the Docker lock in `/Users/jobs/Dev/ssgberk/.docker-smoke.lock` with `mkdir`/`rmdir` if working on the shared machine, and check for ≥5 GiB free disk first. + +**Interfaces:** +- Consumes: the fixture site (Task 1). +- Produces: the image `ssgberk/quality` and the collector CLI and `raw.json` contracts in `plan.md`. + +- [ ] **Step 1: Pin the versions** + +Resolve the latest stable versions, and record them in the commit message: + +```bash +for p in lighthouse chrome-launcher playwright axe-core html-validate; do echo "$p $(npm view $p version)"; done +gh release view --repo lycheeverse/lychee --json tagName,assets --jq '.tagName, [.assets[].name]' +``` + +`quality/package.json` (replace each `X.Y.Z` with the resolved version, exact, no `^`): + +```json +{ + "name": "ssgberk-quality", + "private": true, + "type": "module", + "description": "SSGBerk output quality collector (benchmark-tool spec 009)", + "dependencies": { + "axe-core": "X.Y.Z", + "chrome-launcher": "X.Y.Z", + "html-validate": "X.Y.Z", + "lighthouse": "X.Y.Z", + "playwright": "X.Y.Z" + } +} +``` + +Then generate the lock: `cd quality && npm install --package-lock-only --ignore-scripts && cd ..`. If `npm` is not on the host, run the same command in `docker run --rm -v "$PWD/quality:/q" -w /q node:24 ...`. + +`quality/.dockerignore`: + +``` +node_modules +test +``` + +- [ ] **Step 2: Write `quality/Dockerfile`** + +Set `LYCHEE_VERSION` to the resolved release tag without the `lychee-v` or `v` prefix. Check that the asset names match `lychee--unknown-linux-gnu.tar.gz`, and adjust the URL if the release uses another name. + +```dockerfile +# SSGBerk quality collector (benchmark-tool spec 009): Lighthouse, axe-core, +# html-validate and lychee against Playwright's Chromium. Runs with no network. +FROM ubuntu:24.04 + +ARG DEBIAN_FRONTEND=noninteractive +ARG NODE_VERSION=24.21.0 +ARG LYCHEE_VERSION=X.Y.Z + +RUN apt-get -yqq update \ + && apt-get -yqq install --no-install-recommends ca-certificates curl xz-utils \ + && rm -rf /var/lib/apt/lists/* + +RUN ARCH="$(dpkg --print-architecture)" \ + && case "$ARCH" in amd64) NARCH=x64 ;; arm64) NARCH=arm64 ;; *) echo "unsupported $ARCH"; exit 1 ;; esac \ + && curl -fsSL "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${NARCH}.tar.xz" \ + | tar -xJ -C /usr/local --strip-components=1 \ + && node --version + +RUN ARCH="$(dpkg --print-architecture)" \ + && case "$ARCH" in amd64) LARCH=x86_64 ;; arm64) LARCH=aarch64 ;; *) echo "unsupported $ARCH"; exit 1 ;; esac \ + && curl -fsSL "https://github.com/lycheeverse/lychee/releases/download/lychee-v${LYCHEE_VERSION}/lychee-${LARCH}-unknown-linux-gnu.tar.gz" \ + | tar -xz -C /usr/local/bin lychee \ + && lychee --version + +ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright NPM_CONFIG_UPDATE_NOTIFIER=false +WORKDIR /quality +COPY package.json package-lock.json ./ +RUN npm ci --omit=dev \ + && npx playwright install --with-deps chromium \ + && rm -rf /root/.npm /var/lib/apt/lists/* +COPY run.mjs ./ +RUN mkdir -p /work/out +WORKDIR /work +``` + +- [ ] **Step 3: Write `quality/run.mjs`** + +```js +// SSGBerk quality collector (benchmark-tool spec 009). Writes raw data only: +// toolset/quality/reduce.py turns raw.json into quality.json. +import { execFile } from 'node:child_process'; +import fs from 'node:fs/promises'; +import http from 'node:http'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs, promisify } from 'node:util'; +import zlib from 'node:zlib'; + +import * as chromeLauncher from 'chrome-launcher'; +import { HtmlValidate } from 'html-validate'; +import lighthouse from 'lighthouse'; +import desktopConfig from 'lighthouse/core/config/desktop-config.js'; +import { chromium } from 'playwright'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const RUNS = 3; +const PRESETS = { mobile: undefined, desktop: desktopConfig }; +const CATEGORIES = ['performance', 'accessibility', 'best-practices', 'seo']; +const CHROME_FLAGS = ['--headless=new', '--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage']; +const NOT_SCORED = new Set(['informative', 'manual', 'notApplicable', 'error']); +const TYPES = { + '.html': 'text/html; charset=utf-8', '.css': 'text/css; charset=utf-8', + '.js': 'text/javascript; charset=utf-8', '.mjs': 'text/javascript; charset=utf-8', + '.json': 'application/json', '.xml': 'application/xml', '.txt': 'text/plain; charset=utf-8', + '.svg': 'image/svg+xml', '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', + '.webp': 'image/webp', '.gif': 'image/gif', '.ico': 'image/x-icon', '.woff': 'font/woff', + '.woff2': 'font/woff2', '.wasm': 'application/wasm', +}; +const COMPRESSIBLE = /^(text\/|application\/(json|xml|wasm)|image\/svg\+xml)/; +const execFileP = promisify(execFile); + +async function guard(fn) { + try { + return await fn(); + } catch (e) { + return { error: String(e && e.message ? e.message : e) }; + } +} + +async function isFile(p) { + try { + return (await fs.stat(p)).isFile(); + } catch { + return false; + } +} + +async function resolveFile(root, pathname) { + const rel = decodeURIComponent(pathname).replace(/^\/+/, ''); + const base = path.resolve(root, rel); + if (base !== root && !base.startsWith(root + path.sep)) return null; + const candidates = rel === '' || rel.endsWith('/') + ? [path.join(base, 'index.html')] + : [base, path.join(base, 'index.html'), `${base}.html`]; + for (const c of candidates) if (await isFile(c)) return c; + return null; +} + +// Fixed static server: gzip, no-cache, directory index, 404.html with status 404. +function serve(root) { + const server = http.createServer(async (req, res) => { + const { pathname } = new URL(req.url, 'http://127.0.0.1'); + let file = await resolveFile(root, pathname); + let status = 200; + if (!file) { + status = 404; + file = (await isFile(path.join(root, '404.html'))) ? path.join(root, '404.html') : null; + } + let body = file ? await fs.readFile(file) : Buffer.from('Not found'); + const type = file ? TYPES[path.extname(file).toLowerCase()] || 'application/octet-stream' + : 'text/plain; charset=utf-8'; + const headers = { 'content-type': type, 'cache-control': 'no-cache' }; + if (COMPRESSIBLE.test(type) && /\bgzip\b/.test(req.headers['accept-encoding'] || '')) { + body = zlib.gzipSync(body); + headers['content-encoding'] = 'gzip'; + headers.vary = 'accept-encoding'; + } + headers['content-length'] = body.length; + res.writeHead(status, headers); + res.end(req.method === 'HEAD' ? undefined : body); + }); + return new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve({ + server, origin: `http://127.0.0.1:${server.address().port}`, + }))); +} + +function summarize(lhr) { + const audits = lhr.audits; + const value = (id) => audits[id]?.numericValue ?? null; + const requests = audits['network-requests']?.details?.items ?? []; + const bytes = (type) => requests.filter((r) => r.resourceType === type) + .reduce((sum, r) => sum + (r.transferSize || 0), 0); + return { + scores: Object.fromEntries(CATEGORIES.map((c) => [c, lhr.categories[c]?.score ?? null])), + metrics: { + fcp: value('first-contentful-paint'), + lcp: value('largest-contentful-paint'), + tbt: value('total-blocking-time'), + cls: value('cumulative-layout-shift'), + si: value('speed-index'), + }, + resources: { + totalBytes: value('total-byte-weight'), + jsBytes: bytes('Script'), + cssBytes: bytes('Stylesheet'), + requests: requests.length, + domSize: value('dom-size') ?? value('dom-size-insight'), + }, + failedAudits: Object.values(audits) + .filter((a) => !NOT_SCORED.has(a.scoreDisplayMode) && a.score !== null && a.score < 1) + .map((a) => a.id).sort(), + }; +} + +async function runLighthouse(origin, pages) { + const chrome = await chromeLauncher.launch({ + chromePath: chromium.executablePath(), chromeFlags: CHROME_FLAGS, + }); + try { + const out = {}; + for (const [preset, config] of Object.entries(PRESETS)) { + out[preset] = {}; + for (const [key, page] of Object.entries(pages)) { + const runs = []; + for (let i = 0; i < RUNS; i += 1) { + runs.push(await guard(async () => { + const result = await lighthouse(origin + page.url, { + port: chrome.port, output: 'json', logLevel: 'error', onlyCategories: CATEGORIES, + }, config); + if (!result?.lhr) throw new Error('lighthouse returned no report'); + if (result.lhr.runtimeError) throw new Error(result.lhr.runtimeError.message); + return summarize(result.lhr); + })); + } + out[preset][key] = runs; + } + } + return out; + } finally { + await chrome.kill(); + } +} + +async function runBrowserChecks(origin, pages) { + const axeSource = await fs.readFile(path.join(HERE, 'node_modules', 'axe-core', 'axe.min.js'), 'utf8'); + const browser = await chromium.launch({ args: ['--no-sandbox', '--disable-dev-shm-usage'] }); + try { + const axe = {}; + for (const [key, page] of Object.entries(pages)) { + axe[key] = await guard(async () => { + const tab = await browser.newPage(); + try { + await tab.goto(origin + page.url, { waitUntil: 'load' }); + await tab.addScriptTag({ content: axeSource }); + const result = await tab.evaluate(() => window.axe.run(document, { resultTypes: ['violations'] })); + return { + violations: result.violations.map((v) => ({ id: v.id, impact: v.impact, nodes: v.nodes.length })), + }; + } finally { + await tab.close(); + } + }); + } + const post = await guard(async () => { + const tab = await browser.newPage(); + try { + await tab.goto(origin + pages.post.url, { waitUntil: 'networkidle' }); + const text = async (selector) => ((await tab.locator(selector).first() + .textContent({ timeout: 5000 })) || '').trim(); + return { title: await text('h1.post-title'), firstParagraph: await text('.post-body p') }; + } finally { + await tab.close(); + } + }); + return { axe, rendered: { post }, chromium: browser.version() }; + } finally { + await browser.close(); + } +} + +async function runHtmlValidate(site, pages) { + const validator = new HtmlValidate({ extends: ['html-validate:recommended'] }); + const out = {}; + for (const [key, page] of Object.entries(pages)) { + out[key] = await guard(async () => { + const report = await validator.validateFile(path.join(site, page.file)); + return { + messages: report.results.flatMap((r) => r.messages) + .map((m) => ({ ruleId: m.ruleId, severity: m.severity })), + }; + }); + } + return out; +} + +async function runLychee(site, outDir) { + const report = path.join(outDir, 'lychee.json'); + try { + await execFileP('lychee', ['--offline', '--no-progress', '--include-fragments', '--format', 'json', + '--output', report, '--root-dir', site, site], { maxBuffer: 64 * 1024 * 1024 }); + } catch (e) { + // exit code 2 means broken links; the report is still written + if (!(await isFile(report))) throw e; + } + return { report: JSON.parse(await fs.readFile(report, 'utf8')) }; +} + +async function listFiles(dir) { + const out = []; + for (const entry of await fs.readdir(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) out.push(...await listFiles(full)); + else if (entry.isFile()) out.push(full); + } + return out; +} + +async function sizes(site) { + const files = []; + for (const full of (await listFiles(site)).sort()) { + const data = await fs.readFile(full); + files.push({ + path: path.relative(site, full).split(path.sep).join('/'), + bytes: data.length, + gzip: zlib.gzipSync(data, { level: 9 }).length, + br: zlib.brotliCompressSync(data, { + params: { [zlib.constants.BROTLI_PARAM_QUALITY]: 11 }, + }).length, + }); + } + return files; +} + +async function packageVersion(name) { + const file = path.join(HERE, 'node_modules', name, 'package.json'); + return JSON.parse(await fs.readFile(file, 'utf8')).version; +} + +async function tools(chromiumVersion) { + const lychee = await guard(async () => (await execFileP('lychee', ['--version'])).stdout.trim() + .split(/\s+/).pop()); + return { + node: process.version, + lighthouse: await packageVersion('lighthouse'), + playwright: await packageVersion('playwright'), + chromium: chromiumVersion ?? null, + axeCore: await packageVersion('axe-core'), + htmlValidate: await packageVersion('html-validate'), + chromeLauncher: await packageVersion('chrome-launcher'), + lychee: typeof lychee === 'string' ? lychee : null, + }; +} + +async function main() { + const { values } = parseArgs({ + options: { site: { type: 'string' }, pages: { type: 'string' }, out: { type: 'string' } }, + }); + const site = path.resolve(values.site); + const pages = JSON.parse(values.pages); + const outDir = path.resolve(values.out); + await fs.mkdir(outDir, { recursive: true }); + const raw = { pages }; + const { server, origin } = await serve(site); + try { + const browser = await guard(() => runBrowserChecks(origin, pages)); + raw.axe = browser.error ? browser : browser.axe; + raw.rendered = browser.error ? browser : browser.rendered; + raw.lighthouse = await guard(() => runLighthouse(origin, pages)); + raw.html = await guard(() => runHtmlValidate(site, pages)); + raw.links = await guard(() => runLychee(site, outDir)); + raw.files = await guard(() => sizes(site)); + raw.tools = await guard(() => tools(browser.chromium)); + } finally { + server.close(); + } + await fs.writeFile(path.join(outDir, 'raw.json'), JSON.stringify(raw, null, 2)); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); +``` + +- [ ] **Step 4: Write the collector test files** + +`quality/test/pages.json`: + +```json +{"index": {"url": "/", "file": "index.html"}, "post": {"url": "/post/hello/", "file": "post/hello/index.html"}, "404": {"url": "/404.html", "file": "404.html"}} +``` + +`quality/test/expected-shape.json`. A string is a type (`"a|b"` allows either), an object is checked key by key, and a one-element array means every element must match that element: + +```json +{ + "tools": {"node": "string", "lighthouse": "string", "playwright": "string", "chromium": "string", + "axeCore": "string", "htmlValidate": "string", "chromeLauncher": "string", "lychee": "string"}, + "pages": "object", + "lighthouse": { + "mobile": { + "index": [{"scores": {"performance": "number", "accessibility": "number", "best-practices": "number", "seo": "number"}, + "metrics": {"fcp": "number", "lcp": "number", "tbt": "number", "cls": "number", "si": "number"}, + "resources": {"totalBytes": "number", "jsBytes": "number", "cssBytes": "number", "requests": "number", "domSize": "number|null"}, + "failedAudits": "array"}], + "post": [{"scores": {"performance": "number", "accessibility": "number", "best-practices": "number", "seo": "number"}, + "metrics": {"fcp": "number", "lcp": "number", "tbt": "number", "cls": "number", "si": "number"}, + "resources": {"totalBytes": "number", "jsBytes": "number", "cssBytes": "number", "requests": "number", "domSize": "number|null"}, + "failedAudits": "array"}], + "404": [{"scores": {"performance": "number", "accessibility": "number", "best-practices": "number", "seo": "number"}, + "metrics": {"fcp": "number", "lcp": "number", "tbt": "number", "cls": "number", "si": "number"}, + "resources": {"totalBytes": "number", "jsBytes": "number", "cssBytes": "number", "requests": "number", "domSize": "number|null"}, + "failedAudits": "array"}] + }, + "desktop": { + "index": [{"scores": {"performance": "number", "accessibility": "number", "best-practices": "number", "seo": "number"}, + "metrics": {"fcp": "number", "lcp": "number", "tbt": "number", "cls": "number", "si": "number"}, + "resources": {"totalBytes": "number", "jsBytes": "number", "cssBytes": "number", "requests": "number", "domSize": "number|null"}, + "failedAudits": "array"}], + "post": [{"scores": {"performance": "number", "accessibility": "number", "best-practices": "number", "seo": "number"}, + "metrics": {"fcp": "number", "lcp": "number", "tbt": "number", "cls": "number", "si": "number"}, + "resources": {"totalBytes": "number", "jsBytes": "number", "cssBytes": "number", "requests": "number", "domSize": "number|null"}, + "failedAudits": "array"}], + "404": [{"scores": {"performance": "number", "accessibility": "number", "best-practices": "number", "seo": "number"}, + "metrics": {"fcp": "number", "lcp": "number", "tbt": "number", "cls": "number", "si": "number"}, + "resources": {"totalBytes": "number", "jsBytes": "number", "cssBytes": "number", "requests": "number", "domSize": "number|null"}, + "failedAudits": "array"}] + } + }, + "axe": {"index": {"violations": "array"}, "post": {"violations": "array"}, "404": {"violations": "array"}}, + "rendered": {"post": {"title": "string", "firstParagraph": "string"}}, + "html": {"index": {"messages": "array"}, "post": {"messages": "array"}, "404": {"messages": "array"}}, + "links": {"report": "object"}, + "files": [{"path": "string", "bytes": "number", "gzip": "number", "br": "number"}] +} +``` + +`quality/test/check-shape.mjs`: + +```js +// Checks a collector raw.json against expected-shape.json (keys and types, not values). +import fs from 'node:fs'; + +const [rawPath, shapePath] = process.argv.slice(2); +const raw = JSON.parse(fs.readFileSync(rawPath, 'utf8')); +const shape = JSON.parse(fs.readFileSync(shapePath, 'utf8')); +const errors = []; + +function typeOf(v) { + if (v === null) return 'null'; + if (Array.isArray(v)) return 'array'; + return typeof v; +} + +function check(value, expected, where) { + if (typeof expected === 'string') { + if (!expected.split('|').includes(typeOf(value))) errors.push(`${where}: ${typeOf(value)} is not ${expected}`); + } else if (Array.isArray(expected)) { + if (!Array.isArray(value) || value.length === 0) errors.push(`${where}: expected a non-empty array`); + else value.forEach((v, i) => check(v, expected[0], `${where}[${i}]`)); + } else { + if (typeOf(value) !== 'object') { + errors.push(`${where}: expected an object, got ${JSON.stringify(value).slice(0, 200)}`); + return; + } + for (const [k, e] of Object.entries(expected)) check(value[k], e, `${where}.${k}`); + } +} + +check(raw, shape, 'raw'); +for (const preset of ['mobile', 'desktop']) { + for (const page of ['index', 'post', '404']) { + if ((raw.lighthouse?.[preset]?.[page] || []).length !== 3) errors.push(`lighthouse.${preset}.${page}: expected 3 runs`); + } +} +if (raw.rendered?.post?.title !== 'Hello') errors.push('rendered.post.title is not "Hello"'); +if (raw.lighthouse?.mobile?.post?.[0]?.resources?.jsBytes !== 0) errors.push('fixture post page should request 0 JS bytes'); +if (errors.length) { + console.error(errors.join('\n')); + process.exit(1); +} +console.log('raw.json shape ok'); +``` + +- [ ] **Step 5: Build the image** + +Run: `docker build -t ssgberk/quality quality` +Expected: the build succeeds, and both `node --version` and `lychee --version` print. + +- [ ] **Step 6: Run the collector on the fixture and check the shape** + +```bash +docker run --rm --network none \ + -v "$PWD/tests/fixtures/quality/site:/fixture:ro" -v "$PWD/quality/test:/test:ro" \ + ssgberk/quality sh -c 'node /quality/run.mjs --site /fixture --pages "$(cat /test/pages.json)" --out /tmp/out \ + && node /test/check-shape.mjs /tmp/out/raw.json /test/expected-shape.json' +``` + +Expected: `raw.json shape ok`. If a check fails because the pinned version renamed something (a Lighthouse audit id such as `network-requests` or `dom-size`, a lychee flag or its JSON keys, or the `html-validate` API), fix `run.mjs` for the pinned version. If the lychee JSON uses keys other than `fail_map` or `error_map`, update `reduce.reduce_links` and the `links` part of `tests/fixtures/quality/raw.json` to match. Re-run until it passes. Keep the fixture site unchanged. + +- [ ] **Step 7: Remove the local image and commit** + +```bash +docker rmi ssgberk/quality +git add quality +git commit -m "feat(quality): pinned collector image with Lighthouse, axe-core, html-validate, lychee (spec 009 R-4, R-5) + +Pinned: lighthouse X, chrome-launcher X, playwright X (Chromium X), axe-core X, +html-validate X, lychee X, Node 24.21.0. + +Co-Authored-By: Claude Opus 5.5 (1M context) " +``` + +--- + +### Task 5: Docker plumbing (`toolset/utils/docker_helper.py`) + +**Files:** +- Modify: `toolset/utils/docker_helper.py` + - imports at the top: `io`, `json`, `posixpath`, `tarfile`; + - class attributes next to `last_build`; + - `benchmark()` (signature and the end of its `try` block); + - new methods after `benchmark()`. +- Test: `tests/test_quality_docker.py` + +**Interfaces:** +- Consumes: nothing from earlier tasks (pure Docker plumbing). +- Produces: + - `QUALITY_IMAGE`, `QUALITY_TIMEOUT_SECONDS` + - `DockerHelper.benchmark(framework_test, script, variables, raw_file, resources, timeout_seconds, export=None)` + - `DockerHelper.build_quality_image() -> str` + - `DockerHelper.run_quality(tar_path, site_name, pages, timeout_seconds=QUALITY_TIMEOUT_SECONDS) -> dict` + +- [ ] **Step 1: Write the failing tests** + +`tests/test_quality_docker.py`: + +```python +import io +import json +import tarfile +import types +from unittest import mock + +import docker +import pytest + +from toolset.utils import docker_helper + +RES = {"cpus": 4.0, "memoryBytes": 8589934592, "swap": False, "cpuset": "4-7"} + + +def _helper(): + helper = docker_helper.DockerHelper.__new__(docker_helper.DockerHelper) + helper.benchmarker = mock.Mock() + helper.server = mock.Mock() + return helper + + +def _container(exit_code=0, oom=False): + c = mock.Mock() + c.logs.return_value = iter(()) + c.wait.return_value = {"StatusCode": exit_code} + c.attrs = {"State": {"OOMKilled": oom, "ExitCode": exit_code}, + "Config": {"WorkingDir": "/opt/hugo/src"}} + c.get_archive.return_value = (iter([b"tar", b"bytes"]), {}) + return c + + +def test_benchmark_exports_output_on_success(tmp_path): + helper, c = _helper(), _container() + helper.server.containers.run.return_value = c + dest = tmp_path / "quality" / "hugo" / "site.tar" + out = helper.benchmark(types.SimpleNamespace(name="hugo"), "build.sh", {}, str(tmp_path / "raw.txt"), + RES, 60, export=("public", str(dest))) + assert out["status"] == "ok" + c.get_archive.assert_called_once_with("/opt/hugo/src/public") + assert dest.read_bytes() == b"tarbytes" + c.remove.assert_called_once_with(force=True) + + +@pytest.mark.parametrize("exit_code,oom", [(1, False), (137, True)]) +def test_benchmark_no_export_on_failure(tmp_path, exit_code, oom): + helper, c = _helper(), _container(exit_code, oom) + helper.server.containers.run.return_value = c + helper.benchmark(types.SimpleNamespace(name="hugo"), "build.sh", {}, str(tmp_path / "raw.txt"), + RES, 60, export=("public", str(tmp_path / "site.tar"))) + c.get_archive.assert_not_called() + + +def test_benchmark_export_error_keeps_status(tmp_path): + helper, c = _helper(), _container() + c.get_archive.side_effect = docker.errors.NotFound("no such path") + helper.server.containers.run.return_value = c + out = helper.benchmark(types.SimpleNamespace(name="hugo"), "build.sh", {}, str(tmp_path / "raw.txt"), + RES, 60, export=("public", str(tmp_path / "site.tar"))) + assert out == {"status": "ok", "exitCode": 0} + + +def test_benchmark_without_export_never_reads_archive(tmp_path): + helper, c = _helper(), _container() + helper.server.containers.run.return_value = c + helper.benchmark(types.SimpleNamespace(name="hugo"), "build.sh", {}, str(tmp_path / "raw.txt"), RES, 60) + c.get_archive.assert_not_called() + + +def _raw_tar(payload): + buf = io.BytesIO() + data = json.dumps(payload).encode() + with tarfile.open(fileobj=buf, mode="w") as tar: + info = tarfile.TarInfo("raw.json") + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + return buf.getvalue() + + +def test_run_quality_isolated_and_cleaned_up(tmp_path): + helper = _helper() + helper._quality_image = "sha256:q" + c = mock.Mock() + c.wait.return_value = {"StatusCode": 0} + c.logs.return_value = b"" + c.get_archive.return_value = (iter([_raw_tar({"tools": {"node": "v24"}})]), {}) + helper.server.containers.create.return_value = c + tar_path = tmp_path / "site.tar" + tar_path.write_bytes(b"site-tar") + pages = {"index": {"url": "/", "file": "index.html"}} + raw = helper.run_quality(str(tar_path), "public", pages) + assert raw == {"tools": {"node": "v24"}} + args, kwargs = helper.server.containers.create.call_args + assert args[0] == "sha256:q" + assert kwargs["network_mode"] == "none" + assert "volumes" not in kwargs and "mounts" not in kwargs + assert kwargs["command"][:4] == ["node", "/quality/run.mjs", "--site", "/work/public"] + assert json.loads(kwargs["command"][kwargs["command"].index("--pages") + 1]) == pages + c.put_archive.assert_called_once_with("/work", b"site-tar") + c.get_archive.assert_called_once_with("/work/out/raw.json") + c.remove.assert_called_once_with(force=True) + + +def test_run_quality_without_raw_json_raises_and_cleans_up(tmp_path): + helper = _helper() + helper._quality_image = "sha256:q" + c = mock.Mock() + c.wait.return_value = {"StatusCode": 1} + c.logs.return_value = b"Error: boom" + c.get_archive.side_effect = docker.errors.NotFound("missing") + helper.server.containers.create.return_value = c + (tmp_path / "site.tar").write_bytes(b"x") + with pytest.raises(RuntimeError, match="exit 1.*boom"): + helper.run_quality(str(tmp_path / "site.tar"), "public", {}) + c.remove.assert_called_once_with(force=True) + + +def test_build_quality_image_once(monkeypatch, tmp_path): + builds = [] + + class FakeAPIClient: + def __init__(self, base_url=None): + pass + + def build(self, **kw): + builds.append(kw) + return iter([{"stream": "Step 1/2 : FROM ubuntu:24.04\n"}, {"stream": "Successfully built abc\n"}]) + + monkeypatch.setattr(docker_helper.docker, "APIClient", FakeAPIClient) + helper = _helper() + helper._quality_image = None + helper.benchmarker.config = types.SimpleNamespace(fw_root=str(tmp_path), server_docker_host=None) + helper.server.images.get.return_value = types.SimpleNamespace(id="sha256:abc") + assert helper.build_quality_image() == "sha256:abc" + assert helper.build_quality_image() == "sha256:abc" + assert len(builds) == 1 + assert builds[0]["path"] == str(tmp_path / "quality") and builds[0]["tag"] == docker_helper.QUALITY_IMAGE + + +def test_build_quality_image_error(monkeypatch, tmp_path): + class FakeAPIClient: + def __init__(self, base_url=None): + pass + + def build(self, **kw): + return iter([{"errorDetail": {"message": "npm ci failed"}}]) + + monkeypatch.setattr(docker_helper.docker, "APIClient", FakeAPIClient) + helper = _helper() + helper._quality_image = None + helper.benchmarker.config = types.SimpleNamespace(fw_root=str(tmp_path), server_docker_host=None) + with pytest.raises(RuntimeError, match="npm ci failed"): + helper.build_quality_image() +``` + +- [ ] **Step 2: Run them and see them fail** + +Run: `python3 -m pytest -q tests/test_quality_docker.py` +Expected: FAIL with `TypeError: ... unexpected keyword argument 'export'` and `AttributeError: ... run_quality`. + +- [ ] **Step 3: Implement the changes** + +At the top of `toolset/utils/docker_helper.py`, add `import io`, `import json`, `import posixpath` and `import tarfile` with the other stdlib imports. Next to `RUN_LABEL`, add: + +```python +QUALITY_IMAGE = 'ssgberk/quality' +QUALITY_TIMEOUT_SECONDS = 1800 +QUALITY_MEMORY = '4g' +``` + +In the class body, next to `last_build = None`, add `_quality_image = None`. + +Change the signature of `benchmark` to: + +```python + def benchmark(self, framework_test, script, variables, raw_file, + resources, timeout_seconds, export=None): + ''' + Runs the generator container with fixed resource limits and returns + {"status": "ok"|"timeout"|"oom", "exitCode": int}. The container is + always stopped and removed before returning. With export + (output_folder, dest_tar), a successful build's output is copied to + dest_tar first (spec 009 R-3). + ''' +``` + +At the end of its `try:` block, right after the OOM check and before `finally:`, insert: + +```python + if export and status == 'ok' and exit_code == 0: + DockerHelper.__export_output(container, *export) +``` + +Add these methods after `benchmark`: + +```python + @staticmethod + def __export_output(container, output_folder, dest): + '''Copies / to dest as a tar. Never raises.''' + try: + workdir = (container.attrs.get('Config') or {}).get('WorkingDir') or '/' + bits, _ = container.get_archive(posixpath.join(workdir, output_folder)) + os.makedirs(os.path.dirname(dest), exist_ok=True) + with open(dest, 'wb') as f: + for chunk in bits: + f.write(chunk) + except Exception as e: + log("quality: could not export %s: %s" % (output_folder, e), color=Fore.YELLOW) + + def build_quality_image(self): + ''' + Builds ssgberk/quality from /quality once per process + (spec 009 R-4), outside the generator build-time accounting. + ''' + if self._quality_image: + return self._quality_image + client = docker.APIClient(base_url=self.benchmarker.config.server_docker_host) + for token in client.build(path=os.path.join(self.benchmarker.config.fw_root, 'quality'), + dockerfile='Dockerfile', tag=QUALITY_IMAGE, forcerm=True, + pull=True, timeout=3600, decode=True): + if 'errorDetail' in token: + raise RuntimeError(token['errorDetail']['message']) + if token.get('stream', '').strip(): + log(token['stream'].rstrip(), prefix='quality: ') + self._quality_image = self.server.images.get(QUALITY_IMAGE).id + return self._quality_image + + def run_quality(self, tar_path, site_name, pages, + timeout_seconds=QUALITY_TIMEOUT_SECONDS): + ''' + Runs quality/run.mjs on the exported site with no network and no + mounts (spec 009 R-4, R-5) and returns its raw.json. + ''' + image = self.build_quality_image() + command = ['node', '/quality/run.mjs', '--site', '/work/' + site_name, + '--pages', json.dumps(pages), '--out', '/work/out'] + container = self.server.containers.create( + image, command=command, labels=self._run_labels(), network_mode='none', + mem_limit=QUALITY_MEMORY, working_dir='/work') + try: + with open(tar_path, 'rb') as f: + container.put_archive('/work', f.read()) + container.start() + result = container.wait(timeout=timeout_seconds) + try: + bits, _ = container.get_archive('/work/out/raw.json') + except docker.errors.NotFound: + logs = container.logs().decode('utf-8', 'replace') + raise RuntimeError('collector wrote no raw.json (exit %s): %s' % ( + result.get('StatusCode'), logs[-2000:])) + with tarfile.open(fileobj=io.BytesIO(b''.join(bits))) as tar: + return json.load(tar.extractfile(tar.getmembers()[0])) + finally: + try: + container.remove(force=True) + except docker.errors.NotFound: + pass +``` + +- [ ] **Step 4: Run all tests** + +Run: `python3 -m pytest -q && ruff check toolset tests` +Expected: all pass, including the existing `tests/test_docker_helper.py`. + +- [ ] **Step 5: Commit** + +```bash +git add toolset/utils/docker_helper.py tests/test_quality_docker.py +git commit -m "feat(quality): export build output and run the collector without network (spec 009 R-3..R-5) + +Co-Authored-By: Claude Opus 5.5 (1M context) " +``` + +--- + +### Task 6: The pass in a run (runner, CLI, Benchmarker, Results) + +**Files:** +- Create: `toolset/quality/runner.py` +- Modify: + - `toolset/run-tests.py`: a new `--quality` argument after `--no-cache`; + - `toolset/utils/benchmark_config.py`: `self.quality` next to `self.no_cache`; + - `toolset/benchmark/benchmarker.py`: `benchmark_type` in `__benchmark`, `__noise_control`, and two new private methods; + - `toolset/utils/results.py`: `STORED_RUN_KEYS`, `__init__`, `add_quality`, `__to_jsonable`, `write_summary`. +- Test: `tests/test_quality_runner.py` + +**Interfaces:** +- Consumes: + - `site.generator_config`, `site.extract`, `site.resolve_pages` (Task 1); + - `reduce.build_quality` (Task 3); + - `DockerHelper.run_quality`, `DockerHelper.benchmark(..., export=...)`, `DockerHelper._quality_image` (Task 5). +- Produces: + - `runner.eligible(config)`, `runner.skipped(config)`, `runner.archive_path(results_dir, name)`, `runner.run_pass(docker_helper, test, results_dir)` + - `Results.add_quality(name, quality, image_id=None)` + - `results.json` `quality` and `environment.quality` + - `config.quality` + +- [ ] **Step 1: Write the failing tests** + +`tests/test_quality_runner.py`: + +```python +import json +import pathlib +import tarfile +import types +from unittest import mock + +from toolset.quality import runner +from toolset.utils.results import STORED_RUN_KEYS, Results + +FIX = pathlib.Path(__file__).parent / "fixtures" / "quality" + + +def _cfg(**kw): + return types.SimpleNamespace(**kw) + + +def test_eligible_and_skipped(): + assert not runner.eligible(_cfg()) + assert not runner.skipped(_cfg()) + assert runner.eligible(_cfg(quality=True, suite_info=None, number_of_files="10", content_size="0.500")) + p0 = _cfg(quality=True, suite_info={"name": "P"}, number_of_files="50", content_size="5") + p1 = _cfg(quality=True, suite_info={"name": "P"}, number_of_files="50", content_size="50") + assert runner.eligible(p0) and not runner.skipped(p0) + assert not runner.eligible(p1) and runner.skipped(p1) + + +def _generator(tmp_path): + d = tmp_path / "frameworks" / "Go" / "hugo" + d.mkdir(parents=True) + (d / "benchmark_config.json").write_text(json.dumps( + {"framework": "hugo", "config": [{"output_folder": "public", "output_glob": "post/*/index.html"}]})) + return types.SimpleNamespace(name="hugo", directory=str(d)) + + +def _archive(results_dir): + tar_path = pathlib.Path(runner.archive_path(str(results_dir), "hugo")) + tar_path.parent.mkdir(parents=True) + with tarfile.open(tar_path, "w") as tar: + tar.add(FIX / "site", arcname="public") + return tar_path + + +def test_run_pass_ok(tmp_path): + test, results_dir = _generator(tmp_path), tmp_path / "results" + tar_path = _archive(results_dir) + helper = mock.Mock() + helper.run_quality.return_value = json.loads((FIX / "raw.json").read_text()) + q = runner.run_pass(helper, test, str(results_dir)) + assert q["status"] == "ok" and q["lighthouse"]["status"] == "ok" + args = helper.run_quality.call_args[0] + assert args[0] == str(tar_path) and args[1] == "public" + assert args[2]["post"] == {"url": "/post/hello/", "file": "post/hello/index.html"} + out = results_dir / "quality" / "hugo" + assert json.loads((out / "quality.json").read_text()) == q + assert (out / "raw.json").exists() and (out / "site" / "index.html").exists() + assert not tar_path.exists() + + +def test_run_pass_missing_archive(tmp_path): + test, results_dir = _generator(tmp_path), tmp_path / "results" + q = runner.run_pass(mock.Mock(), test, str(results_dir)) + assert q["status"] == "error" and "no exported output" in q["error"] + assert json.loads((results_dir / "quality" / "hugo" / "quality.json").read_text()) == q + + +def test_run_pass_collector_error(tmp_path): + test, results_dir = _generator(tmp_path), tmp_path / "results" + _archive(results_dir) + helper = mock.Mock() + helper.run_quality.side_effect = RuntimeError("collector wrote no raw.json (exit 1)") + q = runner.run_pass(helper, test, str(results_dir)) + assert q == {"status": "error", "error": "RuntimeError: collector wrote no raw.json (exit 1)"} + + +def _bm(tmp_path, quality=True, results=None, failure=None): + from toolset.benchmark.benchmarker import Benchmarker + tmp_path.mkdir(parents=True, exist_ok=True) + b = Benchmarker.__new__(Benchmarker) + b.config = types.SimpleNamespace(min_runs="1", run_test_timeout_seconds=10, resources={"cpus": 1}, + quality=quality, suite_info=None, number_of_files="50", + content_size="5") + test_type = mock.Mock() + test_type.get_script_name.return_value = "build.sh" + test_type.get_script_variables.return_value = {"min_runs": "1"} + b.config.types = {"datarate": test_type} + b.results = mock.Mock() + b.results.directory = str(tmp_path / "results") + b.results.get_raw_file.return_value = str(tmp_path / "raw.txt") + exports = [] + + def bench(ft, script, variables, raw_file, res, timeout, export=None): + exports.append(export) + return {"status": "ok", "exitCode": 0} + + b.docker_helper = mock.Mock() + b.docker_helper.benchmark.side_effect = bench + b.docker_helper._quality_image = "sha256:q" + b.results.parse_test.return_value = { + "results": [{"cv": None, "median": 1.0}] if results is None else results, + "status": "ok", "failureReason": failure, "unsupported": False} + b._Benchmarker__begin_logging = mock.Mock() + b._Benchmarker__end_logging = mock.Mock() + ft = _generator(tmp_path) + ft.runTests = {"datarate": mock.Mock(failed=False)} + return b, ft, exports + + +def test_quality_pass_runs_after_ok(tmp_path, monkeypatch): + b, ft, exports = _bm(tmp_path) + monkeypatch.setattr(runner, "run_pass", lambda helper, test, d: {"status": "ok", "test": test.name}) + b._Benchmarker__benchmark(ft, open(tmp_path / "log", "w")) + assert exports == [("public", runner.archive_path(b.results.directory, "hugo"))] + b.results.add_quality.assert_called_once_with("hugo", {"status": "ok", "test": "hugo"}, "sha256:q") + + +def test_quality_pass_missing_archive_keeps_result(tmp_path): + reported = [] + for quality in (False, True): + b, ft, _ = _bm(tmp_path / str(quality), quality=quality) + b._Benchmarker__benchmark(ft, open(tmp_path / ("log%s" % quality), "w")) + # everything but the generator object (its directory differs per run) + reported.append(b.results.report_benchmark_results.call_args[0][1:]) + if quality: + name, q, _image = b.results.add_quality.call_args[0] + assert name == "hugo" and q["status"] == "error" + assert reported[0] == reported[1] + + +def test_no_pass_without_results_or_with_failure(tmp_path, monkeypatch): + called = [] + monkeypatch.setattr(runner, "run_pass", lambda *a: called.append(a) or {"status": "ok"}) + for i, kw in enumerate(({"results": []}, {"failure": "nonconformant: index"})): + b, ft, _ = _bm(tmp_path / str(i), **kw) + b._Benchmarker__benchmark(ft, open(tmp_path / "log", "w")) + b.results.add_quality.assert_not_called() + assert called == [] + + +def test_skipped_cell_exports_nothing(tmp_path): + b, ft, exports = _bm(tmp_path) + b.config.suite_info = {"name": "P"} + b.config.content_size = "50" + b._Benchmarker__benchmark(ft, open(tmp_path / "log", "w")) + assert exports == [None] + b.results.add_quality.assert_not_called() + + +def test_results_add_quality(fake_benchmarker): + res = Results(fake_benchmarker) + res._environment = {"fingerprint": "f"} # what environment.capture would have cached + before = res._Results__to_jsonable() + assert "quality" not in before and "quality" not in before["environment"] + res.add_quality("hugo", {"status": "ok", "tools": {"lighthouse": "1.0.0"}}, "sha256:q") + after = res._Results__to_jsonable() + assert after["quality"] == {"hugo": {"status": "ok", "tools": {"lighthouse": "1.0.0"}}} + assert after["rawData"] == before["rawData"] and after["succeeded"] == before["succeeded"] + assert after["environment"] == {"fingerprint": "f", + "quality": {"lighthouse": "1.0.0", "image": "sha256:q"}} + + +def test_reparse_keeps_quality(fake_benchmarker, monkeypatch): + assert "quality" in STORED_RUN_KEYS + monkeypatch.setattr(Results, "_Results__count_commits", lambda self: None) + monkeypatch.setattr(Results, "_Results__count_sloc", lambda self: None) + res = Results(fake_benchmarker) + with open(res.file, "w") as f: + json.dump({"quality": {"hugo": {"status": "ok"}}, "rawData": {"datarate": {}}}, f) + res.reparse([]) + with open(res.file) as f: + assert json.load(f)["quality"] == {"hugo": {"status": "ok"}} +``` + +- [ ] **Step 2: Run them and see them fail** + +Run: `python3 -m pytest -q tests/test_quality_runner.py` +Expected: FAIL with `ImportError: cannot import name 'runner'`. + +- [ ] **Step 3: Implement `toolset/quality/runner.py`** + +```python +''' +The untimed quality pass of one generator (spec 009 R-1..R-3, R-16). +''' +import json +import os + +from toolset.quality import reduce, site + +# R-1: with --suite, the pass runs only in this (numberOfFiles, contentSize) cell +QUALITY_CELL = (50, 5.0) + + +def _cell(config): + return int(config.number_of_files), float(config.content_size) + + +def eligible(config): + if not getattr(config, 'quality', False): + return False + if getattr(config, 'suite_info', None) is None: + return True + return _cell(config) == QUALITY_CELL + + +def skipped(config): + return bool(getattr(config, 'quality', False)) and not eligible(config) + + +def archive_path(results_dir, name): + return os.path.join(results_dir, 'quality', name, 'site.tar') + + +def run_pass(docker_helper, test, results_dir): + '''Runs the pass, writes quality.json and returns it. Never raises.''' + out_dir = os.path.join(results_dir, 'quality', test.name) + tar_path = archive_path(results_dir, test.name) + try: + output_folder, output_glob = site.generator_config(test.directory) + if not os.path.exists(tar_path): + raise RuntimeError('no exported output (%s)' % output_folder) + site_dir = site.extract(tar_path, os.path.join(out_dir, 'site')) + pages = site.resolve_pages(site_dir) + site_name = os.path.basename(output_folder.rstrip('/')) + raw = docker_helper.run_quality(tar_path, site_name, pages) + with open(os.path.join(out_dir, 'raw.json'), 'w') as f: + json.dump(raw, f, indent=2) + quality = reduce.build_quality(raw, site_dir, pages, output_glob) + except Exception as e: + quality = {'status': 'error', 'error': '%s: %s' % (type(e).__name__, e)} + os.makedirs(out_dir, exist_ok=True) + with open(os.path.join(out_dir, 'quality.json'), 'w') as f: + json.dump(quality, f, indent=2) + if os.path.exists(tar_path): + os.remove(tar_path) + return quality +``` + +- [ ] **Step 4: Add the flag** + +In `toolset/run-tests.py`, after the `--no-cache` argument: + +```python + parser.add_argument( + '--quality', + action='store_true', + default=False, + help='After the timed runs, audit the output of the last build (Lighthouse, SEO ' + 'signals, HTML, accessibility, links, weight); untimed. With --suite, only in ' + 'the cell nf50-cs5 (spec 009)') +``` + +In `toolset/utils/benchmark_config.py`, after `self.no_cache = ...`: + +```python + self.quality = bool(getattr(args, 'quality', False)) +``` + +- [ ] **Step 5: Hook the pass into `Benchmarker`** + +In `toolset/benchmark/benchmarker.py`, add the imports `from toolset.quality import runner as quality_runner` and `from toolset.quality import site as quality_site` with the other `toolset` imports. + +In `__benchmark.benchmark_type`, before `if not test.failed:`, add: + +```python + export = self.__quality_export(framework_test) + self._quality_kwargs = {'export': export} if export else {} +``` + +Change both `self.docker_helper.benchmark(...)` calls, in `benchmark_type` and in `__noise_control`, to pass `**self._quality_kwargs` after `self.config.run_test_timeout_seconds`. In `__noise_control`, use `**getattr(self, '_quality_kwargs', {})`, because older tests call it directly. + +After `self.results.report_benchmark_results(...)` in `benchmark_type`, add: + +```python + if export and results['results'] and not results.get('failureReason') \ + and not results.get('unsupported'): + self.__quality_pass(framework_test, benchmark_log) +``` + +Add the two methods to the class: + +```python + def __quality_export(self, framework_test): + '''(output_folder, dest_tar) when the quality pass runs in this cell (spec 009 R-1).''' + if quality_runner.skipped(self.config): + log("quality: skipped (only cell nf50-cs5 of a suite)") + return None + if not quality_runner.eligible(self.config): + return None + try: + output_folder, _glob = quality_site.generator_config(framework_test.directory) + except Exception as e: + log("quality: cannot read output_folder: %s" % e) + return None + return (output_folder, + quality_runner.archive_path(self.results.directory, framework_test.name)) + + def __quality_pass(self, framework_test, benchmark_log): + '''Untimed; never changes the build result (spec 009 R-2).''' + log("QUALITY PASS %s (untimed)" % framework_test.name, file=benchmark_log, border='*') + quality = quality_runner.run_pass(self.docker_helper, framework_test, self.results.directory) + self.results.add_quality(framework_test.name, quality, + getattr(self.docker_helper, '_quality_image', None)) + log("quality: %s" % quality.get('status'), file=benchmark_log) +``` + +- [ ] **Step 6: Record quality in `Results`** + +In `toolset/utils/results.py`: +- Add `'quality'` to the end of `STORED_RUN_KEYS`. +- In `__init__`, next to the other run attributes, set `self.quality = {}` and `self.quality_environment = None`. +- Add: + +```python + def add_quality(self, name, quality, image_id=None): + '''results.json quality. and environment.quality (spec 009 R-6, R-17).''' + self.quality[name] = quality + tools = quality.get('tools') + if self.quality_environment is None and isinstance(tools, dict) \ + and not tools.get('error'): + self.quality_environment = dict(tools, image=image_id) +``` + +- In `__to_jsonable`, replace `toRet['environment'] = self.__environment()` with: + +```python + env = self.__environment() + if env is not None and self.quality_environment: + env = dict(env, quality=self.quality_environment) + toRet['environment'] = env +``` + + Before the `toRet.update(getattr(self, '_stored_run', None) or {})` line, add: + +```python + if getattr(self, 'quality', None): + toRet['quality'] = self.quality +``` + +- [ ] **Step 7: Run all tests** + +Run: `python3 -m pytest -q && ruff check toolset tests` +Expected: all pass. That includes `tests/test_noise.py` and `tests/test_schema.py`, whose `bench` side effects take 6 positional arguments; without `--quality`, no `export` keyword is passed. If `test_schema.py` lists the allowed top-level keys of `results.json`, add `"quality": dict` to that mapping, because the key is only present after a quality pass. + +- [ ] **Step 8: Commit** + +```bash +git add toolset/quality/runner.py toolset/run-tests.py toolset/utils/benchmark_config.py \ + toolset/benchmark/benchmarker.py toolset/utils/results.py tests/test_quality_runner.py tests/test_schema.py +git commit -m "feat(quality): --quality runs the untimed pass after a successful build (spec 009 R-1, R-2, R-6, R-17) + +Co-Authored-By: Claude Opus 5.5 (1M context) " +``` + +--- + +### Task 7: "Qualidade" in `summary.md` and `quality-summary.csv` + +**Files:** +- Modify: + - `toolset/utils/summary.py`: new functions after `to_markdown`, and one hook at the end of `to_markdown`; + - `toolset/utils/results.py`: `write_summary`. +- Test: `tests/test_quality_summary.py` + +**Interfaces:** +- Consumes: the `quality.json` contract (Task 3); `results.json` `quality` (Task 6). +- Produces: `QUALITY_COLUMNS`, `quality_rows(results)`, `quality_csv(rows)`, `quality_markdown(rows)`. + +- [ ] **Step 1: Write the failing tests** + +`tests/test_quality_summary.py`: + +```python +import csv +import io +import json +import pathlib + +from toolset.quality import reduce, site +from toolset.utils import summary +from toolset.utils.results import parse_build_output + +FIX = pathlib.Path(__file__).parent / "fixtures" / "quality" +RAW_OK = pathlib.Path(__file__).parent / "fixtures" / "raw_ok.txt" + + +def _quality(): + raw = json.loads((FIX / "raw.json").read_text()) + pages = site.resolve_pages(str(FIX / "site")) + return reduce.build_quality(raw, str(FIX / "site"), pages, "post/*/index.html") + + +def _data(with_quality): + [entry] = parse_build_output(RAW_OK.read_text(), "50", "5", "1") + data = {"name": "r", "frameworks": ["hugo", "gatsby"], + "rawData": {"datarate": {"hugo": [entry]}}, + "succeeded": {"datarate": ["hugo"]}, "failed": {"datarate": ["gatsby"]}, "excluded": []} + if with_quality: + data["quality"] = {"hugo": _quality(), "gatsby": {"status": "error", "error": "RuntimeError: boom"}} + return data + + +def test_quality_rows(): + rows = summary.quality_rows(_data(True)) + assert [r["framework"] for r in rows] == ["gatsby", "hugo"] + hugo = rows[1] + assert hugo["performance"] == 0.98 and hugo["lcpMs"] == 800 + assert hugo["postJsBytes"] == 0 and hugo["jsClass"] == "none" + assert hugo["htmlErrors"] == 1 and hugo["a11yViolations"] == 2 and hugo["brokenLinks"] == 2 + assert (hugo["seoPresent"], hugo["seoTotal"]) == (2, 12) + assert rows[0]["status"] == "error" and rows[0]["performance"] is None + + +def test_quality_markdown_section(): + lines = summary.quality_markdown(summary.quality_rows(_data(True))) + text = "\n".join(lines) + assert lines[0] == "## Qualidade" + assert "| hugo | 98 | 100 | 100 | 82 | 800 | 0.0 | none |" in text + assert "2/12" in text and "no meta description and no viewport" in text + assert "gatsby: RuntimeError: boom" in text + assert "Rank" not in text + + +def test_quality_csv_has_every_numeric_field(): + rows = summary.quality_rows(_data(True)) + parsed = list(csv.DictReader(io.StringIO(summary.quality_csv(rows)))) + assert parsed[1]["framework"] == "hugo" + header = list(parsed[0].keys()) + assert header[:len(summary.QUALITY_COLUMNS)] == summary.QUALITY_COLUMNS + assert "lighthouse.desktop.index.scores.seo.median" in header + assert "weight.brotliBytes" in header + + +def test_quality_never_changes_timing_summary(): + without, with_q = _data(False), _data(True) + assert summary.build_rows(with_q, {}) == summary.build_rows(without, {}) + md_without = summary.to_markdown(summary.build_rows(without, {}), without) + md_with = summary.to_markdown(summary.build_rows(with_q, {}), with_q) + assert md_with.startswith(md_without.rstrip("\n")) + assert "## Qualidade" in md_with and "## Qualidade" not in md_without +``` + +- [ ] **Step 2: Run them and see them fail** + +Run: `python3 -m pytest -q tests/test_quality_summary.py` +Expected: FAIL with `AttributeError: module 'toolset.utils.summary' has no attribute 'quality_rows'`. + +- [ ] **Step 3: Implement in `toolset/utils/summary.py`** + +Add after `to_markdown`: + +```python +QUALITY_COLUMNS = ['framework', 'status', 'performance', 'accessibility', 'bestPractices', 'seo', + 'lcpMs', 'postJsBytes', 'jsClass', 'gzipBytes', 'htmlErrors', + 'a11yViolations', 'brokenLinks', 'seoPresent', 'seoTotal'] +QUALITY_NOTE = ('The SF 005 templates carry no meta description and no viewport, so every ' + 'generator loses the same Lighthouse SEO points; that is not a difference ' + 'between generators.') + + +def _get(data, *path): + for key in path: + if not isinstance(data, dict): + return None + data = data.get(key) + return data + + +def _sum_pages(q, section, key): + pages = _get(q, section, 'pages') or {} + values = [p.get(key) for p in pages.values() if isinstance(p, dict) and p.get(key) is not None] + return sum(values) if values else None + + +def _flatten(data, prefix=''): + for key, value in (data or {}).items(): + name = prefix + str(key) + if isinstance(value, dict): + yield from _flatten(value, name + '.') + elif isinstance(value, (int, float)) and not isinstance(value, bool): + yield name, value + + +def quality_rows(results): + '''One row per generator with a quality pass (spec 009 R-18), sorted by name.''' + rows = [] + for name in sorted(results.get('quality') or {}): + q = results['quality'][name] + post = _get(q, 'lighthouse', 'mobile', 'post') or {} + rows.append({ + 'framework': name, + 'status': q.get('status'), + 'error': q.get('error'), + 'performance': _get(post, 'scores', 'performance', 'median'), + 'accessibility': _get(post, 'scores', 'accessibility', 'median'), + 'bestPractices': _get(post, 'scores', 'best-practices', 'median'), + 'seo': _get(post, 'scores', 'seo', 'median'), + 'lcpMs': _get(post, 'metrics', 'lcp', 'median'), + 'postJsBytes': _get(q, 'js', 'pages', 'post'), + 'jsClass': _get(q, 'js', 'jsClass'), + 'gzipBytes': _get(q, 'weight', 'gzipBytes'), + 'htmlErrors': _sum_pages(q, 'html', 'errors'), + 'a11yViolations': _sum_pages(q, 'a11y', 'total'), + 'brokenLinks': _get(q, 'links', 'broken'), + 'seoPresent': _get(q, 'seo', 'present', 'post'), + 'seoTotal': _get(q, 'seo', 'total'), + 'numeric': dict(_flatten({k: v for k, v in q.items() if k != 'tools'})), + }) + return rows + + +def quality_csv(rows): + extra = sorted({k for r in rows for k in r['numeric']}) + out = io.StringIO() + writer = csv.writer(out, lineterminator='\n') + writer.writerow(QUALITY_COLUMNS + extra) + for r in rows: + writer.writerow([_blank(r[c]) for c in QUALITY_COLUMNS] + + [_blank(r['numeric'].get(k)) for k in extra]) + return out.getvalue() + + +def _score(value): + return '—' if value is None else '%d' % round(value * 100) + + +def _kb(value): + return '—' if value is None else '%.1f' % (value / 1024) + + +def _int(value): + return '—' if value is None else '%d' % round(value) + + +def quality_markdown(rows): + lines = ['## Qualidade', '', + 'Untimed pass over the output of the last build (spec 009). Scores and LCP: ' + 'Lighthouse mobile preset on the post page, median of 3 runs. No combined score ' + 'and no ranking.', '', + '| Framework | Performance | Accessibility | Best practices | SEO | LCP (ms) ' + '| Post JS (KB) | JS | Gzip (KB) | HTML errors | axe violations | Broken links ' + '| SEO signals |', + '|' + '---|' * 13] + for r in rows: + signals = '—' if r['seoPresent'] is None else '%s/%s' % (r['seoPresent'], r['seoTotal']) + cells = [r['framework'], _score(r['performance']), _score(r['accessibility']), + _score(r['bestPractices']), _score(r['seo']), _int(r['lcpMs']), + _kb(r['postJsBytes']), r['jsClass'] or '—', _kb(r['gzipBytes']), + _int(r['htmlErrors']), _int(r['a11yViolations']), _int(r['brokenLinks']), signals] + lines.append('| ' + ' | '.join(str(c) for c in cells) + ' |') + lines += ['', '- ' + QUALITY_NOTE] + for r in rows: + if r['status'] == 'error': + lines.append('- %s: %s' % (r['framework'], r['error'])) + return lines +``` + +Check that `summary.py` already imports `csv` and `io`, which `to_csv` needs; add them if not. Check that `_blank` maps `None` to `''`; if its behavior differs, use `'' if v is None else v` inline. At the end of `to_markdown`, replace `return '\n'.join(lines) + '\n'` with: + +```python + if meta.get('quality'): + lines += [''] + quality_markdown(quality_rows(meta)) + return '\n'.join(lines) + '\n' +``` + +- [ ] **Step 4: Write `quality-summary.csv` in `Results.write_summary`** + +In `toolset/utils/results.py` `write_summary`, after the `summary.md` write: + +```python + if data.get('quality'): + with open(os.path.join(self.directory, 'quality-summary.csv'), 'w', newline='') as f: + f.write(summary.quality_csv(summary.quality_rows(data))) +``` + +- [ ] **Step 5: Run all tests** + +Run: `python3 -m pytest -q && ruff check toolset tests` +Expected: all pass. + +- [ ] **Step 6: Commit** + +```bash +git add toolset/utils/summary.py toolset/utils/results.py tests/test_quality_summary.py +git commit -m "feat(quality): Qualidade section in summary.md and quality-summary.csv (spec 009 R-18, R-19) + +Co-Authored-By: Claude Opus 5.5 (1M context) " +``` + +--- + +### Task 8: CI, the round workflow and docs + +**Files:** +- Modify: `.github/workflows/ci.yml` (two new jobs), `.github/workflows/benchmark-round.yml` (a `quality` input), `README.md` (a "Quality pass" section after "Results") +- Test: CI itself, run on the PR. + +**Interfaces:** +- Consumes: everything above. +- Produces: goal 4 of the spec. + +- [ ] **Step 1: Add the CI jobs** + +Append to `.github/workflows/ci.yml` under `jobs:`: + +```yaml + quality-image: + needs: toolset + strategy: + matrix: + runner: [ubuntu-24.04, ubuntu-24.04-arm] + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + - run: docker build -t ssgberk/quality quality + - name: Collector on the fixture site + run: | + docker run --rm --network none \ + -v "$PWD/tests/fixtures/quality/site:/fixture:ro" -v "$PWD/quality/test:/test:ro" \ + ssgberk/quality sh -c 'node /quality/run.mjs --site /fixture --pages "$(cat /test/pages.json)" --out /tmp/out \ + && node /test/check-shape.mjs /tmp/out/raw.json /test/expected-shape.json' + + quality: + needs: quality-image + strategy: + matrix: + runner: [ubuntu-24.04, ubuntu-24.04-arm] + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - run: ./ssgberk --test hugo -nf 50 -cs 5 -mr 1 --quality + - run: | + R=$(ls -td results/*/ | head -1) + jq -e '.rawData.datarate.hugo[0].mean > 0' "$R/results.json" + jq -e '.quality.hugo.status == "ok"' "$R/results.json" + for s in lighthouse seo html a11y links weight js extra; do + jq -e --arg s "$s" '.quality.hugo[$s].status == "ok"' "$R/results.json" + done + jq -e '[.quality.hugo.lighthouse.mobile[], .quality.hugo.lighthouse.desktop[] + | .scores[] | .median | select(. < 0 or . > 1)] | length == 0' "$R/results.json" + jq -e '.environment.quality.lighthouse and .environment.quality.chromium' "$R/results.json" + test -s "$R/quality-summary.csv" + grep -q '^## Qualidade' "$R/summary.md" +``` + +- [ ] **Step 2: Add the round input** + +In `.github/workflows/benchmark-round.yml`, add to `workflow_dispatch.inputs`: + +```yaml + quality: + description: Run the untimed quality pass (spec 009); with a suite, only in cell nf50-cs5 + type: boolean + default: false +``` + +In the `run` job's "Run benchmark" step, add `QUALITY: ${{ inputs.quality }}` to `env`, and change the script to: + +```bash + Q="" + if [ "$QUALITY" = "true" ]; then Q="--quality"; fi + if [ -n "$SUITE" ]; then + ./ssgberk --test "$TEST" --suite "$SUITE" --cell "$CELL" $Q + else + ./ssgberk --test "$TEST" -nf "$NF" -cs "$CS" -mr "$MR" $Q + fi +``` + +This job already runs one job per generator. A failing check is a result, so the job still succeeds (spec goal 4). + +- [ ] **Step 3: Document it** + +Add to `README.md` after the "Results" section: + +```markdown +## Quality pass + +`--quality` audits the output of the last timed build of each generator, without timing it (spec `docs/specs/009-output-quality`): + + $ ./ssgberk --test hugo -nf 50 -cs 5 --quality + +With `--suite`, the pass runs only in the cell `nf50-cs5` (suite P, cell 0). It builds the `ssgberk/quality` image from `quality/` once. That image pins Lighthouse, Playwright's Chromium, axe-core, html-validate and lychee. The image runs with no network and with the site copied in, and it audits the index, the first post and `404.html`: +- Lighthouse, mobile and desktop presets, median of 3 runs; +- SEO signals (title, lang, description, viewport, canonical, Open Graph, Twitter card, JSON-LD, robots, hreflang, headings, `alt`, plus `robots.txt`, `sitemap.xml`, feed and favicon); +- HTML validity, axe-core violations and broken internal links; +- weight on disk, gzip and brotli; +- JS bytes, and whether the post works without JS; +- files outside the reference page set. + +Results go to `results//quality//quality.json`, to `quality` in `results.json`, to the "Qualidade" section of `summary.md` and to `quality-summary.csv`. They have no combined score and no ranking. A failing check is a result and never fails the run. +``` + +- [ ] **Step 4: Run all tests** + +Run: `python3 -m pytest -q && ruff check toolset tests`, then validate the workflow files with `python3 -c "import yaml,sys; [yaml.safe_load(open(f)) for f in sys.argv[1:]]" .github/workflows/ci.yml .github/workflows/benchmark-round.yml`. If PyYAML is not installed, use `ruby -ryaml -e 'ARGV.each { |f| YAML.load_file(f) }' .github/workflows/*.yml`. +Expected: all pass, and both files parse. + +- [ ] **Step 5: Commit and push** + +```bash +git add .github/workflows/ci.yml .github/workflows/benchmark-round.yml README.md +git commit -m "ci(quality): quality image and hugo quality pass on amd64 and arm64; round input (spec 009 goal 4) + +Co-Authored-By: Claude Opus 5.5 (1M context) " +git push +``` + +Expected: the `quality-image` and `quality` CI jobs pass on both runners. If a Lighthouse audit id or lychee key differs on CI, fix it as described in Task 4 Step 6. diff --git a/quality/.dockerignore b/quality/.dockerignore new file mode 100644 index 0000000..f05b1f2 --- /dev/null +++ b/quality/.dockerignore @@ -0,0 +1,2 @@ +node_modules +test diff --git a/quality/Dockerfile b/quality/Dockerfile new file mode 100644 index 0000000..749b6a8 --- /dev/null +++ b/quality/Dockerfile @@ -0,0 +1,33 @@ +# SSGBerk quality collector (benchmark-tool spec 009): Lighthouse, axe-core, +# html-validate and lychee against Playwright's Chromium. Runs with no network. +FROM ubuntu:24.04 + +ARG DEBIAN_FRONTEND=noninteractive +ARG NODE_VERSION=24.21.0 +ARG LYCHEE_VERSION=0.24.2 + +RUN apt-get -yqq update \ + && apt-get -yqq install --no-install-recommends ca-certificates curl xz-utils \ + && rm -rf /var/lib/apt/lists/* + +RUN ARCH="$(dpkg --print-architecture)" \ + && case "$ARCH" in amd64) NARCH=x64 ;; arm64) NARCH=arm64 ;; *) echo "unsupported $ARCH"; exit 1 ;; esac \ + && curl -fsSL "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${NARCH}.tar.xz" \ + | tar -xJ -C /usr/local --strip-components=1 \ + && node --version + +RUN ARCH="$(dpkg --print-architecture)" \ + && case "$ARCH" in amd64) LARCH=x86_64 ;; arm64) LARCH=aarch64 ;; *) echo "unsupported $ARCH"; exit 1 ;; esac \ + && curl -fsSL "https://github.com/lycheeverse/lychee/releases/download/lychee-v${LYCHEE_VERSION}/lychee-${LARCH}-unknown-linux-gnu.tar.gz" \ + | tar -xz -C /usr/local/bin --strip-components=1 --wildcards '*/lychee' \ + && lychee --version + +ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright NPM_CONFIG_UPDATE_NOTIFIER=false +WORKDIR /quality +COPY package.json package-lock.json ./ +RUN npm ci --omit=dev \ + && npx playwright install --with-deps chromium \ + && rm -rf /root/.npm /var/lib/apt/lists/* +COPY run.mjs ./ +RUN mkdir -p /work /quality-out +WORKDIR /work diff --git a/quality/package-lock.json b/quality/package-lock.json new file mode 100644 index 0000000..267bb0a --- /dev/null +++ b/quality/package-lock.json @@ -0,0 +1,1496 @@ +{ + "name": "ssgberk-quality", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "ssgberk-quality", + "dependencies": { + "axe-core": "4.14.0", + "chrome-launcher": "1.2.2", + "html-validate": "11.16.2", + "lighthouse": "13.5.0", + "playwright": "1.64.0" + } + }, + "node_modules/@formatjs/ecma402-abstract": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/@formatjs/ecma402-abstract/-/ecma402-abstract-2.3.6.tgz", + "integrity": "sha512-HJnTFeRM2kVFVr5gr5kH1XP6K0JcJtE7Lzvtr3FS/so5f1kpsqqqxy5JF+FRaO6H2qmcMfAUIox7AJteieRtVw==", + "license": "MIT", + "dependencies": { + "@formatjs/fast-memoize": "2.2.7", + "@formatjs/intl-localematcher": "0.6.2", + "decimal.js": "^10.4.3", + "tslib": "^2.8.0" + } + }, + "node_modules/@formatjs/fast-memoize": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/@formatjs/fast-memoize/-/fast-memoize-2.2.7.tgz", + "integrity": "sha512-Yabmi9nSvyOMrlSeGGWDiH7rf3a7sIwplbvo/dlz9WCIjzIQAfy1RMf4S0X3yG724n5Ghu2GmEl5NJIV6O9sZQ==", + "license": "MIT", + "dependencies": { + "tslib": "^2.8.0" + } + }, + "node_modules/@formatjs/icu-messageformat-parser": { + "version": "2.11.4", + "resolved": "https://registry.npmjs.org/@formatjs/icu-messageformat-parser/-/icu-messageformat-parser-2.11.4.tgz", + "integrity": "sha512-7kR78cRrPNB4fjGFZg3Rmj5aah8rQj9KPzuLsmcSn4ipLXQvC04keycTI1F7kJYDwIXtT2+7IDEto842CfZBtw==", + "license": "MIT", + "dependencies": { + "@formatjs/ecma402-abstract": "2.3.6", + "@formatjs/icu-skeleton-parser": "1.8.16", + "tslib": "^2.8.0" + } + }, + "node_modules/@formatjs/icu-skeleton-parser": { + "version": "1.8.16", + "resolved": "https://registry.npmjs.org/@formatjs/icu-skeleton-parser/-/icu-skeleton-parser-1.8.16.tgz", + "integrity": "sha512-H13E9Xl+PxBd8D5/6TVUluSpxGNvFSlN/b3coUp0e0JpuWXXnQDiavIpY3NnvSp4xhEMoXyyBvVfdFX8jglOHQ==", + "license": "MIT", + "dependencies": { + "@formatjs/ecma402-abstract": "2.3.6", + "tslib": "^2.8.0" + } + }, + "node_modules/@formatjs/intl-localematcher": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/@formatjs/intl-localematcher/-/intl-localematcher-0.6.2.tgz", + "integrity": "sha512-XOMO2Hupl0wdd172Y06h6kLpBz6Dv+J4okPLl4LPtzbr8f66WbIoy4ev98EBuZ6ZK4h5ydTN6XneT4QVpD7cdA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.8.0" + } + }, + "node_modules/@html-validate/stylish": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/@html-validate/stylish/-/stylish-6.1.0.tgz", + "integrity": "sha512-ceNvF6/5hgvw/OrxXtdl2KurRVLN/Q/OkD/qminRRY/IIERmVaqJOpDF1nhRZMrOG/vyKKW0+Nxw6lSoHUP4Ow==", + "license": "MIT", + "engines": { + "node": "^22.16.0 || >= 24.0.0" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.220.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.220.0.tgz", + "integrity": "sha512-CmVa4ImJ+ynfrPMNaAXHET6Bhb44SwzmfyVJFq9ni2jgXJR/l7C6gfVFddNmHP+ZOkP9cf4f9DBe68qVLTHc9w==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/core": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.12.0.tgz", + "integrity": "sha512-1HsSAuvT4/my0QrXWsyXFdWjKaPeCHsismKZgEEIb8NK13LP6DrMRlsUbvD/0F9mDWK3xDom9AMv3IscxgOnOQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/instrumentation": { + "version": "0.220.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.220.0.tgz", + "integrity": "sha512-xQx3E2WxP1mDvKzxLxX+CTCtNLa560YJZ3087qYHerl2YmiKpv7AH+dAy7vmx+eVrZ5BwhfWUAVoKOoxCNHcpw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.220.0", + "import-in-the-middle": "^3.0.0", + "require-in-the-middle": "^8.0.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/resources": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.12.0.tgz", + "integrity": "sha512-+rSRoOdln6NhXTVcBdHSlZxJDNzwxadeweK2ltZO3tVtfp6aklABc06xclDk6ovRR+EiGgIcSbh/6NgjFhnb3A==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.12.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace/-/sdk-trace-2.12.0.tgz", + "integrity": "sha512-ImVuRQ6faOsTi1Wna88s2avgrrf/iDgXPigIeynkUhjstMuoQUDOJXgkez2j1mlU4btjXcXBGa+q7qKFYR3GsA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.12.0", + "@opentelemetry/resources": "2.12.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.12.0.tgz", + "integrity": "sha512-JOERgZCxRM06FEJ5kEWxlSRChah6MmXDx3KwCruqEv/iEyLkTokCSemf2y1w8oIHtfrr8EVTpzebq/31vn2Cpg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.12.0", + "@opentelemetry/resources": "2.12.0", + "@opentelemetry/sdk-trace": "2.12.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.43.0.tgz", + "integrity": "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@paulirish/trace_engine": { + "version": "0.0.65", + "resolved": "https://registry.npmjs.org/@paulirish/trace_engine/-/trace_engine-0.0.65.tgz", + "integrity": "sha512-Qsm6F5C8xf6ZzQXbQc2+wcpe6sggfs/gvc/ytqSurdvYg3kyW0ECHCqE0CWBKZpqgjVfPNX9c7SCS3r2nEIRGg==", + "license": "BSD-3-Clause", + "dependencies": { + "legacy-javascript": "latest", + "third-party-web": "latest" + } + }, + "node_modules/@puppeteer/browsers": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-3.2.4.tgz", + "integrity": "sha512-okJ6yQMc0SBexOKxMi966omwAV1uAPwgF/YMZdb04gZQPXVjWHl9bmXjZYtIcsN1+PN3Bz4iFfGK3g+c6lZJ0Q==", + "license": "Apache-2.0", + "dependencies": { + "modern-tar": "^0.8.5", + "yargs": "^18.0.0" + }, + "bin": { + "browsers": "lib/main-cli.js" + }, + "engines": { + "node": ">=22.12.0" + }, + "peerDependencies": { + "proxy-agent": ">=8.0.1", + "yauzl": "^2.10.0 || ^3.4.0" + }, + "peerDependenciesMeta": { + "proxy-agent": { + "optional": true + }, + "yauzl": { + "optional": true + } + } + }, + "node_modules/@puppeteer/browsers/node_modules/ansi-regex": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.4.0.tgz", + "integrity": "sha512-KzTVk2tCWAHtYrvvvaP8bJKJq2pVinhLcGEQdtLIYPbmNGNyYe8QwNaTUYQp2J7/vIsUKt5QCqAfUkYyG9DkOw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@puppeteer/browsers/node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@puppeteer/browsers/node_modules/cliui": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz", + "integrity": "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==", + "license": "ISC", + "dependencies": { + "string-width": "^7.2.0", + "strip-ansi": "^7.1.0", + "wrap-ansi": "^9.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@puppeteer/browsers/node_modules/cliui/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@puppeteer/browsers/node_modules/emoji-regex": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", + "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "license": "MIT" + }, + "node_modules/@puppeteer/browsers/node_modules/string-width": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.3.0.tgz", + "integrity": "sha512-ZbmZM0JCihQN91dWnxoipT2KOEyHqEyfRXUyjuRhW8b/xnqPDoq4gWEVApTVa9db2wN8mmoikgFBbjh71+cGeQ==", + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.5.0", + "strip-ansi": "^7.1.2" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@puppeteer/browsers/node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/@puppeteer/browsers/node_modules/wrap-ansi": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", + "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "string-width": "^7.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/@puppeteer/browsers/node_modules/wrap-ansi/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@puppeteer/browsers/node_modules/yargs": { + "version": "18.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-18.2.0.tgz", + "integrity": "sha512-9OpKOLeaoNFecEp7P6iYbzze/5CqWoH7N3SMs/6y1XF4nMvFMspEGZzJ6uFi9MmQwXhyzqYoSEKO3tvA3Z/o2w==", + "license": "MIT", + "dependencies": { + "cliui": "^9.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "string-width": "^8.2.1", + "y18n": "^5.0.5", + "yargs-parser": "^22.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, + "node_modules/@puppeteer/browsers/node_modules/yargs-parser": { + "version": "22.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz", + "integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==", + "license": "ISC", + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, + "node_modules/@sentry/conventions": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/@sentry/conventions/-/conventions-0.16.0.tgz", + "integrity": "sha512-fO9PLmHdVURcSPUpWCItWAtgKiMwGdJHbovoSEyLplX5sxs2ugvI4CBPTrkkgqhObnZOD0CnWBKDzSVQYBKEyQ==", + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/@sentry/core": { + "version": "10.76.2", + "resolved": "https://registry.npmjs.org/@sentry/core/-/core-10.76.2.tgz", + "integrity": "sha512-vndrQalRiwYatTkjlLWvyf69KocXIb9TKJ4A9oiyWguL8PJYtqIRTlRe4OpkpjLrxS18fvRdV0jEm29c/9p3hQ==", + "license": "MIT", + "dependencies": { + "@sentry/conventions": "^0.16.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@sentry/node": { + "version": "10.76.2", + "resolved": "https://registry.npmjs.org/@sentry/node/-/node-10.76.2.tgz", + "integrity": "sha512-LSwGGE/GceTbkDHRoQQr3lq/dpxxA6RdooBbuGTJ/S6BdqKX2kw2FdV7Se9eJzZJJJ5AxCZum37+ejYS7N2pJg==", + "license": "MIT", + "dependencies": { + "@opentelemetry/api": "^1.9.1", + "@opentelemetry/instrumentation": "^0.220.0", + "@opentelemetry/sdk-trace-base": "^2.9.0", + "@sentry/conventions": "^0.16.0", + "@sentry/core": "10.76.2", + "@sentry/node-core": "10.76.2", + "@sentry/opentelemetry": "10.76.2", + "@sentry/server-utils": "10.76.2", + "import-in-the-middle": "^3.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@sentry/node-core": { + "version": "10.76.2", + "resolved": "https://registry.npmjs.org/@sentry/node-core/-/node-core-10.76.2.tgz", + "integrity": "sha512-TrgAc/SvU5kQfjw8pz6I8LLIUiblOSf9BPPNkTbS0Dzcqa4yQLVbmsbS3rvGT7ot64m9Vjzn327ZRy63MlI6/w==", + "license": "MIT", + "dependencies": { + "@sentry/conventions": "^0.16.0", + "@sentry/core": "10.76.2", + "@sentry/opentelemetry": "10.76.2", + "import-in-the-middle": "^3.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/core": "^1.30.1 || ^2.1.0", + "@opentelemetry/exporter-trace-otlp-http": ">=0.57.0 <1", + "@opentelemetry/instrumentation": ">=0.57.1 <1", + "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/core": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-http": { + "optional": true + }, + "@opentelemetry/instrumentation": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + } + } + }, + "node_modules/@sentry/opentelemetry": { + "version": "10.76.2", + "resolved": "https://registry.npmjs.org/@sentry/opentelemetry/-/opentelemetry-10.76.2.tgz", + "integrity": "sha512-deqWrKJGaku3nhiZchg34z8W9v6kUwtzoQ4cmEq6CeT8/4X93X4YfeJgoh1dhbswFRpwor4pQ6kj0KttagKtfA==", + "license": "MIT", + "dependencies": { + "@sentry/conventions": "^0.16.0", + "@sentry/core": "10.76.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/core": "^1.30.1 || ^2.1.0", + "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0" + } + }, + "node_modules/@sentry/server-utils": { + "version": "10.76.2", + "resolved": "https://registry.npmjs.org/@sentry/server-utils/-/server-utils-10.76.2.tgz", + "integrity": "sha512-wogj6Trj8fAu+aD7HbpvHEBlyANMOvvO9MP5NFJaiQbprLKugtCdE6rFTxDhofXvRo4nswL9m4Cvysg3k5KcvQ==", + "license": "MIT", + "dependencies": { + "@sentry/conventions": "^0.16.0", + "@sentry/core": "10.76.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@sidvind/better-ajv-errors": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@sidvind/better-ajv-errors/-/better-ajv-errors-7.0.0.tgz", + "integrity": "sha512-imVsp5D3KxJ8+uUmu2dsLKnFg3qdX952v/L1gZoCa0NO2ivhQWHMpYM2KmpFrRXHWFjlqkNZ/935nxiTqXEi1A==", + "license": "Apache-2.0", + "engines": { + "node": "^22.12 || >= 24.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + } + }, + "node_modules/@types/node": { + "version": "26.6.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.5.tgz", + "integrity": "sha512-uMzsv9stDdofIZBx4UGh4Z9ZgXnGC9seRDscilBcTHAeOFk67MZg/k9bWVfO+bk4qZmG+l52hRnLwUiH2LnJTg==", + "license": "MIT", + "dependencies": { + "undici-types": "~8.9.0" + } + }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ansi-colors": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", + "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/atomically": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/atomically/-/atomically-2.1.1.tgz", + "integrity": "sha512-P4w9o2dqARji6P7MHprklbfiArZAWvo07yW7qs3pdljb3BWr12FIB7W+p0zJiuiVsUpRO0iZn1kFFcpPegg0tQ==", + "license": "MIT", + "dependencies": { + "stubborn-fs": "^2.0.0", + "when-exit": "^2.1.4" + } + }, + "node_modules/axe-core": { + "version": "4.14.0", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.14.0.tgz", + "integrity": "sha512-9WTZxEjsZ7b13TH8JPmbV2z8CHbl80/2hm3XPEG4JgNdQLK81IBRXmSxHfMAOkSqQeRxT/0dwNDz2GOm3zzpcQ==", + "license": "MPL-2.0", + "engines": { + "node": ">=4" + } + }, + "node_modules/chrome-launcher": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/chrome-launcher/-/chrome-launcher-1.2.2.tgz", + "integrity": "sha512-sI90u5qf3854bTrWsQMMdsQfSjjMbeSn7QtxVb9zrikf+QgMfHQ71KepHW7ZfixQKrDlAFKb+qnVlSR+xepBzw==", + "license": "Apache-2.0", + "dependencies": { + "@types/node": "*", + "escape-string-regexp": "^4.0.0", + "is-wsl": "^2.2.0", + "lighthouse-logger": "^2.0.1" + }, + "bin": { + "print-chrome-path": "bin/print-chrome-path.cjs" + }, + "engines": { + "node": ">=12.13.0" + } + }, + "node_modules/chromium-bidi": { + "version": "157.0.8090-0", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-157.0.8090-0.tgz", + "integrity": "sha512-KJw7IQgqXwxpJgJlEIPg5mr2ZoqA+jkTRtc5EABI7sUJ3bu3cpI7NuEBoj4Gn1AsF78CaT9So5WQWspK3/wYag==", + "license": "Apache-2.0", + "dependencies": { + "mitt": "^3.0.1", + "zod": "^4.6.5" + }, + "engines": { + "node": ">=20.19.0 <22.0.0 || >=22.12.0" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, + "node_modules/cjs-module-lexer": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-2.3.0.tgz", + "integrity": "sha512-lmNyBzi6iYiqyrExTctfmjd8tInx+dHu9PBomTfH5RXS53c27JMFJgzNv5TyAnm3PgGf+MLy0+Ws41/HlVxdKQ==", + "license": "MIT" + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/configstore": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/configstore/-/configstore-7.1.0.tgz", + "integrity": "sha512-N4oog6YJWbR9kGyXvS7jEykLDXIE2C0ILYqNBZBp9iwiJpoCBWYsuAdW6PPFn6w06jjnC+3JstVvWHO4cZqvRg==", + "license": "BSD-2-Clause", + "dependencies": { + "atomically": "^2.0.3", + "dot-prop": "^9.0.0", + "graceful-fs": "^4.2.11", + "xdg-basedir": "^5.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/csp_evaluator": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/csp_evaluator/-/csp_evaluator-1.1.8.tgz", + "integrity": "sha512-EwOnfYuNbTytvbMKsLixTrRgnjOa0WZCxGy8A9nnSYAicrdwn+T/epU/yjgymmOxlgKnvH+8wXt+7p/8ak5Feg==", + "license": "Apache-2.0" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decimal.js": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz", + "integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==", + "license": "MIT" + }, + "node_modules/define-lazy-prop": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz", + "integrity": "sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/devtools-protocol": { + "version": "0.0.1687809", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1687809.tgz", + "integrity": "sha512-t7kSb+UKYdyxcmqy98U2P86ne6rCve3DIJ4+lp54i/hhClgFuG35JrLj23rO14Jw3sWDsp7i+22QhNU0+/qk8g==", + "license": "BSD-3-Clause" + }, + "node_modules/dot-prop": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-9.0.0.tgz", + "integrity": "sha512-1gxPBJpI/pcjQhKgIU91II6Wkay+dLcN3M6rf2uwP8hRur3HtQXjVrdAK3sjC0piaEuxzMwjXChcETiJl47lAQ==", + "license": "MIT", + "dependencies": { + "type-fest": "^4.18.2" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/enquirer": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.4.1.tgz", + "integrity": "sha512-rRqJg/6gd538VHvR3PSrdRBb/1Vy2YfzHqzvbhGIQpDRKIa4FgV/54b5Q1xYSxOOwKvjXweS26E0Q+nAMwp2pQ==", + "license": "MIT", + "dependencies": { + "ansi-colors": "^4.1.1", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/es-module-lexer": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-3.0.3.tgz", + "integrity": "sha512-yWGe1O/j+almnV4hOYrmZo8u58JdnmXEMneRQAYkWtqSiT86yoMIEKF4rpqEDj52tbYl3xceBD3DQIBwcsC9Pw==", + "license": "MIT" + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", + "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-east-asian-width": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.7.0.tgz", + "integrity": "sha512-XjH1AECxf0giL2V1aU8vKyRR2ppRUb5c0EvT7zuJTokQ74bNo52zOtghqdWIqrhUD79fo3x0WfKZdOqxF6LG1Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/html-validate": { + "version": "11.16.2", + "resolved": "https://registry.npmjs.org/html-validate/-/html-validate-11.16.2.tgz", + "integrity": "sha512-oci8F4t1AQdAwkOK895hOOrMTSY0zZ6fk4p2LCvGKQDjL/CYDTGyIoVTzZWII0DfwA7gpq7+qJxAKubUofgo1Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/html-validate" + } + ], + "license": "MIT", + "dependencies": { + "@html-validate/stylish": "^6.0.0", + "@sidvind/better-ajv-errors": "7.0.0", + "ajv": "^8.0.0", + "kleur": "^4.1.0", + "prompts": "^2.0.0", + "semver": "^7.0.0" + }, + "bin": { + "html-validate": "bin/html-validate.mjs" + }, + "engines": { + "node": "^22.22.0 || >= 24.8.0" + }, + "peerDependencies": { + "@jest/globals": "^29.0.3 || ^30.0.0", + "@vitest/expect": "^3.2.0 || ^4.0.1 || ^5.0.0", + "jest": "^29.0.3 || ^30.0.0", + "jest-snapshot": "^29.0.3 || ^30.0.0", + "vitest": "^3.2.0 || ^4.0.1 || ^5.0.0" + }, + "peerDependenciesMeta": { + "@jest/globals": { + "optional": true + }, + "@vitest/expect": { + "optional": true + }, + "jest": { + "optional": true + }, + "jest-snapshot": { + "optional": true + }, + "vitest": { + "optional": true + } + } + }, + "node_modules/http-link-header": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/http-link-header/-/http-link-header-1.1.4.tgz", + "integrity": "sha512-xT3GPW6/ZbGuw4UvwHqErSCEjNUlwbQJuZn9/q5U4WEKfp2kENVCAlousG1zLxHeaQ/ffOHUNpWamvkbBW0eNw==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/image-ssim": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/image-ssim/-/image-ssim-0.2.0.tgz", + "integrity": "sha512-W7+sO6/yhxy83L0G7xR8YAc5Z5QFtYEXXRV6EaE8tuYBZJnA3gVgp3q7X7muhLZVodeb9UfvjSbwt9VJwjIYAg==", + "license": "MIT" + }, + "node_modules/import-in-the-middle": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-3.5.2.tgz", + "integrity": "sha512-WseIA/4o56+GYO3RmlOPvejYi771OFeJEeK/ejROsr9bdjW+E5NXTOD1Di8TL40Qgg3JJX9aa97Hf8iiSVIKZw==", + "license": "Apache-2.0", + "dependencies": { + "cjs-module-lexer": "^2.2.0", + "es-module-lexer": "^3.0.2", + "module-details-from-path": "^1.0.4" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/intl-messageformat": { + "version": "10.7.18", + "resolved": "https://registry.npmjs.org/intl-messageformat/-/intl-messageformat-10.7.18.tgz", + "integrity": "sha512-m3Ofv/X/tV8Y3tHXLohcuVuhWKo7BBq62cqY15etqmLxg2DZ34AGGgQDeR+SCta2+zICb1NX83af0GJmbQ1++g==", + "license": "BSD-3-Clause", + "dependencies": { + "@formatjs/ecma402-abstract": "2.3.6", + "@formatjs/fast-memoize": "2.2.7", + "@formatjs/icu-messageformat-parser": "2.11.4", + "tslib": "^2.8.0" + } + }, + "node_modules/is-docker": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", + "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-wsl": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "license": "MIT", + "dependencies": { + "is-docker": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/jpeg-js": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz", + "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==", + "license": "BSD-3-Clause" + }, + "node_modules/js-library-detector": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/js-library-detector/-/js-library-detector-6.7.0.tgz", + "integrity": "sha512-c80Qupofp43y4cJ7+8TTDN/AsDwLi5oOm/plBrWI+iQt485vKXCco+yVmOwEgdo9VOdsYTuV0UlTeetVPTriXA==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/legacy-javascript": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/legacy-javascript/-/legacy-javascript-0.0.3.tgz", + "integrity": "sha512-1r6MT4dVaJyXt4G07Cm2cQ0AUaLJdefiEp4SmIAylA++gnEQ82YvmzE58RkM5EP0fk5hROLCslUPGnwIYdZQVA==", + "license": "Apache-2.0" + }, + "node_modules/lighthouse": { + "version": "13.5.0", + "resolved": "https://registry.npmjs.org/lighthouse/-/lighthouse-13.5.0.tgz", + "integrity": "sha512-w/yYIwg7mP9pCwkcbAXRztnvKHua2dJnZaNptPLa1cEw+akr6lbsIDiUOLsQwvaiAUr0xvxdmwvAzl3pj3Np0A==", + "license": "Apache-2.0", + "dependencies": { + "@paulirish/trace_engine": "0.0.65", + "@sentry/node": "^10.0.0", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", + "axe-core": "^4.13.0", + "chrome-launcher": "^1.2.1", + "configstore": "^7.0.0", + "csp_evaluator": "1.1.8", + "devtools-protocol": "0.0.1687809", + "enquirer": "^2.3.6", + "http-link-header": "^1.1.1", + "intl-messageformat": "^10.5.3", + "jpeg-js": "^0.4.4", + "js-library-detector": "^6.7.0", + "lighthouse-logger": "^2.0.2", + "lighthouse-stack-packs": "1.12.3", + "lodash-es": "^4.17.21", + "lookup-closest-locale": "6.2.0", + "open": "^8.4.0", + "puppeteer-core": "^25.9.0", + "robots-parser": "^3.0.1", + "speedline-core": "^1.4.3", + "third-party-web": "^0.30.0", + "tldts-icann": "^7.4.13", + "web-features": "^3.39.0", + "ws": "^7.0.0", + "yargs": "^17.3.1", + "yargs-parser": "^21.0.0" + }, + "bin": { + "chrome-debug": "core/scripts/manual-chrome-launcher.js", + "lighthouse": "cli/index.js", + "smokehouse": "cli/test/smokehouse/frontends/smokehouse-bin.js" + }, + "engines": { + "node": ">=22.19" + } + }, + "node_modules/lighthouse-logger": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/lighthouse-logger/-/lighthouse-logger-2.0.2.tgz", + "integrity": "sha512-vWl2+u5jgOQuZR55Z1WM0XDdrJT6mzMP8zHUct7xTlWhuQs+eV0g+QL0RQdFjT54zVmbhLCP8vIVpy1wGn/gCg==", + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.4.1", + "marky": "^1.2.2" + } + }, + "node_modules/lighthouse-stack-packs": { + "version": "1.12.3", + "resolved": "https://registry.npmjs.org/lighthouse-stack-packs/-/lighthouse-stack-packs-1.12.3.tgz", + "integrity": "sha512-d8IsOpE83kbANgnM+Tp8+x6HcMpX9o2ITBiUERssgzAIFdZCQzs/f4k6D0DLQTE59enml9mbAOU52Wu35exWtg==", + "license": "Apache-2.0" + }, + "node_modules/lodash-es": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.18.1.tgz", + "integrity": "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==", + "license": "MIT" + }, + "node_modules/lookup-closest-locale": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/lookup-closest-locale/-/lookup-closest-locale-6.2.0.tgz", + "integrity": "sha512-/c2kL+Vnp1jnV6K6RpDTHK3dgg0Tu2VVp+elEiJpjfS1UyY7AjOYHohRug6wT0OpoX2qFgNORndE9RqesfVxWQ==", + "license": "MIT" + }, + "node_modules/marky": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/marky/-/marky-1.3.0.tgz", + "integrity": "sha512-ocnPZQLNpvbedwTy9kNrQEsknEfgvcLMvOtz3sFeWApDq1MXH1TqkCIx58xlpESsfwQOnuBO9beyQuNGzVvuhQ==", + "license": "Apache-2.0" + }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "license": "MIT" + }, + "node_modules/modern-tar": { + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/modern-tar/-/modern-tar-0.8.5.tgz", + "integrity": "sha512-snEhs+6G5Tjd4I7tLCDOaoln2RgE0bD19RzEKgvgK2hZ5VKy3MpLhLTZ2fWpXSTg4K2cyPwp+VHATFJhxfnOeA==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/module-details-from-path": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz", + "integrity": "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==", + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/open": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/open/-/open-8.4.2.tgz", + "integrity": "sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==", + "license": "MIT", + "dependencies": { + "define-lazy-prop": "^2.0.0", + "is-docker": "^2.1.1", + "is-wsl": "^2.2.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/playwright": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.64.0.tgz", + "integrity": "sha512-kVzTnYFYEyQ9RGzBjNCd2+yEqWmVbxl3AHPynv3qQ6JTjv4a+kwRcnqQwIpuP3NnrnIrE/Q6wWAIAvMfIE2/Yg==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.64.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.64.0.tgz", + "integrity": "sha512-T9r+MZkTECl2+oUcZ26YgZzTrYidTsIhYYkJix+6iDOymOb1FB2RLLSOxRGnZGv5DStt+aduIVbt+E8WygXYpg==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/prompts": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", + "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", + "license": "MIT", + "dependencies": { + "kleur": "^3.0.3", + "sisteransi": "^1.0.5" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/prompts/node_modules/kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/puppeteer-core": { + "version": "25.13.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-25.13.0.tgz", + "integrity": "sha512-cyBS0OEx3e9NS53B5F6+UazXtR7f9Tw0cR1/FbPW59/hV6JPZbkIiMnxK2543IkFp3rsGlLn1iPDLF77TJsJsA==", + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "3.2.4", + "chromium-bidi": "157.0.8090-0", + "devtools-protocol": "0.0.1696802", + "typed-query-selector": "^2.12.3", + "webdriver-bidi-protocol": "1.0.1", + "ws": "^8.21.3" + }, + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/puppeteer-core/node_modules/devtools-protocol": { + "version": "0.0.1696802", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1696802.tgz", + "integrity": "sha512-FO5/SENfvc2K8wRnWQflKcwuoTH4dBkXCWTQ9ebIWtuX8JM5+ZvCotaEmkfh21pO7TvIKGTM3hG4xFiq9WTwmg==", + "license": "BSD-3-Clause" + }, + "node_modules/puppeteer-core/node_modules/ws": { + "version": "8.22.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.22.0.tgz", + "integrity": "sha512-Ydggc987+RO0AnWtZ/7Wq9FtNvcrL1b/RO0ud9mWjUPgDrsAAwQSF51sm2hm1XofbU/4jkpGEsLFsZZxU+1DOg==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-in-the-middle": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-8.0.1.tgz", + "integrity": "sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.3.5", + "module-details-from-path": "^1.0.3" + }, + "engines": { + "node": ">=9.3.0 || >=8.10.0 <9.0.0" + } + }, + "node_modules/robots-parser": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/robots-parser/-/robots-parser-3.0.1.tgz", + "integrity": "sha512-s+pyvQeIKIZ0dx5iJiQk1tPLJAWln39+MI5jtM8wnyws+G5azk+dMnMX0qfbqNetKKNgcWWOdi0sfm+FbQbgdQ==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sisteransi": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", + "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", + "license": "MIT" + }, + "node_modules/speedline-core": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/speedline-core/-/speedline-core-1.4.3.tgz", + "integrity": "sha512-DI7/OuAUD+GMpR6dmu8lliO2Wg5zfeh+/xsdyJZCzd8o5JgFUjCeLsBDuZjIQJdwXS3J0L/uZYrELKYqx+PXog==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "image-ssim": "^0.2.0", + "jpeg-js": "^0.4.1" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/stubborn-fs": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/stubborn-fs/-/stubborn-fs-2.0.0.tgz", + "integrity": "sha512-Y0AvSwDw8y+nlSNFXMm2g6L51rBGdAQT20J3YSOqxC53Lo3bjWRtr2BKcfYoAf352WYpsZSTURrA0tqhfgudPA==", + "license": "MIT", + "dependencies": { + "stubborn-utils": "^1.0.1" + } + }, + "node_modules/stubborn-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/stubborn-utils/-/stubborn-utils-1.0.2.tgz", + "integrity": "sha512-zOh9jPYI+xrNOyisSelgym4tolKTJCQd5GBhK0+0xJvcYDcwlOoxF/rnFKQ2KRZknXSG9jWAp66fwP6AxN9STg==", + "license": "MIT" + }, + "node_modules/third-party-web": { + "version": "0.30.0", + "resolved": "https://registry.npmjs.org/third-party-web/-/third-party-web-0.30.0.tgz", + "integrity": "sha512-p+PfyL5U0ediGzvwPzMSdD9MnC8rHW6nWJJVLIQvqN7RsSqfaZ+7n4c9GILqH36EYR0eXB9CzAeCRnKFr/9uJg==", + "license": "MIT" + }, + "node_modules/tldts-core": { + "version": "7.4.18", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.4.18.tgz", + "integrity": "sha512-KI2PKvCmvQI/xD6ypPqkU0i7oxNXR/O58RLRaHR4svsgVUQ2JE/XC8BtMbG6PSehD27I8sZJRcC1ASnCy6lp3g==", + "license": "MIT" + }, + "node_modules/tldts-icann": { + "version": "7.4.18", + "resolved": "https://registry.npmjs.org/tldts-icann/-/tldts-icann-7.4.18.tgz", + "integrity": "sha512-+29GkEj4qOTb8oQRzg7Lxe1TQdkIDFvyqVTcu775faf9iLczNTix8L/wDFiQi4/Ncib7ZjyWPcVf0iRQCrmUjg==", + "license": "MIT", + "dependencies": { + "tldts-core": "^7.4.18" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/type-fest": { + "version": "4.41.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", + "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/typed-query-selector": { + "version": "2.12.3", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.3.tgz", + "integrity": "sha512-/dtuMZyutHLlrxWhhC1uib00ELUr2dZ8C9B8YzZYsJmeE3V56OxRblrSM2UaF142pcMtak7ehZ6+poWGeehNIg==", + "license": "MIT" + }, + "node_modules/undici-types": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", + "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", + "license": "MIT" + }, + "node_modules/web-features": { + "version": "3.42.0", + "resolved": "https://registry.npmjs.org/web-features/-/web-features-3.42.0.tgz", + "integrity": "sha512-Vq97bzyedg6EuPFnz1GIJqsI9me8CbxzwYrcBI/cNwjj66v77IsGmIZi9SxPHTu2t25F7ta4kZJPu4qvuYlhfQ==", + "license": "Apache-2.0" + }, + "node_modules/webdriver-bidi-protocol": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-1.0.1.tgz", + "integrity": "sha512-WXWs/aXF2a6Al1Z2/V/7Gp2oPZOAAmHjJOTmv4KCXRJ122IMj3tSSMc6a4q3K4QOmhQbse/Ahr+oTVJOU2dUiw==", + "license": "Apache-2.0" + }, + "node_modules/when-exit": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/when-exit/-/when-exit-2.1.5.tgz", + "integrity": "sha512-VGkKJ564kzt6Ms1dbgPP/yuIoQCrsFAnRbptpC5wOEsDaNsbCB2bnfnaA8i/vRs5tjUSEOtIuvl9/MyVsvQZCg==", + "license": "MIT" + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/ws": { + "version": "7.5.13", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.13.tgz", + "integrity": "sha512-rsKI6xDBFVf4r/x8XyChGK04QR/XHroxs/jUcoWvtEZM8TPU/X/uIY9B1CsSzYws9ZJb/6bbBu7dPhFW00CAoA==", + "license": "MIT", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xdg-basedir": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-5.1.0.tgz", + "integrity": "sha512-GCPAHLvrIH13+c0SuacwvRYj2SxJXQ4kaVTT5xgL3kPrz56XxkF21IGhjSE1+W0aw7gpBWRGXLCPnPby6lSpmQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz", + "integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/quality/package.json b/quality/package.json new file mode 100644 index 0000000..bc1c618 --- /dev/null +++ b/quality/package.json @@ -0,0 +1,13 @@ +{ + "name": "ssgberk-quality", + "private": true, + "type": "module", + "description": "SSGBerk output quality collector (benchmark-tool spec 009)", + "dependencies": { + "axe-core": "4.14.0", + "chrome-launcher": "1.2.2", + "html-validate": "11.16.2", + "lighthouse": "13.5.0", + "playwright": "1.64.0" + } +} diff --git a/quality/run.mjs b/quality/run.mjs new file mode 100644 index 0000000..a9e7fe2 --- /dev/null +++ b/quality/run.mjs @@ -0,0 +1,298 @@ +// SSGBerk quality collector (benchmark-tool spec 009). Writes raw data only: +// toolset/quality/reduce.py turns raw.json into quality.json. +import { execFile } from 'node:child_process'; +import fs from 'node:fs/promises'; +import http from 'node:http'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs, promisify } from 'node:util'; +import zlib from 'node:zlib'; + +import * as chromeLauncher from 'chrome-launcher'; +import { HtmlValidate } from 'html-validate'; +import lighthouse from 'lighthouse'; +import desktopConfig from 'lighthouse/core/config/desktop-config.js'; +import { chromium } from 'playwright'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const RUNS = 3; +const PRESETS = { mobile: undefined, desktop: desktopConfig }; +const CATEGORIES = ['performance', 'accessibility', 'best-practices', 'seo']; +const CHROME_FLAGS = ['--headless=new', '--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage']; +const NOT_SCORED = new Set(['informative', 'manual', 'notApplicable', 'error']); +const TYPES = { + '.html': 'text/html; charset=utf-8', '.css': 'text/css; charset=utf-8', + '.js': 'text/javascript; charset=utf-8', '.mjs': 'text/javascript; charset=utf-8', + '.json': 'application/json', '.xml': 'application/xml', '.txt': 'text/plain; charset=utf-8', + '.svg': 'image/svg+xml', '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', + '.webp': 'image/webp', '.gif': 'image/gif', '.ico': 'image/x-icon', '.woff': 'font/woff', + '.woff2': 'font/woff2', '.wasm': 'application/wasm', +}; +const COMPRESSIBLE = /^(text\/|application\/(json|xml|wasm)|image\/svg\+xml)/; +const execFileP = promisify(execFile); + +async function guard(fn) { + try { + return await fn(); + } catch (e) { + return { error: String(e && e.message ? e.message : e) }; + } +} + +async function isFile(p) { + try { + return (await fs.stat(p)).isFile(); + } catch { + return false; + } +} + +async function resolveFile(root, pathname) { + let rel; + try { + rel = decodeURIComponent(pathname).replace(/^\/+/, ''); + } catch { + return null; + } + const base = path.resolve(root, rel); + if (base !== root && !base.startsWith(root + path.sep)) return null; + const candidates = rel === '' || rel.endsWith('/') + ? [path.join(base, 'index.html')] + : [base, path.join(base, 'index.html'), `${base}.html`]; + for (const c of candidates) if (await isFile(c)) return c; + return null; +} + +// Fixed static server: gzip, no-cache, directory index, 404.html with status 404. +function serve(root) { + const handle = async (req, res) => { + const { pathname } = new URL(req.url, 'http://127.0.0.1'); + let file = await resolveFile(root, pathname); + let status = 200; + if (!file) { + status = 404; + file = (await isFile(path.join(root, '404.html'))) ? path.join(root, '404.html') : null; + } + let body = file ? await fs.readFile(file) : Buffer.from('Not found'); + const type = file ? TYPES[path.extname(file).toLowerCase()] || 'application/octet-stream' + : 'text/plain; charset=utf-8'; + const headers = { 'content-type': type, 'cache-control': 'no-cache' }; + if (COMPRESSIBLE.test(type) && /\bgzip\b/.test(req.headers['accept-encoding'] || '')) { + body = zlib.gzipSync(body); + headers['content-encoding'] = 'gzip'; + headers.vary = 'accept-encoding'; + } + headers['content-length'] = body.length; + res.writeHead(status, headers); + res.end(req.method === 'HEAD' ? undefined : body); + }; + // A failing request must never take the collector down (spec R-16). + const server = http.createServer(async (req, res) => { + try { + await handle(req, res); + } catch (e) { + if (!res.headersSent) res.writeHead(500, { 'content-type': 'text/plain; charset=utf-8' }); + res.end('Internal error'); + } + }); + return new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve({ + server, origin: `http://127.0.0.1:${server.address().port}`, + }))); +} + +function summarize(lhr) { + const audits = lhr.audits; + const value = (id) => audits[id]?.numericValue ?? null; + const requests = audits['network-requests']?.details?.items ?? []; + const bytes = (type) => requests.filter((r) => r.resourceType === type) + .reduce((sum, r) => sum + (r.transferSize || 0), 0); + return { + scores: Object.fromEntries(CATEGORIES.map((c) => [c, lhr.categories[c]?.score ?? null])), + metrics: { + fcp: value('first-contentful-paint'), + lcp: value('largest-contentful-paint'), + tbt: value('total-blocking-time'), + cls: value('cumulative-layout-shift'), + si: value('speed-index'), + }, + resources: { + totalBytes: value('total-byte-weight'), + jsBytes: bytes('Script'), + cssBytes: bytes('Stylesheet'), + requests: requests.length, + domSize: value('dom-size') ?? value('dom-size-insight'), + }, + failedAudits: Object.values(audits) + .filter((a) => !NOT_SCORED.has(a.scoreDisplayMode) && a.score !== null && a.score < 1) + .map((a) => a.id).sort(), + }; +} + +async function runLighthouse(origin, pages) { + const chrome = await chromeLauncher.launch({ + chromePath: chromium.executablePath(), chromeFlags: CHROME_FLAGS, + }); + try { + const out = {}; + for (const [preset, config] of Object.entries(PRESETS)) { + out[preset] = {}; + for (const [key, page] of Object.entries(pages)) { + const runs = []; + for (let i = 0; i < RUNS; i += 1) { + runs.push(await guard(async () => { + const result = await lighthouse(origin + page.url, { + port: chrome.port, output: 'json', logLevel: 'error', onlyCategories: CATEGORIES, + }, config); + if (!result?.lhr) throw new Error('lighthouse returned no report'); + if (result.lhr.runtimeError) throw new Error(result.lhr.runtimeError.message); + return summarize(result.lhr); + })); + } + out[preset][key] = runs; + } + } + return out; + } finally { + await chrome.kill(); + } +} + +async function runBrowserChecks(origin, pages) { + const axeSource = await fs.readFile(path.join(HERE, 'node_modules', 'axe-core', 'axe.min.js'), 'utf8'); + const browser = await chromium.launch({ args: ['--no-sandbox', '--disable-dev-shm-usage'] }); + try { + const axe = {}; + for (const [key, page] of Object.entries(pages)) { + axe[key] = await guard(async () => { + const tab = await browser.newPage(); + try { + await tab.goto(origin + page.url, { waitUntil: 'load' }); + await tab.addScriptTag({ content: axeSource }); + const result = await tab.evaluate(() => window.axe.run(document, { resultTypes: ['violations'] })); + return { + violations: result.violations.map((v) => ({ id: v.id, impact: v.impact, nodes: v.nodes.length })), + }; + } finally { + await tab.close(); + } + }); + } + const post = await guard(async () => { + const tab = await browser.newPage(); + try { + await tab.goto(origin + pages.post.url, { waitUntil: 'networkidle' }); + const text = async (selector) => ((await tab.locator(selector).first() + .textContent({ timeout: 5000 })) || '').trim(); + return { title: await text('h1.post-title'), firstParagraph: await text('.post-body p') }; + } finally { + await tab.close(); + } + }); + return { axe, rendered: { post }, chromium: browser.version() }; + } finally { + await browser.close(); + } +} + +async function runHtmlValidate(site, pages) { + const validator = new HtmlValidate({ extends: ['html-validate:recommended'] }); + const out = {}; + for (const [key, page] of Object.entries(pages)) { + out[key] = await guard(async () => { + const report = await validator.validateFile(path.join(site, page.file)); + return { + messages: report.results.flatMap((r) => r.messages) + .map((m) => ({ ruleId: m.ruleId, severity: m.severity })), + }; + }); + } + return out; +} + +async function runLychee(site, outDir) { + const report = path.join(outDir, 'lychee.json'); + try { + await execFileP('lychee', ['--offline', '--no-progress', '--include-fragments', '--index-files', 'index.html', + '--format', 'json', '--output', report, '--root-dir', site, site], { maxBuffer: 64 * 1024 * 1024 }); + } catch (e) { + // exit code 2 means broken links; the report is still written + if (!(await isFile(report))) throw e; + } + return { report: JSON.parse(await fs.readFile(report, 'utf8')) }; +} + +async function listFiles(dir) { + const out = []; + for (const entry of await fs.readdir(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) out.push(...await listFiles(full)); + else if (entry.isFile()) out.push(full); + } + return out; +} + +async function sizes(site) { + const files = []; + for (const full of (await listFiles(site)).sort()) { + const data = await fs.readFile(full); + files.push({ + path: path.relative(site, full).split(path.sep).join('/'), + bytes: data.length, + gzip: zlib.gzipSync(data, { level: 9 }).length, + br: zlib.brotliCompressSync(data, { + params: { [zlib.constants.BROTLI_PARAM_QUALITY]: 11 }, + }).length, + }); + } + return files; +} + +async function packageVersion(name) { + const file = path.join(HERE, 'node_modules', name, 'package.json'); + return JSON.parse(await fs.readFile(file, 'utf8')).version; +} + +async function tools(chromiumVersion) { + const lychee = await guard(async () => (await execFileP('lychee', ['--version'])).stdout.trim() + .split(/\s+/).pop()); + return { + node: process.version, + lighthouse: await packageVersion('lighthouse'), + playwright: await packageVersion('playwright'), + chromium: chromiumVersion ?? null, + axeCore: await packageVersion('axe-core'), + htmlValidate: await packageVersion('html-validate'), + chromeLauncher: await packageVersion('chrome-launcher'), + lychee: typeof lychee === 'string' ? lychee : null, + }; +} + +async function main() { + const { values } = parseArgs({ + options: { site: { type: 'string' }, pages: { type: 'string' }, out: { type: 'string' } }, + }); + const site = path.resolve(values.site); + const pages = JSON.parse(values.pages); + const outDir = path.resolve(values.out); + await fs.mkdir(outDir, { recursive: true }); + const raw = { pages }; + const { server, origin } = await serve(site); + try { + const browser = await guard(() => runBrowserChecks(origin, pages)); + raw.axe = browser.error ? browser : browser.axe; + raw.rendered = browser.error ? browser : browser.rendered; + raw.lighthouse = await guard(() => runLighthouse(origin, pages)); + raw.html = await guard(() => runHtmlValidate(site, pages)); + raw.links = await guard(() => runLychee(site, outDir)); + raw.files = await guard(() => sizes(site)); + raw.tools = await guard(() => tools(browser.chromium)); + } finally { + server.close(); + } + await fs.writeFile(path.join(outDir, 'raw.json'), JSON.stringify(raw, null, 2)); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/quality/test/bad-url.mjs b/quality/test/bad-url.mjs new file mode 100644 index 0000000..a1f599b --- /dev/null +++ b/quality/test/bad-url.mjs @@ -0,0 +1,10 @@ +// Regression check: a malformed percent-escape in a page URL must not crash the collector. +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; + +const pages = { post: { url: '/%E0%A4%A/', file: 'post/hello/index.html' } }; +const r = spawnSync('node', ['/quality/run.mjs', '--site', '/fixture', '--pages', JSON.stringify(pages), '--out', '/tmp/bad'], + { encoding: 'utf8' }); +if (r.status !== 0) { console.error(`collector exited ${r.status}\n${r.stderr}`); process.exit(1); } +if (!fs.existsSync('/tmp/bad/raw.json')) { console.error('raw.json not written'); process.exit(1); } +console.log('bad url survived, raw.json written'); diff --git a/quality/test/check-shape.mjs b/quality/test/check-shape.mjs new file mode 100644 index 0000000..3cba4d7 --- /dev/null +++ b/quality/test/check-shape.mjs @@ -0,0 +1,42 @@ +// Checks a collector raw.json against expected-shape.json (keys and types, not values). +import fs from 'node:fs'; + +const [rawPath, shapePath] = process.argv.slice(2); +const raw = JSON.parse(fs.readFileSync(rawPath, 'utf8')); +const shape = JSON.parse(fs.readFileSync(shapePath, 'utf8')); +const errors = []; + +function typeOf(v) { + if (v === null) return 'null'; + if (Array.isArray(v)) return 'array'; + return typeof v; +} + +function check(value, expected, where) { + if (typeof expected === 'string') { + if (!expected.split('|').includes(typeOf(value))) errors.push(`${where}: ${typeOf(value)} is not ${expected}`); + } else if (Array.isArray(expected)) { + if (!Array.isArray(value) || value.length === 0) errors.push(`${where}: expected a non-empty array`); + else value.forEach((v, i) => check(v, expected[0], `${where}[${i}]`)); + } else { + if (typeOf(value) !== 'object') { + errors.push(`${where}: expected an object, got ${JSON.stringify(value).slice(0, 200)}`); + return; + } + for (const [k, e] of Object.entries(expected)) check(value[k], e, `${where}.${k}`); + } +} + +check(raw, shape, 'raw'); +for (const preset of ['mobile', 'desktop']) { + for (const page of ['index', 'post', '404']) { + if ((raw.lighthouse?.[preset]?.[page] || []).length !== 3) errors.push(`lighthouse.${preset}.${page}: expected 3 runs`); + } +} +if (raw.rendered?.post?.title !== 'Hello') errors.push('rendered.post.title is not "Hello"'); +if (raw.lighthouse?.mobile?.post?.[0]?.resources?.jsBytes !== 0) errors.push('fixture post page should request 0 JS bytes'); +if (errors.length) { + console.error(errors.join('\n')); + process.exit(1); +} +console.log('raw.json shape ok'); diff --git a/quality/test/expected-shape.json b/quality/test/expected-shape.json new file mode 100644 index 0000000..5ac2df6 --- /dev/null +++ b/quality/test/expected-shape.json @@ -0,0 +1,208 @@ +{ + "tools": { + "node": "string", + "lighthouse": "string", + "playwright": "string", + "chromium": "string", + "axeCore": "string", + "htmlValidate": "string", + "chromeLauncher": "string", + "lychee": "string" + }, + "pages": "object", + "lighthouse": { + "mobile": { + "index": [ + { + "scores": { + "performance": "number", + "accessibility": "number", + "best-practices": "number", + "seo": "number" + }, + "metrics": { + "fcp": "number", + "lcp": "number", + "tbt": "number", + "cls": "number", + "si": "number" + }, + "resources": { + "totalBytes": "number", + "jsBytes": "number", + "cssBytes": "number", + "requests": "number", + "domSize": "number|null" + }, + "failedAudits": "array" + } + ], + "post": [ + { + "scores": { + "performance": "number", + "accessibility": "number", + "best-practices": "number", + "seo": "number" + }, + "metrics": { + "fcp": "number", + "lcp": "number", + "tbt": "number", + "cls": "number", + "si": "number" + }, + "resources": { + "totalBytes": "number", + "jsBytes": "number", + "cssBytes": "number", + "requests": "number", + "domSize": "number|null" + }, + "failedAudits": "array" + } + ], + "404": [ + { + "scores": { + "performance": "number", + "accessibility": "number", + "best-practices": "number", + "seo": "number" + }, + "metrics": { + "fcp": "number", + "lcp": "number", + "tbt": "number", + "cls": "number", + "si": "number" + }, + "resources": { + "totalBytes": "number", + "jsBytes": "number", + "cssBytes": "number", + "requests": "number", + "domSize": "number|null" + }, + "failedAudits": "array" + } + ] + }, + "desktop": { + "index": [ + { + "scores": { + "performance": "number", + "accessibility": "number", + "best-practices": "number", + "seo": "number" + }, + "metrics": { + "fcp": "number", + "lcp": "number", + "tbt": "number", + "cls": "number", + "si": "number" + }, + "resources": { + "totalBytes": "number", + "jsBytes": "number", + "cssBytes": "number", + "requests": "number", + "domSize": "number|null" + }, + "failedAudits": "array" + } + ], + "post": [ + { + "scores": { + "performance": "number", + "accessibility": "number", + "best-practices": "number", + "seo": "number" + }, + "metrics": { + "fcp": "number", + "lcp": "number", + "tbt": "number", + "cls": "number", + "si": "number" + }, + "resources": { + "totalBytes": "number", + "jsBytes": "number", + "cssBytes": "number", + "requests": "number", + "domSize": "number|null" + }, + "failedAudits": "array" + } + ], + "404": [ + { + "scores": { + "performance": "number", + "accessibility": "number", + "best-practices": "number", + "seo": "number" + }, + "metrics": { + "fcp": "number", + "lcp": "number", + "tbt": "number", + "cls": "number", + "si": "number" + }, + "resources": { + "totalBytes": "number", + "jsBytes": "number", + "cssBytes": "number", + "requests": "number", + "domSize": "number|null" + }, + "failedAudits": "array" + } + ] + } + }, + "axe": { + "index": { + "violations": "array" + }, + "post": { + "violations": "array" + }, + "404": { + "violations": "array" + } + }, + "rendered": { + "post": { + "title": "string", + "firstParagraph": "string" + } + }, + "html": { + "index": { + "messages": "array" + }, + "post": { + "messages": "array" + }, + "404": { + "messages": "array" + } + }, + "links": { + "report": "object" + }, + "files": [ + { + "path": "string", + "bytes": "number", + "gzip": "number", + "br": "number" + } + ] +} diff --git a/quality/test/pages.json b/quality/test/pages.json new file mode 100644 index 0000000..af21b59 --- /dev/null +++ b/quality/test/pages.json @@ -0,0 +1 @@ +{"index": {"url": "/", "file": "index.html"}, "post": {"url": "/post/hello/", "file": "post/hello/index.html"}, "404": {"url": "/404.html", "file": "404.html"}} diff --git a/tests/fixtures/quality/raw.json b/tests/fixtures/quality/raw.json new file mode 100644 index 0000000..18b90f4 --- /dev/null +++ b/tests/fixtures/quality/raw.json @@ -0,0 +1,606 @@ +{ + "tools": { + "node": "v24.21.0", + "lighthouse": "1.0.0", + "playwright": "1.0.0", + "chromium": "1.0", + "axeCore": "1.0.0", + "htmlValidate": "1.0.0", + "chromeLauncher": "1.0.0", + "lychee": "1.0.0" + }, + "pages": { + "index": { + "url": "/", + "file": "index.html" + }, + "post": { + "url": "/post/hello/", + "file": "post/hello/index.html" + }, + "404": { + "url": "/404.html", + "file": "404.html" + } + }, + "lighthouse": { + "mobile": { + "index": [ + { + "scores": { + "performance": 0.97, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 900, + "lcp": 900, + "tbt": 0, + "cls": 0, + "si": 900 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "a" + ] + }, + { + "scores": { + "performance": 0.99, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 700, + "lcp": 700, + "tbt": 0, + "cls": 0, + "si": 700 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "c" + ] + }, + { + "scores": { + "performance": 0.98, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 800, + "lcp": 800, + "tbt": 0, + "cls": 0, + "si": 800 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "b" + ] + } + ], + "post": [ + { + "scores": { + "performance": 0.97, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 900, + "lcp": 900, + "tbt": 0, + "cls": 0, + "si": 900 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "a" + ] + }, + { + "scores": { + "performance": 0.99, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 700, + "lcp": 700, + "tbt": 0, + "cls": 0, + "si": 700 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "c" + ] + }, + { + "scores": { + "performance": 0.98, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 800, + "lcp": 800, + "tbt": 0, + "cls": 0, + "si": 800 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "b" + ] + } + ], + "404": [ + { + "scores": { + "performance": 0.97, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 900, + "lcp": 900, + "tbt": 0, + "cls": 0, + "si": 900 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "a" + ] + }, + { + "scores": { + "performance": 0.99, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 700, + "lcp": 700, + "tbt": 0, + "cls": 0, + "si": 700 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "c" + ] + }, + { + "scores": { + "performance": 0.98, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 800, + "lcp": 800, + "tbt": 0, + "cls": 0, + "si": 800 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "b" + ] + } + ] + }, + "desktop": { + "index": [ + { + "scores": { + "performance": 0.97, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 900, + "lcp": 900, + "tbt": 0, + "cls": 0, + "si": 900 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "a" + ] + }, + { + "scores": { + "performance": 0.99, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 700, + "lcp": 700, + "tbt": 0, + "cls": 0, + "si": 700 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "c" + ] + }, + { + "scores": { + "performance": 0.98, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 800, + "lcp": 800, + "tbt": 0, + "cls": 0, + "si": 800 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "b" + ] + } + ], + "post": [ + { + "scores": { + "performance": 0.97, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 900, + "lcp": 900, + "tbt": 0, + "cls": 0, + "si": 900 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "a" + ] + }, + { + "scores": { + "performance": 0.99, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 700, + "lcp": 700, + "tbt": 0, + "cls": 0, + "si": 700 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "c" + ] + }, + { + "scores": { + "performance": 0.98, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 800, + "lcp": 800, + "tbt": 0, + "cls": 0, + "si": 800 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "b" + ] + } + ], + "404": [ + { + "scores": { + "performance": 0.97, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 900, + "lcp": 900, + "tbt": 0, + "cls": 0, + "si": 900 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "a" + ] + }, + { + "scores": { + "performance": 0.99, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 700, + "lcp": 700, + "tbt": 0, + "cls": 0, + "si": 700 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "c" + ] + }, + { + "scores": { + "performance": 0.98, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 800, + "lcp": 800, + "tbt": 0, + "cls": 0, + "si": 800 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "b" + ] + } + ] + } + }, + "axe": { + "index": { + "violations": [] + }, + "post": { + "violations": [ + { + "id": "color-contrast", + "impact": "serious", + "nodes": 3 + }, + { + "id": "region", + "impact": "moderate", + "nodes": 1 + } + ] + }, + "404": { + "violations": [] + } + }, + "rendered": { + "post": { + "title": "Hello", + "firstParagraph": "First paragraph of the post." + } + }, + "html": { + "index": { + "messages": [] + }, + "post": { + "messages": [ + { + "ruleId": "no-inline-style", + "severity": 2 + }, + { + "ruleId": "prefer-native-element", + "severity": 1 + } + ] + }, + "404": { + "messages": [] + } + }, + "links": { + "report": { + "fail_map": { + "tests/fixtures/quality/site/index.html": [ + { + "url": "file:///x/missing.css", + "status": { + "text": "Cannot find file" + } + }, + { + "url": "file:///x/post/hello/#nope", + "status": { + "text": "Cannot find fragment" + } + } + ] + } + } + }, + "files": [ + { + "path": "404.html", + "bytes": 559, + "gzip": 319, + "br": 318 + }, + { + "path": "assets/ssgberk.css", + "bytes": 68, + "gzip": 85, + "br": 84 + }, + { + "path": "assets/ssgberk.png", + "bytes": 69, + "gzip": 82, + "br": 81 + }, + { + "path": "feed.xml", + "bytes": 116, + "gzip": 114, + "br": 113 + }, + { + "path": "index.html", + "bytes": 705, + "gzip": 365, + "br": 364 + }, + { + "path": "post/hello/index.html", + "bytes": 927, + "gzip": 448, + "br": 447 + }, + { + "path": "tags/a/index.html", + "bytes": 116, + "gzip": 111, + "br": 110 + } + ] +} diff --git a/tests/fixtures/quality/site/404.html b/tests/fixtures/quality/site/404.html new file mode 100644 index 0000000..5656812 --- /dev/null +++ b/tests/fixtures/quality/site/404.html @@ -0,0 +1,11 @@ + + +Not found + + +

    SSGBerk Reference +
    +

    Page not found

    +

    Built for the SSGBerk build-time benchmark.

    + + diff --git a/tests/fixtures/quality/site/assets/ssgberk.css b/tests/fixtures/quality/site/assets/ssgberk.css new file mode 100644 index 0000000..3ce4b15 --- /dev/null +++ b/tests/fixtures/quality/site/assets/ssgberk.css @@ -0,0 +1 @@ +body { font-family: sans-serif; margin: 0 auto; max-width: 40rem; } diff --git a/tests/fixtures/quality/site/assets/ssgberk.png b/tests/fixtures/quality/site/assets/ssgberk.png new file mode 100644 index 0000000..94381b4 Binary files /dev/null and b/tests/fixtures/quality/site/assets/ssgberk.png differ diff --git a/tests/fixtures/quality/site/feed.xml b/tests/fixtures/quality/site/feed.xml new file mode 100644 index 0000000..e51b66e --- /dev/null +++ b/tests/fixtures/quality/site/feed.xml @@ -0,0 +1,2 @@ + +SSGBerk Reference diff --git a/tests/fixtures/quality/site/index.html b/tests/fixtures/quality/site/index.html new file mode 100644 index 0000000..c9439e9 --- /dev/null +++ b/tests/fixtures/quality/site/index.html @@ -0,0 +1,13 @@ + + +SSGBerk Reference + + +
    SSGBerk Reference +
    +
    +

    Built for the SSGBerk build-time benchmark.

    + + diff --git a/tests/fixtures/quality/site/post/hello/index.html b/tests/fixtures/quality/site/post/hello/index.html new file mode 100644 index 0000000..10b77e4 --- /dev/null +++ b/tests/fixtures/quality/site/post/hello/index.html @@ -0,0 +1,16 @@ + + +Hello + + +
    SSGBerk Reference +
    +
    +

    Hello

    +

    SSGBerk

    +
    • a
    • b
    • c
    +

    First paragraph of the post.

    Second paragraph.

    +
    +

    Built for the SSGBerk build-time benchmark.

    + + diff --git a/tests/fixtures/quality/site/tags/a/index.html b/tests/fixtures/quality/site/tags/a/index.html new file mode 100644 index 0000000..ca28365 --- /dev/null +++ b/tests/fixtures/quality/site/tags/a/index.html @@ -0,0 +1,2 @@ + +Tag a

    a

    diff --git a/tests/test_quality_docker.py b/tests/test_quality_docker.py new file mode 100644 index 0000000..bb89c6b --- /dev/null +++ b/tests/test_quality_docker.py @@ -0,0 +1,169 @@ +import io +import json +import tarfile +import types +from unittest import mock + +import docker +import pytest + +from toolset.utils import docker_helper + +RES = {"cpus": 4.0, "memoryBytes": 8589934592, "swap": False, "cpuset": "4-7"} + + +def _helper(): + helper = docker_helper.DockerHelper.__new__(docker_helper.DockerHelper) + helper.benchmarker = mock.Mock() + helper.server = mock.Mock() + return helper + + +def _container(exit_code=0, oom=False): + c = mock.Mock() + c.logs.return_value = iter(()) + c.wait.return_value = {"StatusCode": exit_code} + c.attrs = {"State": {"OOMKilled": oom, "ExitCode": exit_code}, + "Config": {"WorkingDir": "/opt/hugo/src"}} + c.get_archive.return_value = (iter([b"tar", b"bytes"]), {}) + return c + + +def test_benchmark_exports_output_on_success(tmp_path): + helper, c = _helper(), _container() + helper.server.containers.run.return_value = c + dest = tmp_path / "quality" / "hugo" / "site.tar" + out = helper.benchmark(types.SimpleNamespace(name="hugo"), "build.sh", {}, str(tmp_path / "raw.txt"), + RES, 60, export=("public", str(dest))) + assert out["status"] == "ok" + c.get_archive.assert_called_once_with("/opt/hugo/src/public") + assert dest.read_bytes() == b"tarbytes" + c.remove.assert_called_once_with(force=True) + + +@pytest.mark.parametrize("exit_code,oom", [(1, False), (137, True)]) +def test_benchmark_no_export_on_failure(tmp_path, exit_code, oom): + helper, c = _helper(), _container(exit_code, oom) + helper.server.containers.run.return_value = c + helper.benchmark(types.SimpleNamespace(name="hugo"), "build.sh", {}, str(tmp_path / "raw.txt"), + RES, 60, export=("public", str(tmp_path / "site.tar"))) + c.get_archive.assert_not_called() + + +def test_benchmark_export_error_keeps_status(tmp_path): + helper, c = _helper(), _container() + c.get_archive.side_effect = docker.errors.NotFound("no such path") + helper.server.containers.run.return_value = c + out = helper.benchmark(types.SimpleNamespace(name="hugo"), "build.sh", {}, str(tmp_path / "raw.txt"), + RES, 60, export=("public", str(tmp_path / "site.tar"))) + assert out == {"status": "ok", "exitCode": 0} + + +def test_benchmark_without_export_never_reads_archive(tmp_path): + helper, c = _helper(), _container() + helper.server.containers.run.return_value = c + helper.benchmark(types.SimpleNamespace(name="hugo"), "build.sh", {}, str(tmp_path / "raw.txt"), RES, 60) + c.get_archive.assert_not_called() + + +def _raw_tar(payload): + buf = io.BytesIO() + data = json.dumps(payload).encode() + with tarfile.open(fileobj=buf, mode="w") as tar: + info = tarfile.TarInfo("raw.json") + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + return buf.getvalue() + + +def test_run_quality_isolated_and_cleaned_up(tmp_path): + helper = _helper() + helper._quality_image = "sha256:q" + c = mock.Mock() + c.wait.return_value = {"StatusCode": 0} + c.logs.return_value = b"" + c.get_archive.return_value = (iter([_raw_tar({"tools": {"node": "v24"}})]), {}) + helper.server.containers.create.return_value = c + tar_path = tmp_path / "site.tar" + tar_path.write_bytes(b"site-tar") + pages = {"index": {"url": "/", "file": "index.html"}} + raw = helper.run_quality(str(tar_path), "public", pages) + assert raw == {"tools": {"node": "v24"}} + args, kwargs = helper.server.containers.create.call_args + assert args[0] == "sha256:q" + assert kwargs["network_mode"] == "none" + assert "volumes" not in kwargs and "mounts" not in kwargs + assert kwargs["command"][:4] == ["node", "/quality/run.mjs", "--site", "/work/public"] + assert json.loads(kwargs["command"][kwargs["command"].index("--pages") + 1]) == pages + c.put_archive.assert_called_once_with("/work", b"site-tar") + c.get_archive.assert_called_once_with("/quality-out/raw.json") + c.remove.assert_called_once_with(force=True) + + +def test_run_quality_out_dir_never_inside_site(tmp_path): + helper = _helper() + helper._quality_image = "sha256:q" + c = mock.Mock() + c.wait.return_value = {"StatusCode": 0} + c.get_archive.return_value = (iter([_raw_tar({})]), {}) + helper.server.containers.create.return_value = c + (tmp_path / "site.tar").write_bytes(b"x") + helper.run_quality(str(tmp_path / "site.tar"), "out", {}) + command = helper.server.containers.create.call_args[1]["command"] + site = command[command.index("--site") + 1] + out = command[command.index("--out") + 1] + assert site == "/work/out" + assert out != site and not out.startswith(site + "/") + assert out == docker_helper.QUALITY_OUT and not out.startswith("/work") + + +def test_run_quality_without_raw_json_raises_and_cleans_up(tmp_path): + helper = _helper() + helper._quality_image = "sha256:q" + c = mock.Mock() + c.wait.return_value = {"StatusCode": 1} + c.logs.return_value = b"Error: boom" + c.get_archive.side_effect = docker.errors.NotFound("missing") + helper.server.containers.create.return_value = c + (tmp_path / "site.tar").write_bytes(b"x") + with pytest.raises(RuntimeError, match="exit 1.*boom"): + helper.run_quality(str(tmp_path / "site.tar"), "public", {}) + c.remove.assert_called_once_with(force=True) + + +def test_build_quality_image_once(monkeypatch, tmp_path): + builds = [] + + class FakeAPIClient: + def __init__(self, base_url=None): + pass + + def build(self, **kw): + builds.append(kw) + return iter([{"stream": "Step 1/2 : FROM ubuntu:24.04\n"}, {"stream": "Successfully built abc\n"}]) + + monkeypatch.setattr(docker_helper.docker, "APIClient", FakeAPIClient) + helper = _helper() + helper._quality_image = None + helper.benchmarker.config = types.SimpleNamespace(fw_root=str(tmp_path), server_docker_host=None) + helper.server.images.get.return_value = types.SimpleNamespace(id="sha256:abc") + assert helper.build_quality_image() == "sha256:abc" + assert helper.build_quality_image() == "sha256:abc" + assert len(builds) == 1 + assert builds[0]["path"] == str(tmp_path / "quality") and builds[0]["tag"] == docker_helper.QUALITY_IMAGE + + +def test_build_quality_image_error(monkeypatch, tmp_path): + class FakeAPIClient: + def __init__(self, base_url=None): + pass + + def build(self, **kw): + return iter([{"errorDetail": {"message": "npm ci failed"}}]) + + monkeypatch.setattr(docker_helper.docker, "APIClient", FakeAPIClient) + helper = _helper() + helper._quality_image = None + helper.benchmarker.config = types.SimpleNamespace(fw_root=str(tmp_path), server_docker_host=None) + with pytest.raises(RuntimeError, match="npm ci failed"): + helper.build_quality_image() diff --git a/tests/test_quality_reduce.py b/tests/test_quality_reduce.py new file mode 100644 index 0000000..e692959 --- /dev/null +++ b/tests/test_quality_reduce.py @@ -0,0 +1,113 @@ +import json +import pathlib + +from toolset.quality import reduce, site + +FIX = pathlib.Path(__file__).parent / "fixtures" / "quality" +SITE = FIX / "site" +GLOB = "post/*/index.html" + + +def _raw(): + return json.loads((FIX / "raw.json").read_text()) + + +def test_spread(): + assert reduce.spread([3, 1, 2]) == {"median": 2, "min": 1, "max": 3} + assert reduce.spread([None, 4]) == {"median": 4, "min": 4, "max": 4} + assert reduce.spread([None]) is None + + +def test_reduce_lighthouse_median_and_median_run_audits(): + lh = reduce.reduce_lighthouse(_raw()["lighthouse"]) + post = lh["mobile"]["post"] + assert post["status"] == "ok" and post["runs"] == 3 + assert post["scores"]["performance"] == {"median": 0.98, "min": 0.97, "max": 0.99} + assert post["metrics"]["lcp"] == {"median": 800, "min": 700, "max": 900} + # the run whose performance is the median (0.98) failed audit "b" + assert post["failedAudits"] == ["b"] + assert set(lh) == {"mobile", "desktop"} + + +def test_reduce_lighthouse_page_errors(): + raw = {"mobile": {"index": {"error": "chrome died"}, + "post": [{"error": "timeout"}, {"error": "timeout"}], + "404": [{"error": "x"}, _raw()["lighthouse"]["mobile"]["post"][0]]}, + "desktop": {}} + lh = reduce.reduce_lighthouse(raw) + assert lh["mobile"]["index"] == {"status": "error", "error": "chrome died"} + assert lh["mobile"]["post"] == {"status": "error", "error": "timeout"} + assert lh["mobile"]["404"]["status"] == "ok" and lh["mobile"]["404"]["runs"] == 1 + + +def test_reduce_axe_counts_by_impact(): + a = reduce.reduce_axe(_raw()["axe"]) + assert a["post"] == {"violations": {"critical": 0, "serious": 1, "moderate": 1, "minor": 0}, + "total": 2, "rules": ["color-contrast", "region"]} + assert a["index"]["total"] == 0 + assert reduce.reduce_axe({"index": {"error": "boom"}})["index"] == {"status": "error", "error": "boom"} + + +def test_reduce_html_counts(): + h = reduce.reduce_html(_raw()["html"]) + assert h["post"] == {"errors": 1, "warnings": 1, "rules": ["no-inline-style", "prefer-native-element"]} + + +def test_reduce_links_kinds(): + links = reduce.reduce_links(_raw()["links"]) + assert links["broken"] == 2 + assert links["byKind"] == {"link": 0, "anchor": 1, "asset": 1} + assert links["items"][0]["status"] == "Cannot find file" + + +def test_reduce_links_caps_items(): + fail = {"s": [{"url": "/p%d/" % i, "status": {"text": "x"}} for i in range(30)]} + links = reduce.reduce_links({"report": {"fail_map": fail}}) + assert links["broken"] == 30 and len(links["items"]) == reduce.MAX_BROKEN_LISTED == 20 + + +def test_weight(): + files = [{"path": "index.html", "bytes": 100, "gzip": 50, "br": 40}, + {"path": "post/a/index.html", "bytes": 300, "gzip": 100, "br": 90}, + {"path": "post/b/index.html", "bytes": 100, "gzip": 60, "br": 50}, + {"path": "assets/ssgberk.css", "bytes": 1000, "gzip": 400, "br": 300}] + assert reduce.weight(files, GLOB) == {"bytes": 1500, "gzipBytes": 610, "brotliBytes": 480, + "bytesPerPost": 200, "generatorAddedBytes": 500} + + +def test_js_section_classes(): + lh = {"mobile": {"index": {"status": "ok", "resources": {"jsBytes": {"median": 0}}}, + "post": {"status": "ok", "resources": {"jsBytes": {"median": 0}}}}} + files = [{"path": "a.js", "bytes": 10, "gzip": 5, "br": 4}] + js = reduce.js_section(lh, files, True) + assert js == {"pages": {"index": 0, "post": 0}, "diskBytes": 10, "jsClass": "none", "worksWithoutJs": True} + lh["mobile"]["post"]["resources"]["jsBytes"]["median"] = 5000 + assert reduce.js_section(lh, files, False)["jsClass"] == "hydrated" + assert reduce.js_section({"mobile": {}}, [], None)["jsClass"] is None + + +def test_build_quality_fixture_all_ok(): + pages = site.resolve_pages(str(SITE)) + q = reduce.build_quality(_raw(), str(SITE), pages, GLOB) + assert q["status"] == "ok" + assert q["pages"] == {"index": "/", "post": "/post/hello/", "404": "/404.html"} + assert q["postSource"] == "index" + for section in reduce.SECTIONS: + assert q[section]["status"] == "ok", (section, q[section]) + assert q["js"]["worksWithoutJs"] is True and q["js"]["jsClass"] == "none" + assert q["extra"]["files"] == 2 and q["seo"]["present"]["post"] == 2 + + +def test_build_quality_partial_errors(): + raw = _raw() + raw["lighthouse"] = {"error": "lighthouse crashed"} + del raw["links"] + raw["files"] = {"error": "EACCES"} + pages = site.resolve_pages(str(SITE)) + q = reduce.build_quality(raw, str(SITE), pages, GLOB) + assert q["lighthouse"] == {"status": "error", "error": "RuntimeError: lighthouse crashed"} + assert q["links"]["status"] == "error" and "missing" in q["links"]["error"] + assert q["weight"]["status"] == "error" and q["js"]["status"] == "error" + for section in ("seo", "html", "a11y", "extra"): + assert q[section]["status"] == "ok" + assert q["status"] == "ok" diff --git a/tests/test_quality_runner.py b/tests/test_quality_runner.py new file mode 100644 index 0000000..ae6f5d3 --- /dev/null +++ b/tests/test_quality_runner.py @@ -0,0 +1,238 @@ +import json +import pathlib +import tarfile +import types +from unittest import mock + +from toolset.quality import runner +from toolset.utils.results import STORED_RUN_KEYS, Results + +FIX = pathlib.Path(__file__).parent / "fixtures" / "quality" + + +def _cfg(**kw): + return types.SimpleNamespace(**kw) + + +def test_eligible_and_skipped(): + assert not runner.eligible(_cfg()) + assert not runner.skipped(_cfg()) + assert runner.eligible(_cfg(quality=True, suite_info=None, number_of_files="10", content_size="0.500")) + p0 = _cfg(quality=True, suite_info={"name": "P"}, number_of_files="50", content_size="5") + p1 = _cfg(quality=True, suite_info={"name": "P"}, number_of_files="50", content_size="50") + assert runner.eligible(p0) and not runner.skipped(p0) + assert not runner.eligible(p1) and runner.skipped(p1) + + +def _generator(tmp_path): + d = tmp_path / "frameworks" / "Go" / "hugo" + d.mkdir(parents=True) + (d / "benchmark_config.json").write_text(json.dumps( + {"framework": "hugo", "config": [{"output_folder": "public", "output_glob": "post/*/index.html"}]})) + return types.SimpleNamespace(name="hugo", directory=str(d)) + + +def _archive(results_dir): + tar_path = pathlib.Path(runner.archive_path(str(results_dir), "hugo")) + tar_path.parent.mkdir(parents=True) + with tarfile.open(tar_path, "w") as tar: + tar.add(FIX / "site", arcname="public") + return tar_path + + +def test_run_pass_ok(tmp_path): + test, results_dir = _generator(tmp_path), tmp_path / "results" + tar_path = _archive(results_dir) + helper = mock.Mock() + helper.run_quality.return_value = json.loads((FIX / "raw.json").read_text()) + q = runner.run_pass(helper, test, str(results_dir)) + assert q["status"] == "ok" and q["lighthouse"]["status"] == "ok" + args = helper.run_quality.call_args[0] + assert args[0] == str(tar_path) and args[1] == "public" + assert args[2]["post"] == {"url": "/post/hello/", "file": "post/hello/index.html", "source": "index"} + out = results_dir / "quality" / "hugo" + assert json.loads((out / "quality.json").read_text()) == q + assert (out / "raw.json").exists() and (out / "site" / "index.html").exists() + assert not tar_path.exists() + + +def test_run_pass_uses_output_glob_when_index_has_no_post_link(tmp_path): + test, results_dir = _generator(tmp_path), tmp_path / "results" + src = tmp_path / "plain" + (src / "post" / "a").mkdir(parents=True) + (src / "index.html").write_text("

    no layout

    ") + (src / "post" / "a" / "index.html").write_text("

    a

    ") + tar_path = pathlib.Path(runner.archive_path(str(results_dir), "hugo")) + tar_path.parent.mkdir(parents=True) + with tarfile.open(tar_path, "w") as tar: + tar.add(src, arcname="public") + helper = mock.Mock() + helper.run_quality.return_value = json.loads((FIX / "raw.json").read_text()) + q = runner.run_pass(helper, test, str(results_dir)) + pages = helper.run_quality.call_args[0][2] + assert pages["post"] == {"url": "/post/a/", "file": "post/a/index.html", "source": "glob"} + assert "404" not in pages + assert q["postSource"] == "glob" and q["pages"] == {"index": "/", "post": "/post/a/"} + + +def test_run_pass_missing_archive(tmp_path): + test, results_dir = _generator(tmp_path), tmp_path / "results" + q = runner.run_pass(mock.Mock(), test, str(results_dir)) + assert q["status"] == "error" and "no exported output" in q["error"] + assert json.loads((results_dir / "quality" / "hugo" / "quality.json").read_text()) == q + + +def test_run_pass_collector_error(tmp_path): + test, results_dir = _generator(tmp_path), tmp_path / "results" + _archive(results_dir) + helper = mock.Mock() + helper.run_quality.side_effect = RuntimeError("collector wrote no raw.json (exit 1)") + q = runner.run_pass(helper, test, str(results_dir)) + assert q == {"status": "error", "error": "RuntimeError: collector wrote no raw.json (exit 1)"} + + +def _bm(tmp_path, quality=True, results=None, failure=None): + from toolset.benchmark.benchmarker import Benchmarker + tmp_path.mkdir(parents=True, exist_ok=True) + b = Benchmarker.__new__(Benchmarker) + b.config = types.SimpleNamespace(min_runs="1", run_test_timeout_seconds=10, resources={"cpus": 1}, + quality=quality, suite_info=None, number_of_files="50", + content_size="5") + test_type = mock.Mock() + test_type.get_script_name.return_value = "build.sh" + test_type.get_script_variables.return_value = {"min_runs": "1"} + b.config.types = {"datarate": test_type} + b.results = mock.Mock() + b.results.directory = str(tmp_path / "results") + b.results.get_raw_file.return_value = str(tmp_path / "raw.txt") + exports = [] + + def bench(ft, script, variables, raw_file, res, timeout, export=None): + exports.append(export) + return {"status": "ok", "exitCode": 0} + + b.docker_helper = mock.Mock() + b.docker_helper.benchmark.side_effect = bench + b.docker_helper._quality_image = "sha256:q" + b.results.parse_test.return_value = { + "results": [{"cv": None, "median": 1.0}] if results is None else results, + "status": "ok", "failureReason": failure, "unsupported": False} + b._Benchmarker__begin_logging = mock.Mock() + b._Benchmarker__end_logging = mock.Mock() + ft = _generator(tmp_path) + ft.runTests = {"datarate": mock.Mock(failed=False)} + return b, ft, exports + + +def test_quality_pass_runs_after_ok(tmp_path, monkeypatch): + b, ft, exports = _bm(tmp_path) + monkeypatch.setattr(runner, "run_pass", lambda helper, test, d: {"status": "ok", "test": test.name}) + b._Benchmarker__benchmark(ft, open(tmp_path / "log", "w")) + assert exports == [("public", runner.archive_path(b.results.directory, "hugo"))] + b.results.add_quality.assert_called_once_with("hugo", {"status": "ok", "test": "hugo"}, "sha256:q") + + +def test_quality_pass_missing_archive_keeps_result(tmp_path): + reported = [] + for quality in (False, True): + b, ft, _ = _bm(tmp_path / str(quality), quality=quality) + b._Benchmarker__benchmark(ft, open(tmp_path / ("log%s" % quality), "w")) + # everything but the generator object (its directory differs per run) + reported.append(b.results.report_benchmark_results.call_args[0][1:]) + if quality: + name, q, _image = b.results.add_quality.call_args[0] + assert name == "hugo" and q["status"] == "error" + assert reported[0] == reported[1] + + +def test_no_pass_without_results_or_with_failure(tmp_path, monkeypatch): + called = [] + monkeypatch.setattr(runner, "run_pass", lambda *a: called.append(a) or {"status": "ok"}) + for i, kw in enumerate(({"results": []}, {"failure": "nonconformant: index"})): + b, ft, _ = _bm(tmp_path / str(i), **kw) + b._Benchmarker__benchmark(ft, open(tmp_path / "log", "w")) + b.results.add_quality.assert_not_called() + assert called == [] + + +def test_skipped_cell_exports_nothing(tmp_path): + b, ft, exports = _bm(tmp_path) + b.config.suite_info = {"name": "P"} + b.config.content_size = "50" + b._Benchmarker__benchmark(ft, open(tmp_path / "log", "w")) + assert exports == [None] + b.results.add_quality.assert_not_called() + + +def test_results_add_quality(fake_benchmarker): + res = Results(fake_benchmarker) + res._environment = {"fingerprint": "f"} # what environment.capture would have cached + before = res._Results__to_jsonable() + assert "quality" not in before and "quality" not in before["environment"] + res.add_quality("hugo", {"status": "ok", "tools": {"lighthouse": "1.0.0"}}, "sha256:q") + after = res._Results__to_jsonable() + assert after["quality"] == {"hugo": {"status": "ok", "tools": {"lighthouse": "1.0.0"}}} + assert after["rawData"] == before["rawData"] and after["succeeded"] == before["succeeded"] + assert after["environment"] == {"fingerprint": "f", + "quality": {"lighthouse": "1.0.0", "image": "sha256:q"}} + + +def test_reparse_keeps_quality(fake_benchmarker, monkeypatch): + assert "quality" in STORED_RUN_KEYS + monkeypatch.setattr(Results, "_Results__count_commits", lambda self: None) + monkeypatch.setattr(Results, "_Results__count_sloc", lambda self: None) + res = Results(fake_benchmarker) + with open(res.file, "w") as f: + json.dump({"quality": {"hugo": {"status": "ok"}}, "rawData": {"datarate": {}}}, f) + res.reparse([]) + with open(res.file) as f: + assert json.load(f)["quality"] == {"hugo": {"status": "ok"}} + + +def test_run_pass_write_failure_still_returns(tmp_path, monkeypatch): + test, results_dir = _generator(tmp_path), tmp_path / "results" + tar_path = _archive(results_dir) + helper = mock.Mock() + helper.run_quality.return_value = json.loads((FIX / "raw.json").read_text()) + real_open = open + + def fake_open(path, *a, **kw): + if str(path).endswith("quality.json"): + raise OSError("disk full") + return real_open(path, *a, **kw) + + monkeypatch.setattr("builtins.open", fake_open) + q = runner.run_pass(helper, test, str(results_dir)) + assert isinstance(q, dict) and q["status"] == "ok" + assert not tar_path.exists() + + +def test_quality_failure_never_changes_build_result(tmp_path, monkeypatch): + def boom(*a): + raise RuntimeError("kaboom") + + outcomes = [] + for quality in (False, True): + b, ft, _ = _bm(tmp_path / str(quality), quality=quality) + if quality: + monkeypatch.setattr(runner, "run_pass", boom) + ret = b._Benchmarker__benchmark(ft, open(tmp_path / ("log%s" % quality), "w")) + outcomes.append((ret, b.results.report_benchmark_results.call_args[0][1:])) + if quality: + name, q, image = b.results.add_quality.call_args[0] + assert name == "hugo" and q == {"status": "error", "error": "RuntimeError: kaboom"} + assert image is None + assert outcomes[0] == outcomes[1] + + +def test_unused_export_tar_removed(tmp_path, monkeypatch): + called = [] + monkeypatch.setattr(runner, "run_pass", lambda *a: called.append(a) or {"status": "ok"}) + for i, kw in enumerate(({"results": []}, {"failure": "nonconformant: index"})): + b, ft, _ = _bm(tmp_path / str(i), **kw) + tar_path = pathlib.Path(runner.archive_path(b.results.directory, "hugo")) + tar_path.parent.mkdir(parents=True) + tar_path.write_bytes(b"x") + b._Benchmarker__benchmark(ft, open(tmp_path / "log", "w")) + assert not tar_path.exists() + assert called == [] diff --git a/tests/test_quality_seo.py b/tests/test_quality_seo.py new file mode 100644 index 0000000..48154d1 --- /dev/null +++ b/tests/test_quality_seo.py @@ -0,0 +1,86 @@ +import pathlib + +from toolset.quality import seo, site + +SITE = pathlib.Path(__file__).parent / "fixtures" / "quality" / "site" + +RICH = """ +Post title + + + + + + + + + + +

    T

    S

    a""" + + +def test_reference_page_has_only_title_and_lang(): + s = seo.page_signals((SITE / "post" / "hello" / "index.html").read_text()) + assert s["title"] is True and s["titleLength"] == 5 and s["lang"] == "en" + assert s["description"] is False and s["viewport"] is False and s["canonical"] is False + assert s["jsonLd"] is False and s["jsonLdValid"] is None and s["robotsMeta"] is None + assert s["h1"] == 1 and s["headingSkip"] is False and s["imgWithoutAlt"] == 0 + assert s["present"] == 2 + + +def test_rich_page_has_every_signal(): + s = seo.page_signals(RICH) + assert s["present"] == len(seo.PAGE_SIGNALS) == 12 + assert s["descriptionLength"] == len("A post about things") + assert s["robotsMeta"] == "index,follow" and s["hreflang"] == 2 + assert s["jsonLdBlocks"] == 1 and s["jsonLdValid"] is True + assert s["faviconLink"] is True and s["imgWithoutAlt"] == 0 + + +def test_broken_json_ld(): + s = seo.page_signals('') + assert s["jsonLd"] is True and s["jsonLdValid"] is False + + +def test_heading_skip_images_and_empty_links(): + html = ('

    a

    b

    ' + '' + 'logo no href') + s = seo.page_signals(html) + assert s["headingSkip"] is True + assert s["imgWithoutAlt"] == 1 + assert s["emptyLinks"] == 2 + + +def test_page_signals_malformed_html(): + html = b"x

    unclosed \xff\xfe".decode("utf-8", "replace") + s = seo.page_signals(html) + assert s["title"] is True and s["lang"] == "en" + + +def test_site_signals(tmp_path): + (tmp_path / "robots.txt").write_text("User-agent: *\n") + (tmp_path / "sitemap.xml").write_text( + '' + '/a//b/') + (tmp_path / "feed.xml").write_text("") + (tmp_path / "favicon.ico").write_bytes(b"\x00") + s = seo.site_signals(str(tmp_path), []) + assert s == {"robotsTxt": True, "sitemap": {"present": True, "valid": True, "urls": 2}, + "feed": True, "favicon": True} + + +def test_site_signals_invalid_sitemap_and_link_favicon(tmp_path): + (tmp_path / "sitemap.xml").write_text("") + s = seo.site_signals(str(tmp_path), [{"faviconLink": True}]) + assert s["sitemap"] == {"present": True, "valid": False, "urls": None} + assert s["favicon"] is True and s["robotsTxt"] is False and s["feed"] is False + + +def test_seo_section_fixture(): + pages = site.resolve_pages(str(SITE)) + section = seo.seo_section(str(SITE), pages) + assert section["total"] == 12 + assert section["present"] == {"index": 2, "post": 2, "404": 2} + assert section["site"]["feed"] is True and section["site"]["sitemap"]["present"] is False + assert set(section["pages"]) == {"index", "post", "404"} diff --git a/tests/test_quality_site.py b/tests/test_quality_site.py new file mode 100644 index 0000000..1ec23ac --- /dev/null +++ b/tests/test_quality_site.py @@ -0,0 +1,168 @@ +import io +import json +import pathlib +import tarfile + +import pytest + +from toolset.quality import site + +SITE = pathlib.Path(__file__).parent / "fixtures" / "quality" / "site" + + +def test_generator_config(tmp_path): + (tmp_path / "benchmark_config.json").write_text(json.dumps( + {"framework": "hugo", "config": [{"output_folder": "public", "output_glob": "post/*/index.html"}]})) + assert site.generator_config(str(tmp_path)) == ("public", "post/*/index.html") + + +def _tar(entries): + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w") as tar: + for name, data in entries: + info = tarfile.TarInfo(name) + if data is None: + info.type = tarfile.DIRTYPE + tar.addfile(info) + else: + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + return buf.getvalue() + + +def test_extract_strips_top_dir(tmp_path): + tar_path = tmp_path / "site.tar" + tar_path.write_bytes(_tar([("public", None), ("public/index.html", b"

    i

    "), + ("public/post/a/index.html", b"

    a

    ")])) + dest = tmp_path / "site" + assert site.extract(str(tar_path), str(dest)) == str(dest) + assert (dest / "index.html").read_bytes() == b"

    i

    " + assert (dest / "post" / "a" / "index.html").exists() + + +def test_extract_rejects_path_traversal(tmp_path): + tar_path = tmp_path / "site.tar" + tar_path.write_bytes(_tar([("public/../../evil.html", b"x")])) + with pytest.raises(tarfile.TarError): + site.extract(str(tar_path), str(tmp_path / "site")) + assert not (tmp_path.parent / "evil.html").exists() + + +def test_resolve_pages_fixture(): + assert site.resolve_pages(str(SITE)) == { + "index": {"url": "/", "file": "index.html"}, + "post": {"url": "/post/hello/", "file": "post/hello/index.html", "source": "index"}, + "404": {"url": "/404.html", "file": "404.html"}, + } + + +@pytest.mark.parametrize("href,url,file", [ + ("/post/a/", "/post/a/", "post/a/index.html"), + ("post/a/", "/post/a/", "post/a/index.html"), + ("./post/a/index.html", "/post/a/index.html", "post/a/index.html"), + ("/post/a?x=1#t", "/post/a", "post/a/index.html"), + ("/post/a", "/post/a", "post/a/index.html"), + ("/post/b", "/post/b", "post/b.html"), +]) +def test_resolve_pages_variants(tmp_path, href, url, file): + (tmp_path / "post" / "a").mkdir(parents=True) + (tmp_path / "post" / "a" / "index.html").write_text("a") + (tmp_path / "post" / "b.html").write_text("b") + (tmp_path / "index.html").write_text( + '' % href) + pages = site.resolve_pages(str(tmp_path)) + assert pages["post"] == {"url": url, "file": file, "source": "index"} + + +def test_resolve_pages_without_post_link(tmp_path): + (tmp_path / "index.html").write_text('x') + with pytest.raises(ValueError, match="post-item"): + site.resolve_pages(str(tmp_path)) + + +def test_resolve_pages_link_to_missing_file_without_glob(tmp_path): + (tmp_path / "index.html").write_text('
  • g
    • ') + with pytest.raises(ValueError, match="post-item"): + site.resolve_pages(str(tmp_path)) + + +def _site(tmp_path, index, files): + (tmp_path / "index.html").write_text(index) + for f in files: + (tmp_path / f).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / f).write_text("x") + + +def test_resolve_pages_glob_fallback_dir_style(tmp_path): + _site(tmp_path, 'x', ["post/b/index.html", "post/a/index.html", "404.html"]) + pages = site.resolve_pages(str(tmp_path), "post/*/index.html") + assert pages["post"] == {"url": "/post/a/", "file": "post/a/index.html", "source": "glob"} + assert pages["404"] == {"url": "/404.html", "file": "404.html"} + + +def test_resolve_pages_glob_fallback_html_style(tmp_path): + _site(tmp_path, "

    none

    ", ["posts/b.html", "posts/a.html"]) + pages = site.resolve_pages(str(tmp_path), "posts/*.html") + assert pages["post"] == {"url": "/posts/a.html", "file": "posts/a.html", "source": "glob"} + + +def test_resolve_pages_missing_link_target_falls_back_to_glob(tmp_path): + _site(tmp_path, '
  • g
    • ', ["post/a/index.html"]) + pages = site.resolve_pages(str(tmp_path), "post/*/index.html") + assert pages["post"]["source"] == "glob" and pages["post"]["url"] == "/post/a/" + + +def test_resolve_pages_no_404_file(tmp_path): + _site(tmp_path, '
  • a
    • ', ["post/a/index.html"]) + pages = site.resolve_pages(str(tmp_path), "post/*/index.html") + assert set(pages) == {"index", "post"} + + +def test_resolve_pages_nothing_found(tmp_path): + _site(tmp_path, "

    none

    ", ["other/file.txt"]) + with pytest.raises(ValueError, match=r"post-item.*post/\*/index\.html"): + site.resolve_pages(str(tmp_path), "post/*/index.html") + + +def test_resolve_pages_requires_index(tmp_path): + with pytest.raises(ValueError, match="index.html"): + site.resolve_pages(str(tmp_path), "*.html") + + +def test_list_files_is_sorted_posix(): + files = site.list_files(str(SITE)) + assert files == sorted(files) and "post/hello/index.html" in files + + +def test_extra_output_fixture(): + extra = site.extra_output(str(SITE), "post/*/index.html") + assert extra["files"] == 2 + assert set(extra["byExtension"]) == {".html", ".xml"} + assert extra["byExtension"][".html"]["files"] == 1 + assert extra["bytes"] == sum(e["bytes"] for e in extra["byExtension"].values()) + + +def test_works_without_js(): + post = "post/hello/index.html" + rendered = {"title": "Hello", "firstParagraph": "First paragraph of the post."} + assert site.works_without_js(str(SITE), post, rendered) is True + assert site.works_without_js(str(SITE), post, {"title": "Hello", "firstParagraph": "Only after JS"}) is False + assert site.works_without_js(str(SITE), post, None) is None + assert site.works_without_js(str(SITE), post, {"error": "boom"}) is None + + +def test_text_of_collapses_whitespace(): + assert site.text_of("

    a\n b

    ") == "a b" + + +def test_text_of_keeps_inline_punctuation(): + assert site.text_of("

    See x.

    ") == "See x." + + +def test_works_without_js_inline_markup(tmp_path): + (tmp_path / "post.html").write_text( + 'T' + '

    T

    See x.

    ' + '') + rendered = {"title": "T", "firstParagraph": "See x."} + assert site.works_without_js(str(tmp_path), "post.html", rendered) is True diff --git a/tests/test_quality_summary.py b/tests/test_quality_summary.py new file mode 100644 index 0000000..3c36584 --- /dev/null +++ b/tests/test_quality_summary.py @@ -0,0 +1,67 @@ +import csv +import io +import json +import pathlib + +from toolset.quality import reduce, site +from toolset.utils import summary +from toolset.utils.results import parse_build_output + +FIX = pathlib.Path(__file__).parent / "fixtures" / "quality" +RAW_OK = pathlib.Path(__file__).parent / "fixtures" / "raw_ok.txt" + + +def _quality(): + raw = json.loads((FIX / "raw.json").read_text()) + pages = site.resolve_pages(str(FIX / "site")) + return reduce.build_quality(raw, str(FIX / "site"), pages, "post/*/index.html") + + +def _data(with_quality): + [entry] = parse_build_output(RAW_OK.read_text(), "50", "5", "1") + data = {"name": "r", "frameworks": ["hugo", "gatsby"], + "rawData": {"datarate": {"hugo": [entry]}}, + "succeeded": {"datarate": ["hugo"]}, "failed": {"datarate": ["gatsby"]}, "excluded": []} + if with_quality: + data["quality"] = {"hugo": _quality(), "gatsby": {"status": "error", "error": "RuntimeError: boom"}} + return data + + +def test_quality_rows(): + rows = summary.quality_rows(_data(True)) + assert [r["framework"] for r in rows] == ["gatsby", "hugo"] + hugo = rows[1] + assert hugo["performance"] == 0.98 and hugo["lcpMs"] == 800 + assert hugo["postJsBytes"] == 0 and hugo["jsClass"] == "none" + assert hugo["htmlErrors"] == 1 and hugo["a11yViolations"] == 2 and hugo["brokenLinks"] == 2 + assert (hugo["seoPresent"], hugo["seoTotal"]) == (2, 12) + assert rows[0]["status"] == "error" and rows[0]["performance"] is None + + +def test_quality_markdown_section(): + lines = summary.quality_markdown(summary.quality_rows(_data(True))) + text = "\n".join(lines) + assert lines[0] == "## Qualidade" + assert "| hugo | 98 | 100 | 100 | 82 | 800 | 0.0 | none |" in text + assert "2/12" in text and "no meta description and no viewport" in text + assert "gatsby: RuntimeError: boom" in text + assert "Rank" not in text + + +def test_quality_csv_has_every_numeric_field(): + rows = summary.quality_rows(_data(True)) + parsed = list(csv.DictReader(io.StringIO(summary.quality_csv(rows)))) + assert parsed[1]["framework"] == "hugo" + header = list(parsed[0].keys()) + assert header[:len(summary.QUALITY_COLUMNS)] == summary.QUALITY_COLUMNS + assert "lighthouse.desktop.index.scores.seo.median" in header + assert "weight.brotliBytes" in header + + +def test_quality_never_changes_timing_summary(): + without, with_q = _data(False), _data(True) + assert summary.build_rows(with_q, {}) == summary.build_rows(without, {}) + md_without = summary.to_markdown(summary.build_rows(without, {}), without) + md_with = summary.to_markdown(summary.build_rows(with_q, {}), with_q) + assert md_with.startswith(md_without.rstrip("\n")) + assert "## Qualidade" in md_with and "## Qualidade" not in md_without diff --git a/toolset/benchmark/benchmarker.py b/toolset/benchmark/benchmarker.py index c230153..8ffdc38 100644 --- a/toolset/benchmark/benchmarker.py +++ b/toolset/benchmark/benchmarker.py @@ -16,6 +16,8 @@ from toolset.utils.audit import Audit from toolset.utils import resources from toolset.benchmark import noise +from toolset.quality import runner as quality_runner +from toolset.quality import site as quality_site class Benchmarker: @@ -242,6 +244,8 @@ def benchmark_type(test_type): with open(raw_file, 'w'): pass + export = self.__quality_export(framework_test) + self._quality_kwargs = {'export': export} if export else {} if not test.failed: script = self.config.types[test_type].get_script_name() script_variables = self.config.types[test_type].get_script_variables() @@ -251,7 +255,8 @@ def benchmark_type(test_type): try: outcome = self.docker_helper.benchmark( framework_test, script, script_variables, raw_file, - self.resolve_resources(), self.config.run_test_timeout_seconds) + self.resolve_resources(), self.config.run_test_timeout_seconds, + **self._quality_kwargs) finally: self.__end_logging() @@ -272,6 +277,12 @@ def benchmark_type(test_type): self.results.report_benchmark_results(framework_test, test_type, results['results'], results.get('unsupported', False), results.get('failureReason')) + if export: + if results['results'] and not results.get('failureReason') \ + and not results.get('unsupported'): + self.__quality_pass(framework_test, benchmark_log) + else: + self.__discard_export(export[1]) log("Complete", file=benchmark_log) return bool(results['results']) or bool(results.get('unsupported', False)) @@ -281,6 +292,45 @@ def benchmark_type(test_type): ok = False return ok + def __quality_export(self, framework_test): + '''(output_folder, dest_tar) when the quality pass runs in this cell (spec 009 R-1).''' + if quality_runner.skipped(self.config): + log("quality: skipped (only cell nf50-cs5 of a suite)") + return None + if not quality_runner.eligible(self.config): + return None + try: + output_folder, _glob = quality_site.generator_config(framework_test.directory) + except Exception as e: + log("quality: cannot read output_folder: %s" % e) + return None + return (output_folder, + quality_runner.archive_path(self.results.directory, framework_test.name)) + + def __quality_pass(self, framework_test, benchmark_log): + '''Untimed; never changes the build result (spec 009 R-2).''' + try: + log("QUALITY PASS %s (untimed)" % framework_test.name, file=benchmark_log, border='*') + quality = quality_runner.run_pass(self.docker_helper, framework_test, self.results.directory) + self.results.add_quality(framework_test.name, quality, + getattr(self.docker_helper, '_quality_image', None)) + log("quality: %s" % quality.get('status'), file=benchmark_log) + except Exception as e: + error = "%s: %s" % (type(e).__name__, e) + try: + log("quality: pass failed: %s" % error) + self.results.add_quality(framework_test.name, {'status': 'error', 'error': error}, None) + except Exception: + pass + + def __discard_export(self, tar_path): + '''The pass will not run: drop the exported site so it does not linger (spec 009 R-2).''' + try: + if os.path.exists(tar_path): + os.remove(tar_path) + except Exception as e: + log("quality: could not remove %s: %s" % (tar_path, e)) + def __noise_control(self, framework_test, test_type, script, script_variables, raw_file, results, benchmark_log): ''' @@ -304,7 +354,8 @@ def __noise_control(self, framework_test, test_type, script, try: outcome = self.docker_helper.benchmark( framework_test, script, variables, raw_file, - self.resolve_resources(), self.config.run_test_timeout_seconds) + self.resolve_resources(), self.config.run_test_timeout_seconds, + **getattr(self, '_quality_kwargs', {})) finally: self.__end_logging() second = self.results.parse_test(framework_test, test_type) diff --git a/toolset/quality/__init__.py b/toolset/quality/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/toolset/quality/reduce.py b/toolset/quality/reduce.py new file mode 100644 index 0000000..354c4da --- /dev/null +++ b/toolset/quality/reduce.py @@ -0,0 +1,176 @@ +''' +Reduces the collector output (quality/run.mjs raw.json) into quality.json +(spec 009 R-8, R-10..R-16). Pure functions; the contracts are in plan.md. +''' +import fnmatch +import posixpath +import statistics + +from toolset.quality import seo, site + +SECTIONS = ('lighthouse', 'seo', 'html', 'a11y', 'links', 'weight', 'js', 'extra') +PRESETS = ('mobile', 'desktop') +SCORES = ('performance', 'accessibility', 'best-practices', 'seo') +METRICS = ('fcp', 'lcp', 'tbt', 'cls', 'si') +RESOURCES = ('totalBytes', 'jsBytes', 'cssBytes', 'requests', 'domSize') +IMPACTS = ('critical', 'serious', 'moderate', 'minor') +MAX_BROKEN_LISTED = 20 + + +def spread(values): + vs = [v for v in values if v is not None] + if not vs: + return None + return {'median': statistics.median(vs), 'min': min(vs), 'max': max(vs)} + + +def _is_error(part): + return isinstance(part, dict) and 'error' in part + + +def _error(part): + return {'status': 'error', 'error': part['error']} + + +def _median_run(runs): + ranked = sorted(runs, key=lambda r: (r['scores'].get('performance') is None, + r['scores'].get('performance') or 0)) + return ranked[(len(ranked) - 1) // 2] + + +def _lighthouse_page(runs): + if _is_error(runs): + return _error(runs) + good = [r for r in runs if not _is_error(r)] + if not good: + return {'status': 'error', 'error': runs[0]['error'] if runs else 'no runs'} + return { + 'status': 'ok', + 'runs': len(good), + 'scores': {k: spread([r['scores'].get(k) for r in good]) for k in SCORES}, + 'metrics': {k: spread([r['metrics'].get(k) for r in good]) for k in METRICS}, + 'resources': {k: spread([r['resources'].get(k) for r in good]) for k in RESOURCES}, + 'failedAudits': _median_run(good)['failedAudits'], + } + + +def reduce_lighthouse(raw_lh): + return {preset: {page: _lighthouse_page(runs) + for page, runs in (raw_lh.get(preset) or {}).items()} + for preset in PRESETS} + + +def reduce_axe(raw_axe): + out = {} + for page, data in raw_axe.items(): + if _is_error(data): + out[page] = _error(data) + continue + counts = {impact: 0 for impact in IMPACTS} + for v in data.get('violations') or []: + if v.get('impact') in counts: + counts[v['impact']] += 1 + out[page] = {'violations': counts, 'total': len(data.get('violations') or []), + 'rules': sorted({v['id'] for v in data.get('violations') or []})} + return out + + +def reduce_html(raw_html): + out = {} + for page, data in raw_html.items(): + if _is_error(data): + out[page] = _error(data) + continue + messages = data.get('messages') or [] + out[page] = {'errors': sum(1 for m in messages if m.get('severity') == 2), + 'warnings': sum(1 for m in messages if m.get('severity') == 1), + 'rules': sorted({m['ruleId'] for m in messages if m.get('ruleId')})} + return out + + +def _kind(url): + if '#' in url: + return 'anchor' + path = url.split('?', 1)[0] + if path.endswith('/') or path.endswith('.html') or not posixpath.splitext(path)[1]: + return 'link' + return 'asset' + + +def reduce_links(raw_links): + report = raw_links.get('report') or {} + fail_map = report.get('fail_map') or report.get('error_map') or {} + items = [] + for source, failures in sorted(fail_map.items()): + for f in failures: + status = f.get('status') + items.append({'source': source, 'url': f.get('url'), + 'status': status.get('text') if isinstance(status, dict) else status}) + by_kind = {'link': 0, 'anchor': 0, 'asset': 0} + for item in items: + by_kind[_kind(item['url'] or '')] += 1 + return {'broken': len(items), 'byKind': by_kind, 'items': items[:MAX_BROKEN_LISTED]} + + +def weight(files, output_glob): + posts = [f['bytes'] for f in files if fnmatch.fnmatch(f['path'], output_glob)] + total = sum(f['bytes'] for f in files) + reference = sum(f['bytes'] for f in files if f['path'] in site.REFERENCE_ASSETS) + return {'bytes': total, + 'gzipBytes': sum(f['gzip'] for f in files), + 'brotliBytes': sum(f['br'] for f in files), + 'bytesPerPost': round(sum(posts) / len(posts)) if posts else None, + 'generatorAddedBytes': total - reference} + + +def js_section(lighthouse, files, works): + pages = {} + for page, data in (lighthouse.get('mobile') or {}).items(): + js = (data.get('resources') or {}).get('jsBytes') if data.get('status') == 'ok' else None + pages[page] = js['median'] if js else None + known = [v for v in pages.values() if v is not None] + js_class = None if not known else ('none' if all(v == 0 for v in known) else 'hydrated') + return {'pages': pages, + 'diskBytes': sum(f['bytes'] for f in files if f['path'].endswith(('.js', '.mjs'))), + 'jsClass': js_class, + 'worksWithoutJs': works} + + +def _raw(raw, key): + part = raw.get(key) + if part is None: + raise RuntimeError('%s missing in collector output' % key) + if _is_error(part): + raise RuntimeError(part['error']) + return part + + +def _section(fn): + try: + return dict({'status': 'ok'}, **fn()) + except Exception as e: + return {'status': 'error', 'error': '%s: %s' % (type(e).__name__, e)} + + +def build_quality(raw, site_dir, pages, output_glob): + '''quality.json (R-16): one section per check, each ok or error.''' + q = {'status': 'ok', 'pages': {k: v['url'] for k, v in pages.items()}, + 'postSource': pages['post'].get('source'), + 'tools': raw.get('tools')} + q['lighthouse'] = _section(lambda: reduce_lighthouse(_raw(raw, 'lighthouse'))) + q['seo'] = _section(lambda: seo.seo_section(site_dir, pages)) + q['html'] = _section(lambda: {'pages': reduce_html(_raw(raw, 'html'))}) + q['a11y'] = _section(lambda: {'pages': reduce_axe(_raw(raw, 'axe'))}) + q['links'] = _section(lambda: reduce_links(_raw(raw, 'links'))) + q['weight'] = _section(lambda: weight(_raw(raw, 'files'), output_glob)) + + def js(): + if q['lighthouse']['status'] != 'ok': + raise RuntimeError('lighthouse section failed') + rendered = (raw.get('rendered') or {}).get('post') if not _is_error(raw.get('rendered')) else None + works = site.works_without_js(site_dir, pages['post']['file'], rendered) + return js_section(q['lighthouse'], _raw(raw, 'files'), works) + + q['js'] = _section(js) + q['extra'] = _section(lambda: site.extra_output(site_dir, output_glob)) + return q diff --git a/toolset/quality/runner.py b/toolset/quality/runner.py new file mode 100644 index 0000000..30109cc --- /dev/null +++ b/toolset/quality/runner.py @@ -0,0 +1,62 @@ +''' +The untimed quality pass of one generator (spec 009 R-1..R-3, R-16). +''' +import json +import os + +from toolset.quality import reduce, site +from toolset.utils.output_helper import log + +# R-1: with --suite, the pass runs only in this (numberOfFiles, contentSize) cell +QUALITY_CELL = (50, 5.0) + + +def _cell(config): + return int(config.number_of_files), float(config.content_size) + + +def eligible(config): + if not getattr(config, 'quality', False): + return False + if getattr(config, 'suite_info', None) is None: + return True + return _cell(config) == QUALITY_CELL + + +def skipped(config): + return bool(getattr(config, 'quality', False)) and not eligible(config) + + +def archive_path(results_dir, name): + return os.path.join(results_dir, 'quality', name, 'site.tar') + + +def run_pass(docker_helper, test, results_dir): + '''Runs the pass, writes quality.json and returns it. Never raises.''' + out_dir = os.path.join(results_dir, 'quality', test.name) + tar_path = archive_path(results_dir, test.name) + try: + output_folder, output_glob = site.generator_config(test.directory) + if not os.path.exists(tar_path): + raise RuntimeError('no exported output (%s)' % output_folder) + site_dir = site.extract(tar_path, os.path.join(out_dir, 'site')) + pages = site.resolve_pages(site_dir, output_glob) + site_name = os.path.basename(output_folder.rstrip('/')) + raw = docker_helper.run_quality(tar_path, site_name, pages) + with open(os.path.join(out_dir, 'raw.json'), 'w') as f: + json.dump(raw, f, indent=2) + quality = reduce.build_quality(raw, site_dir, pages, output_glob) + except Exception as e: + quality = {'status': 'error', 'error': '%s: %s' % (type(e).__name__, e)} + try: + os.makedirs(out_dir, exist_ok=True) + with open(os.path.join(out_dir, 'quality.json'), 'w') as f: + json.dump(quality, f, indent=2) + except Exception as e: + log("quality: could not write quality.json: %s: %s" % (type(e).__name__, e)) + try: + if os.path.exists(tar_path): + os.remove(tar_path) + except Exception as e: + log("quality: could not remove %s: %s: %s" % (tar_path, type(e).__name__, e)) + return quality diff --git a/toolset/quality/seo.py b/toolset/quality/seo.py new file mode 100644 index 0000000..a605b2b --- /dev/null +++ b/toolset/quality/seo.py @@ -0,0 +1,164 @@ +''' +SEO signals of a generated page and site (spec 009 R-9). Reported, not scored. +''' +import json +import os +import re +import xml.etree.ElementTree as ET +from html.parser import HTMLParser + +from toolset.quality import site + +# The per-page signals counted in seo.present (out of len(PAGE_SIGNALS)) +PAGE_SIGNALS = ('title', 'lang', 'description', 'viewport', 'canonical', 'ogTitle', + 'ogDescription', 'ogType', 'ogUrl', 'twitterCard', 'jsonLd', 'robotsMeta') +META = {('name', 'description'): 'description', ('name', 'viewport'): 'viewport', + ('property', 'og:title'): 'ogTitle', ('property', 'og:description'): 'ogDescription', + ('property', 'og:type'): 'ogType', ('property', 'og:url'): 'ogUrl', + ('name', 'twitter:card'): 'twitterCard', ('name', 'robots'): 'robotsMeta'} +FEEDS = ('feed.xml', 'atom.xml', 'rss.xml', 'index.xml', 'feed/index.xml', 'rss/index.xml') +FAVICONS = ('favicon.ico', 'favicon.svg', 'favicon.png') + + +class _Collector(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.lang = None + self.title = None + self.meta = {} + self.canonical = None + self.hreflang = 0 + self.favicon_link = False + self.jsonld = [] + self.headings = [] + self.img_without_alt = 0 + self.empty_links = 0 + self._capture = None # 'title' or 'jsonld' + self._buf = [] + self._links = [] # one bool per open : has an accessible name + + def handle_starttag(self, tag, attrs): + a = {k: (v or '') for k, v in attrs} + if tag == 'html' and a.get('lang'): + self.lang = a['lang'] + elif tag == 'title' and self.title is None: + self._capture, self._buf = 'title', [] + elif tag == 'meta': + for (attr, value), key in META.items(): + if a.get(attr, '').lower() == value: + self.meta[key] = a.get('content', '') + elif tag == 'link': + rel = a.get('rel', '').lower().split() + if 'canonical' in rel and a.get('href'): + self.canonical = a['href'] + if 'alternate' in rel and 'hreflang' in a: + self.hreflang += 1 + if 'icon' in rel: + self.favicon_link = True + elif tag == 'script' and a.get('type', '').lower() == 'application/ld+json': + self._capture, self._buf = 'jsonld', [] + elif re.fullmatch(r'h[1-6]', tag): + self.headings.append(int(tag[1])) + elif tag == 'img': + if 'alt' not in a: + self.img_without_alt += 1 + elif a['alt'].strip() and self._links: + self._links[-1] = True + elif tag == 'a' and 'href' in a: + self._links.append(bool(a.get('aria-label', '').strip() or a.get('title', '').strip())) + + def handle_endtag(self, tag): + if tag == 'title' and self._capture == 'title': + self.title = ''.join(self._buf).strip() + self._capture = None + elif tag == 'script' and self._capture == 'jsonld': + self.jsonld.append(''.join(self._buf)) + self._capture = None + elif tag == 'a' and self._links: + if not self._links.pop(): + self.empty_links += 1 + + def handle_data(self, data): + if self._capture: + self._buf.append(data) + if self._links and data.strip(): + self._links[-1] = True + + +def _skips(levels): + return any(b > a + 1 for a, b in zip(levels, levels[1:])) + + +def page_signals(html): + c = _Collector() + c.feed(html) + c.close() + valid = [] + for block in c.jsonld: + try: + json.loads(block) + valid.append(True) + except ValueError: + valid.append(False) + s = { + 'title': bool(c.title), + 'titleLength': len(c.title or ''), + 'lang': c.lang, + 'description': bool(c.meta.get('description')), + 'descriptionLength': len(c.meta.get('description', '')), + 'viewport': 'viewport' in c.meta, + 'canonical': bool(c.canonical), + 'ogTitle': 'ogTitle' in c.meta, + 'ogDescription': 'ogDescription' in c.meta, + 'ogType': 'ogType' in c.meta, + 'ogUrl': 'ogUrl' in c.meta, + 'twitterCard': 'twitterCard' in c.meta, + 'jsonLd': bool(c.jsonld), + 'jsonLdBlocks': len(c.jsonld), + 'jsonLdValid': all(valid) if valid else None, + 'robotsMeta': c.meta.get('robotsMeta'), + 'hreflang': c.hreflang, + 'h1': c.headings.count(1), + 'headingSkip': _skips(c.headings), + 'imgWithoutAlt': c.img_without_alt, + 'emptyLinks': c.empty_links, + 'faviconLink': c.favicon_link, + } + s['present'] = sum(1 for k in PAGE_SIGNALS if s[k] not in (False, None, '')) + return s + + +def _sitemap(site_dir): + path = os.path.join(site_dir, 'sitemap.xml') + if not os.path.isfile(path): + return {'present': False, 'valid': None, 'urls': None} + try: + root = ET.parse(path).getroot() + except ET.ParseError: + return {'present': True, 'valid': False, 'urls': None} + if not root.tag.endswith(('urlset', 'sitemapindex')): + return {'present': True, 'valid': False, 'urls': None} + return {'present': True, 'valid': True, + 'urls': sum(1 for el in root.iter() if el.tag.endswith('loc'))} + + +def site_signals(site_dir, page_signals_list): + exists = lambda rel: os.path.isfile(os.path.join(site_dir, rel)) # noqa: E731 + return { + 'robotsTxt': exists('robots.txt'), + 'sitemap': _sitemap(site_dir), + 'feed': any(exists(f) for f in FEEDS), + 'favicon': any(exists(f) for f in FAVICONS) + or any(s.get('faviconLink') for s in page_signals_list), + } + + +def seo_section(site_dir, pages): + signals = {key: page_signals(site._read(os.path.join(site_dir, page['file']))) + for key, page in pages.items()} + return { + 'total': len(PAGE_SIGNALS), + 'pages': signals, + 'present': {key: s['present'] for key, s in signals.items()}, + 'site': site_signals(site_dir, list(signals.values())), + } diff --git a/toolset/quality/site.py b/toolset/quality/site.py new file mode 100644 index 0000000..754c50a --- /dev/null +++ b/toolset/quality/site.py @@ -0,0 +1,188 @@ +''' +Pages and files of a generated site (spec 009 R-3, R-7, R-14, R-15). +''' +import fnmatch +import json +import os +import posixpath +import re +import tarfile +from html.parser import HTMLParser +from urllib.parse import unquote, urlsplit + +# SF 005 reference assets: every generator ships them unchanged (R-13, R-15) +REFERENCE_ASSETS = ('assets/ssgberk.css', 'assets/ssgberk.png') + + +def generator_config(test_directory): + '''(output_folder, output_glob) of benchmark_config.json config[0].''' + with open(os.path.join(test_directory, 'benchmark_config.json')) as f: + config = json.load(f)['config'][0] + return config['output_folder'], config['output_glob'] + + +def extract(tar_path, dest): + ''' + Extracts the archive DockerHelper.benchmark exported (a single top + directory named after output_folder) into dest, without that directory. + ''' + os.makedirs(dest, exist_ok=True) + with tarfile.open(tar_path) as tar: + members = [] + for member in tar.getmembers(): + parts = member.name.split('/', 1) + if len(parts) < 2 or not parts[1]: + continue + member.name = parts[1] + members.append(member) + # filter='data' rejects absolute paths, '..' and links out of dest + tar.extractall(dest, members=members, filter='data') + return dest + + +def url_to_file(site_dir, url): + '''The file a static server would return for url, or None.''' + p = unquote(urlsplit(url).path).lstrip('/') + if p == '' or p.endswith('/'): + candidates = [posixpath.join(p, 'index.html')] + else: + candidates = [p, p + '/index.html', p + '.html'] + for c in candidates: + if os.path.isfile(os.path.join(site_dir, c)): + return posixpath.normpath(c) + return None + + +class _PostLink(HTMLParser): + '''First a[href] after the first element whose class list has post-item.''' + + def __init__(self): + super().__init__(convert_charrefs=True) + self.in_item = False + self.href = None + + def handle_starttag(self, tag, attrs): + a = {k: (v or '') for k, v in attrs} + if 'post-item' in a.get('class', '').split(): + self.in_item = True + if tag == 'a' and self.in_item and self.href is None and a.get('href'): + self.href = a['href'] + + +def _read(path): + with open(path, encoding='utf-8', errors='replace') as f: + return f.read() + + +def _glob_post(site_dir, output_glob): + '''(url, file) of the first sorted file matching output_glob, or None.''' + for f in list_files(site_dir): + if fnmatch.fnmatch(f, output_glob): + url = '/' + f + if posixpath.basename(f) == 'index.html': + url = url[:-len('index.html')] + return url, f + return None + + +def resolve_pages(site_dir, output_glob=None): + ''' + The audited pages (R-7): index, the post (first .post-item link, else the + first file matching output_glob) and 404 when 404.html exists. + ''' + index_path = os.path.join(site_dir, 'index.html') + if not os.path.isfile(index_path): + raise ValueError('no index.html at the site root') + parser = _PostLink() + parser.feed(_read(index_path)) + post = None + if parser.href: + path = urlsplit(parser.href).path + url = posixpath.normpath(posixpath.join('/', path)) + if path.endswith('/') and url != '/': + url += '/' + post_file = url_to_file(site_dir, url) + if post_file is not None: + post = {'url': url, 'file': post_file, 'source': 'index'} + if post is None and output_glob: + found = _glob_post(site_dir, output_glob) + if found: + post = {'url': found[0], 'file': found[1], 'source': 'glob'} + if post is None: + raise ValueError('no post page: no a[href] to an existing file inside ' + '.post-item on the index and no file matches %s' % output_glob) + pages = {'index': {'url': '/', 'file': 'index.html'}, 'post': post} + if os.path.isfile(os.path.join(site_dir, '404.html')): + pages['404'] = {'url': '/404.html', 'file': '404.html'} + return pages + + +def list_files(site_dir): + out = [] + for root, _dirs, files in os.walk(site_dir): + for name in files: + rel = os.path.relpath(os.path.join(root, name), site_dir) + out.append(rel.replace(os.sep, '/')) + return sorted(out) + + +def extra_output(site_dir, output_glob): + '''R-15: files outside the SF 005 page set, by extension.''' + by_ext = {} + total_files = total_bytes = 0 + for f in list_files(site_dir): + if f in ('index.html', '404.html') or f in REFERENCE_ASSETS \ + or fnmatch.fnmatch(f, output_glob): + continue + size = os.path.getsize(os.path.join(site_dir, f)) + ext = os.path.splitext(f)[1].lower() or '(none)' + entry = by_ext.setdefault(ext, {'files': 0, 'bytes': 0}) + entry['files'] += 1 + entry['bytes'] += size + total_files += 1 + total_bytes += size + return {'files': total_files, 'bytes': total_bytes, + 'byExtension': dict(sorted(by_ext.items()))} + + +class _Text(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.parts = [] + self.skip = 0 + + def handle_starttag(self, tag, attrs): + if tag in ('script', 'style', 'template'): + self.skip += 1 + + def handle_endtag(self, tag): + if tag in ('script', 'style', 'template') and self.skip: + self.skip -= 1 + + def handle_data(self, data): + if not self.skip: + self.parts.append(data) + + +def text_of(html): + parser = _Text() + parser.feed(html) + parser.close() + # join without a separator, as textContent does, so inline markup does not + # insert spaces ("See x." -> "See x."); whitespace is collapsed after + return re.sub(r'\s+', ' ', ''.join(parser.parts)).strip() + + +def works_without_js(site_dir, post_file, rendered): + ''' + R-14: True when the static post HTML already holds the title and first + body paragraph that the browser shows after scripts ran. + ''' + if not isinstance(rendered, dict) or 'error' in rendered: + return None + title = re.sub(r'\s+', ' ', rendered.get('title') or '').strip() + first = re.sub(r'\s+', ' ', rendered.get('firstParagraph') or '').strip() + if not title or not first: + return None + static = text_of(_read(os.path.join(site_dir, post_file))) + return title in static and first in static diff --git a/toolset/run-tests.py b/toolset/run-tests.py index c7d59f9..45dc4d1 100644 --- a/toolset/run-tests.py +++ b/toolset/run-tests.py @@ -221,6 +221,13 @@ def build_parser(): action='store_true', default=False, help='Build generator images without the Docker layer cache') + parser.add_argument( + '--quality', + action='store_true', + default=False, + help='After the timed runs, audit the output of the last build (Lighthouse, SEO ' + 'signals, HTML, accessibility, links, weight); untimed. With --suite, only in ' + 'the cell nf50-cs5 (spec 009)') # Resource limits applied to every generator container of the run parser.add_argument( '--cpus', diff --git a/toolset/utils/benchmark_config.py b/toolset/utils/benchmark_config.py index 171d2f0..b0e019d 100644 --- a/toolset/utils/benchmark_config.py +++ b/toolset/utils/benchmark_config.py @@ -43,6 +43,7 @@ def __init__(self, args): self.require_conformance = bool(getattr(args, 'require_conformance', False)) self.verbose_build = args.verbose self.no_cache = bool(getattr(args, 'no_cache', False)) + self.quality = bool(getattr(args, 'quality', False)) self.cpus = getattr(args, 'cpus', 4.0) self.memory = getattr(args, 'memory', 8 * 1024 ** 3) self.cpuset = getattr(args, 'cpuset', 'auto') diff --git a/toolset/utils/docker_helper.py b/toolset/utils/docker_helper.py index 8ebfdb8..d6ec412 100644 --- a/toolset/utils/docker_helper.py +++ b/toolset/utils/docker_helper.py @@ -1,5 +1,9 @@ +import io +import json import os +import posixpath import socket +import tarfile import time import re import traceback @@ -18,6 +22,12 @@ TEST_IMAGE_PREFIX = 'ssgberk/test.' RUN_LABEL = 'ssgberk.run' +QUALITY_IMAGE = 'ssgberk/quality' +QUALITY_TIMEOUT_SECONDS = 1800 +QUALITY_MEMORY = '4g' +# collector output dir: outside /work so it can never land inside the site (a site +# folder named "out" is /work/out) +QUALITY_OUT = '/quality-out' class DockerHelper: @@ -33,6 +43,7 @@ def __init__(self, benchmarker=None): base_url=self.benchmarker.config.server_docker_host) last_build = None + _quality_image = None def _run_labels(self): return {RUN_LABEL: str(self.benchmarker.config.run_id)} @@ -330,11 +341,13 @@ def server_container_exists(self, container_id_or_name): return False def benchmark(self, framework_test, script, variables, raw_file, - resources, timeout_seconds): + resources, timeout_seconds, export=None): ''' Runs the generator container with fixed resource limits and returns {"status": "ok"|"timeout"|"oom", "exitCode": int}. The container is - always stopped and removed before returning. + always stopped and removed before returning. With export + (output_folder, dest_tar), a successful build's output is copied to + dest_tar first (spec 009 R-3). ''' def watch_container(container): @@ -408,9 +421,73 @@ def watch_container(container): # stopping the container after a timeout if status == 'ok' and (state.get('OOMKilled') or exit_code == 137): status = 'oom' + if export and status == 'ok' and exit_code == 0: + DockerHelper.__export_output(container, *export) finally: try: container.remove(force=True) except docker.errors.NotFound: pass return {'status': status, 'exitCode': exit_code} + + @staticmethod + def __export_output(container, output_folder, dest): + '''Copies / to dest as a tar. Never raises.''' + try: + workdir = (container.attrs.get('Config') or {}).get('WorkingDir') or '/' + bits, _ = container.get_archive(posixpath.join(workdir, output_folder)) + os.makedirs(os.path.dirname(dest), exist_ok=True) + with open(dest, 'wb') as f: + for chunk in bits: + f.write(chunk) + except Exception as e: + log("quality: could not export %s: %s" % (output_folder, e), color=Fore.YELLOW) + + def build_quality_image(self): + ''' + Builds ssgberk/quality from /quality once per process + (spec 009 R-4), outside the generator build-time accounting. + ''' + if self._quality_image: + return self._quality_image + client = docker.APIClient(base_url=self.benchmarker.config.server_docker_host) + for token in client.build(path=os.path.join(self.benchmarker.config.fw_root, 'quality'), + dockerfile='Dockerfile', tag=QUALITY_IMAGE, forcerm=True, + pull=True, timeout=3600, decode=True): + if 'errorDetail' in token: + raise RuntimeError(token['errorDetail']['message']) + if token.get('stream', '').strip(): + log(token['stream'].rstrip(), prefix='quality: ') + self._quality_image = self.server.images.get(QUALITY_IMAGE).id + return self._quality_image + + def run_quality(self, tar_path, site_name, pages, + timeout_seconds=QUALITY_TIMEOUT_SECONDS): + ''' + Runs quality/run.mjs on the exported site with no network and no + mounts (spec 009 R-4, R-5) and returns its raw.json. + ''' + image = self.build_quality_image() + command = ['node', '/quality/run.mjs', '--site', '/work/' + site_name, + '--pages', json.dumps(pages), '--out', QUALITY_OUT] + container = self.server.containers.create( + image, command=command, labels=self._run_labels(), network_mode='none', + mem_limit=QUALITY_MEMORY, working_dir='/work') + try: + with open(tar_path, 'rb') as f: + container.put_archive('/work', f.read()) + container.start() + result = container.wait(timeout=timeout_seconds) + try: + bits, _ = container.get_archive(QUALITY_OUT + '/raw.json') + except docker.errors.NotFound: + logs = container.logs().decode('utf-8', 'replace') + raise RuntimeError('collector wrote no raw.json (exit %s): %s' % ( + result.get('StatusCode'), logs[-2000:])) + with tarfile.open(fileobj=io.BytesIO(b''.join(bits))) as tar: + return json.load(tar.extractfile(tar.getmembers()[0])) + finally: + try: + container.remove(force=True) + except docker.errors.NotFound: + pass diff --git a/toolset/utils/results.py b/toolset/utils/results.py index cd55480..d5302d8 100644 --- a/toolset/utils/results.py +++ b/toolset/utils/results.py @@ -28,7 +28,7 @@ # Run-level keys a --parse keeps from the stored results.json: they describe # how the run was made, which raw.txt cannot tell. STORED_RUN_KEYS = ('suite', 'profile', 'resources', 'protocol', 'environment', - 'generators', 'imageBuild') + 'generators', 'imageBuild', 'quality') # Per-result keys a --parse keeps from the stored result. STORED_RESULT_KEYS = ('noisy', 'attempts', 'resources') # Failure reasons decided by the container outcome, not by raw.txt. @@ -214,11 +214,21 @@ def __init__(self, benchmarker): self.unsupported = dict() self.unsupported['datarate'] = [] self.failureReasons = dict() + self.quality = {} + self.quality_environment = None ############################################################################# # PUBLIC FUNCTIONS ############################################################################# + def add_quality(self, name, quality, image_id=None): + '''results.json quality. and environment.quality (spec 009 R-6, R-17).''' + self.quality[name] = quality + tools = quality.get('tools') + if self.quality_environment is None and isinstance(tools, dict) \ + and not tools.get('error'): + self.quality_environment = dict(tools, image=image_id) + def parse(self, tests): ''' Ensures that the system has all necessary software to run @@ -252,6 +262,9 @@ def write_summary(self): f.write(summary.to_csv(rows)) with open(os.path.join(self.directory, 'summary.md'), 'w') as f: f.write(markdown) + if data.get('quality'): + with open(os.path.join(self.directory, 'quality-summary.csv'), 'w', newline='') as f: + f.write(summary.quality_csv(summary.quality_rows(data))) return markdown except Exception as e: log("Error writing summary: %s" % e) @@ -502,9 +515,14 @@ def __to_jsonable(self): toRet['failureReasons'] = self.failureReasons toRet['resources'] = getattr(self.config, 'resources', None) toRet['protocol'] = self.__protocol() - toRet['environment'] = self.__environment() + env = self.__environment() + if env is not None and self.quality_environment: + env = dict(env, quality=self.quality_environment) + toRet['environment'] = env toRet['generators'] = self.__generators() toRet['imageBuild'] = self.__image_build() + if getattr(self, 'quality', None): + toRet['quality'] = self.quality # --parse: what the stored run recorded wins over this process's config toRet.update(getattr(self, '_stored_run', None) or {}) diff --git a/toolset/utils/summary.py b/toolset/utils/summary.py index 140fa6d..f83fd42 100644 --- a/toolset/utils/summary.py +++ b/toolset/utils/summary.py @@ -339,9 +339,116 @@ def to_markdown(rows, meta): lines += _ranking_section(rows, ranking.not_ranked_reason(meta)) lines += _caveats(rows, meta) lines += _failed_section(rows) + if meta.get('quality'): + lines += [''] + quality_markdown(quality_rows(meta)) return '\n'.join(lines) + '\n' +QUALITY_COLUMNS = ['framework', 'status', 'performance', 'accessibility', 'bestPractices', 'seo', + 'lcpMs', 'postJsBytes', 'jsClass', 'gzipBytes', 'htmlErrors', + 'a11yViolations', 'brokenLinks', 'seoPresent', 'seoTotal'] +QUALITY_NOTE = ('The SF 005 templates carry no meta description and no viewport, so every ' + 'generator loses the same Lighthouse SEO points; that is not a difference ' + 'between generators.') + + +def _get(data, *path): + for key in path: + if not isinstance(data, dict): + return None + data = data.get(key) + return data + + +def _sum_pages(q, section, key): + pages = _get(q, section, 'pages') or {} + values = [p.get(key) for p in pages.values() if isinstance(p, dict) and p.get(key) is not None] + return sum(values) if values else None + + +def _flatten(data, prefix=''): + for key, value in (data or {}).items(): + name = prefix + str(key) + if isinstance(value, dict): + yield from _flatten(value, name + '.') + elif isinstance(value, (int, float)) and not isinstance(value, bool): + yield name, value + + +def quality_rows(results): + '''One row per generator with a quality pass (spec 009 R-18), sorted by name.''' + rows = [] + for name in sorted(results.get('quality') or {}): + q = results['quality'][name] + post = _get(q, 'lighthouse', 'mobile', 'post') or {} + rows.append({ + 'framework': name, + 'status': q.get('status'), + 'error': q.get('error'), + 'performance': _get(post, 'scores', 'performance', 'median'), + 'accessibility': _get(post, 'scores', 'accessibility', 'median'), + 'bestPractices': _get(post, 'scores', 'best-practices', 'median'), + 'seo': _get(post, 'scores', 'seo', 'median'), + 'lcpMs': _get(post, 'metrics', 'lcp', 'median'), + 'postJsBytes': _get(q, 'js', 'pages', 'post'), + 'jsClass': _get(q, 'js', 'jsClass'), + 'gzipBytes': _get(q, 'weight', 'gzipBytes'), + 'htmlErrors': _sum_pages(q, 'html', 'errors'), + 'a11yViolations': _sum_pages(q, 'a11y', 'total'), + 'brokenLinks': _get(q, 'links', 'broken'), + 'seoPresent': _get(q, 'seo', 'present', 'post'), + 'seoTotal': _get(q, 'seo', 'total'), + 'numeric': dict(_flatten({k: v for k, v in q.items() if k != 'tools'})), + }) + return rows + + +def quality_csv(rows): + extra = sorted({k for r in rows for k in r['numeric']}) + out = io.StringIO() + writer = csv.writer(out, lineterminator='\n') + writer.writerow(QUALITY_COLUMNS + extra) + for r in rows: + writer.writerow([_blank(r[c]) for c in QUALITY_COLUMNS] + + [_blank(r['numeric'].get(k)) for k in extra]) + return out.getvalue() + + +def _score(value): + return '—' if value is None else '%d' % round(value * 100) + + +def _kb(value): + return '—' if value is None else '%.1f' % (value / 1024) + + +def _int(value): + return '—' if value is None else '%d' % round(value) + + +def quality_markdown(rows): + lines = ['## Qualidade', '', + 'Untimed pass over the output of the last build (spec 009). Scores and LCP: ' + 'Lighthouse mobile preset on the post page, median of 3 runs. No combined score ' + 'and no ranking.', '', + '| Framework | Performance | Accessibility | Best practices | SEO | LCP (ms) ' + '| Post JS (KB) | JS | Gzip (KB) | HTML errors | axe violations | Broken links ' + '| SEO signals |', + '|' + '---|' * 13] + for r in rows: + signals = '—' if r['seoPresent'] is None else '%s/%s' % (r['seoPresent'], r['seoTotal']) + cells = [r['framework'], _score(r['performance']), _score(r['accessibility']), + _score(r['bestPractices']), _score(r['seo']), _int(r['lcpMs']), + _kb(r['postJsBytes']), r['jsClass'] or '—', _kb(r['gzipBytes']), + _int(r['htmlErrors']), _int(r['a11yViolations']), _int(r['brokenLinks']), signals] + lines.append('| ' + ' | '.join(str(c) for c in cells) + ' |') + lines += ['', '- ' + QUALITY_NOTE] + for r in rows: + if r['status'] == 'error': + lines.append('- %s: %s' % (r['framework'], r['error'])) + return lines + + def _read_json(path): with open(path) as f: return json.load(f)