From 7afe4b0780763ae80d1c50860bac997261cfd68f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matheus=20Bregu=C3=AAz?= Date: Sat, 10 Oct 2026 14:17:24 -0300 Subject: [PATCH 01/15] docs(specs): 009 output quality spec (Lighthouse, SEO signals, HTML, a11y, links, weight) Untimed quality pass over the output of one build per generator, as proposed in #22. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/specs/009-output-quality/spec.md | 102 ++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 docs/specs/009-output-quality/spec.md diff --git a/docs/specs/009-output-quality/spec.md b/docs/specs/009-output-quality/spec.md new file mode 100644 index 0000000..d22694b --- /dev/null +++ b/docs/specs/009-output-quality/spec.md @@ -0,0 +1,102 @@ +# 009 Output Quality + +- **Date:** 2026-10-10 +- **Status:** proposed, awaiting review +- **Repo:** `ssgberk/benchmark-tool` (BT). No change to `ssgberk/ssg-frameworks` (SF). +- **Issue:** #22 +- **Related:** BT `008-benchmark-methodology` (suites, `results.json`, environment fingerprint), BT `006-results-summary` (`summary.md`), SF `005-reference-site-design` (what is built, R-14 page set), SF `006-layout-conformance` (post page resolved from the index). + +## Context + +The toolset measures how long a cold build takes and what it costs: wall time, CPU, peak RSS, input and output bytes, image build time (BT 008). SF 001 and SF 006 only prove that the output is the reference site: same pages, same selectors, same counts. SF 006 leaves out visual comparison, CSS rendering and accessibility audits. + +Every generator renders the same reference site, with the same content, templates and stylesheet, so the output is directly comparable. The differences come from the generator itself: hydration and client-side JS, extra requests, minification, extra pages, markup validity and accessibility, and the SEO signals it injects. None of this is measured today. + +This spec adds an **untimed quality pass** over the output of one build per generator. It adds no build metric and does not change how builds are timed. + +## Requirements + +### Running the pass + +- **R-1** `--quality` enables the pass. Without `--suite`, it runs in the cell given by `-nf`/`-cs`. With `--suite`, it runs only in the cell `numberOfFiles=50, contentSize=5` (suite P, cell 0), and in every other cell it only logs that the pass was skipped. +- **R-2** The pass runs after the timed runs of a generator succeed (status `ok`). It never runs for `failed`, `timeout`, `oom`, `nonconformant` or `unsupported` results, and it never changes a timing, a status or a ranking. +- **R-3** Before the build container is removed, `DockerHelper` copies `/` out of it with `container.get_archive()` into `results//quality//site/`. `WorkingDir` comes from the container config, and `output_folder` comes from `benchmark_config.json`. That folder holds the output of the last timed build. +- **R-4** The checks run in a BT-owned image built from `quality/Dockerfile`, with pinned versions of Node, Chrome for Testing, Lighthouse, axe-core, html-validate and lychee. Its container runs with `--network none`, with the site mounted read-only and an output directory mounted read-write. +- **R-5** Inside that container, one static server serves the site on `127.0.0.1` with gzip on and fixed headers. `quality/run.mjs` runs every check against it and writes `quality.json`. +- **R-6** The pinned tool versions (Lighthouse, Chrome, axe-core, html-validate, lychee, Node) and the quality image id are recorded in `results.json` `environment.quality`. + +### URLs + +- **R-7** Three URLs are audited: the index `/`, one post page and `/404.html`. The post page is the first `a[href]` inside `.post-item`, the same link SF 006 resolves. + +### Checks + +- **R-8 Lighthouse:** the mobile preset (simulated throttling) and the desktop preset, with 3 runs per URL and preset. Each run records: + - the scores for performance, accessibility, best-practices and SEO; + - FCP, LCP, TBT, CLS and Speed Index; + - total byte weight, JS bytes, CSS bytes, request count and DOM size; + - the ids of failed audits. + + `quality.json` stores, for each numeric value, the median and range of the 3 runs, and stores the failed audit ids of the median run. +- **R-9 SEO signals:** reported per page, not scored. + - `` (present, length), `html[lang]`, `meta[name=description]` (present, length), `meta[name=viewport]`, `link[rel=canonical]`. + - Open Graph `og:title`, `og:description`, `og:type` and `og:url`, and `twitter:card`. + - JSON-LD: blocks present and whether each one parses. + - The `meta[name=robots]` value and `link[rel=alternate][hreflang]` count. + - The `h1` count, whether the heading levels skip, `img` without `alt`, and `a` with no text or accessible name. + + Site-wide: `robots.txt` present, `sitemap.xml` present and valid (and its URL count), RSS or Atom feed present, and favicon present. For a page, `seo.present` counts the per-page signals found, out of a fixed total that `plan.md` lists. +- **R-10 HTML validity:** `html-validate` error and warning counts per page, plus the rule ids. +- **R-11 Accessibility:** axe-core violations per page, counted by impact (critical, serious, moderate, minor), plus the rule ids. +- **R-12 Links:** `lychee --offline` over the whole site. Count the broken internal links, anchors and asset references, and list up to 20 of them. +- **R-13 Weight:** + - total output bytes, gzip bytes and brotli bytes; + - bytes per post page; + - generator-added bytes, which is the total minus the SF 005 reference assets (`assets/ssgberk.css`, `assets/ssgberk.png`). Every generator ships those assets unchanged. +- **R-14 JavaScript:** + - JS bytes requested per audited page (from Lighthouse) and JS bytes on disk; + - `worksWithoutJs`, true when the post page's static HTML already contains the post title and the first body paragraph. + + `jsClass` is `none` (0 bytes of JS on every audited page) or `hydrated` (any JS). +- **R-15 Extra output:** files outside the SF 005 page set (index, post pages, `404.html`, reference assets), counted and sized and grouped by extension. + +### Results + +- **R-16** `quality.json` has one section per check (`lighthouse`, `seo`, `html`, `a11y`, `links`, `weight`, `js`, `extra`). Each section carries `status` (`ok` or `error`) and, on error, `error` with the message. A failing check is a result, not an error: it never fails the generator, the cell or the run. +- **R-17** `results.json` gets `quality.<generator>`: the parsed `quality.json`, or `{"status": "error", "error": ...}` when the pass could not run. +- **R-18** `summary.md` gets a "Qualidade" section with one row per generator. The columns are: + - the 4 mobile Lighthouse scores and mobile LCP; + - the post page's JS bytes, `jsClass` and gzip bytes; + - the HTML error, axe violation and broken link counts; + - SEO signals present, as `n/total`. + + Rows are sorted by name, with no combined score and no ranking. `quality-summary.csv` holds the same columns plus every numeric field. The rounds page reads that file; the page itself is not in this repository. +- **R-19** The section notes that the SF 005 templates carry no meta description and no viewport, so every generator loses the same SEO points. That constant isn't a difference between generators. + +## Goals and success criteria + +1. **One command audits the output.** *Measured by:* `./ssgberk --test hugo -nf 50 -cs 5 --quality` writes `results/<ts>/quality/hugo/quality.json` with all 8 sections and `status: ok`. A unit test asserts that `rawData`, `succeeded`, `failed` and the rankings in `summary.md` are identical with and without a `quality` section for the same parsed build output. +2. **The pass is isolated.** *Measured by:* a unit test asserts that the quality container runs with `network_mode="none"` and the site mount `ro`. Another asserts that a raised error in the pass leaves the build result unchanged and records `quality.<fw>.status = "error"`. +3. **Checks are pinned and recorded.** *Measured by:* `environment.quality` lists every tool version from R-6, and `quality/Dockerfile` names an exact version for each one. +4. **The matrix runs in CI.** *Measured by:* a BT CI job runs the pass on hugo for `ubuntu-24.04` and `ubuntu-24.04-arm`, and checks that every section is present and every score is between 0 and 1. A `workflow_dispatch` input runs it on every generator, one job each, and the job succeeds even when a check reports failures. + +## Testing + +- **pytest (no Docker):** + - SEO signal extraction from the SF 005 reference page and from variants (with description, with OG tags, with a broken JSON-LD block, a skipped heading level, `img` without `alt`); + - the Lighthouse median and range reduction; + - the axe and html-validate count reduction; + - extra output classification; + - weight arithmetic; + - the `quality.json` to `results.json` and `summary.md` merge; + - the R-1 skip rules. +- **Inside the quality image:** `quality/run.mjs` runs against a 3-page fixture site (index, one post, 404) and its `quality.json` matches a checked-in expected shape (keys and types, not Lighthouse values). + +## Out of scope + +- New build metrics: install time, image size breakdown, incremental builds, startup cost. +- Determinism (building twice and comparing the trees), because it needs a second build. +- A combined quality score or ranking. +- Changes to the SF 005 reference site, the SF 006 conformance rules or any generator. +- Field data, RUM and real-user Core Web Vitals. +- Tuning generators to score better: the pass audits the reference configuration as built. From c3d585d80bde04f9501e15ade0e596b5736b391d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matheus=20Bregu=C3=AAz?= <matbrgz@gmail.com> Date: Sat, 10 Oct 2026 14:34:32 -0300 Subject: [PATCH 02/15] docs(specs): 009 output quality plan and tasks; collector runs without mounts on Playwright's Chromium The toolset runs in a container, so the site goes in with put_archive instead of a bind mount; Chrome for Testing has no Linux arm64 build, so Chromium comes from the pinned Playwright. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> --- docs/specs/009-output-quality/plan.md | 193 ++ docs/specs/009-output-quality/spec.md | 6 +- docs/specs/009-output-quality/tasks.md | 2647 ++++++++++++++++++++++++ 3 files changed, 2843 insertions(+), 3 deletions(-) create mode 100644 docs/specs/009-output-quality/plan.md create mode 100644 docs/specs/009-output-quality/tasks.md diff --git a/docs/specs/009-output-quality/plan.md b/docs/specs/009-output-quality/plan.md new file mode 100644 index 0000000..3ac33bc --- /dev/null +++ b/docs/specs/009-output-quality/plan.md @@ -0,0 +1,193 @@ +# 009 Output Quality: Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. The steps are in `tasks.md` and use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** an untimed quality pass that audits the output of one build per generator (Lighthouse, SEO signals, HTML validity, accessibility, links, weight, JS, extra output) and writes `quality.json`, `results.json` `quality`, a "Qualidade" section in `summary.md` and `quality-summary.csv`. + +**Architecture:** +1. **Export.** `DockerHelper.benchmark` copies the build output out of the generator container before removing it. +2. **Collection.** A pinned BT image (`quality/`) runs a Node collector (`quality/run.mjs`) with no network. The collector serves the site, runs Lighthouse, axe-core, html-validate and lychee, and measures file sizes. It writes raw data only (`raw.json`). +3. **Reduction.** Pure Python in `toolset/quality/` reduces `raw.json` into `quality.json` and computes the SEO signals, the pages, the JS check and the extra output from the exported files. Most of the logic is therefore testable with pytest and no Docker. + +**Tech stack:** +- **Toolset:** Python 3.12 stdlib (`html.parser`, `tarfile`, `xml.etree`, `statistics`, `fnmatch`) and docker-py 7.1.0, as in the existing toolset. +- **Collector:** Node 24 with the `lighthouse`, `chrome-launcher`, `playwright` (Chromium only), `axe-core` and `html-validate` npm packages, plus the `lychee` binary. + +**Spec:** `docs/specs/009-output-quality/spec.md`. Read it before any task; requirement ids (R-n) below refer to it. + +## Global Constraints + +- The pass never changes a timing, a status, `succeeded`/`failed` or a ranking (R-2, goal 1). +- The quality container runs with `network_mode="none"` and has no volumes or mounts. The site goes in with `put_archive`; `raw.json` comes out with `get_archive` (R-4, goal 2). +- Every tool version is pinned exactly: + - npm: exact versions in `quality/package.json` and a committed `quality/package-lock.json`; + - Node, lychee: `ARG` with a full version in `quality/Dockerfile`; + - Chromium: whatever the pinned `playwright` version installs (R-4, R-6). +- With `--suite`, the pass runs only in the cell `numberOfFiles=50, contentSize=5` (R-1). +- A failing check is a section with `status: "error"` and `error: "<message>"`, never an exception out of the pass (R-16). +- Three audited URLs: `/`, the first `a[href]` inside `.post-item` on the index, and `/404.html` (R-7). +- Lighthouse: mobile and desktop presets, 3 runs per URL and preset, median and range per value, failed audit ids of the median run (R-8). +- No combined score and no ranking in any output (R-18). +- Toolset code is stdlib plus `docker`, and passes `ruff check toolset tests`. Tests run with `pytest -q` and need no Docker. +- Commits: author and committer `Matheus Breguêz <matbrgz@gmail.com>`, GPG-signed (key `B6FA8458D5176E83`), trailer `Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>`. Never use `--no-gpg-sign` or `--author`. + +## Review Focus + +1. **The post link is relative, has a query or fragment, or ends without `/`.** Examples: `post/a/`, `./post/a/index.html`, `/post/a?x=1#t`, `/post/a`. `resolve_pages` must still find the file on disk. Pinned in Task 1 (`test_resolve_pages_variants`). +2. **A generator writes non-UTF-8 bytes or a malformed HTML page.** SEO extraction and `works_without_js` must not raise; they read with `errors="replace"` and `html.parser` tolerates bad markup. Pinned in Task 2 (`test_page_signals_malformed_html`). +3. **The collector fails for one page or one check.** For example, Lighthouse times out on the 404 page, or lychee is missing. The other pages and sections must still be reported, and only the affected entry is an `error`. Pinned in Task 3 (`test_build_quality_partial_errors`). +4. **The exported archive is missing,** because the export failed or the noise re-run timed out. The pass records `quality.<fw>.status = "error"` and the build result is unchanged. Pinned in Task 6 (`test_quality_pass_missing_archive_keeps_result`). +5. **`--parse` on a run that had a quality pass.** `quality` must survive the re-parse, because raw.txt cannot rebuild it. Pinned in Task 6 (`test_reparse_keeps_quality`). + +## File structure + +| File | Responsibility | Task | +|---|---|---| +| `toolset/quality/__init__.py` | package marker | 1 | +| `toolset/quality/site.py` | generator config (`output_folder`, `output_glob`), archive extraction, page resolution (R-7), URL to file, extra output (R-15), `works_without_js` (R-14) | 1 | +| `toolset/quality/seo.py` | per-page and site-wide SEO signals (R-9) | 2 | +| `toolset/quality/reduce.py` | `raw.json` to `quality.json`: Lighthouse, axe, html-validate, lychee, weight, JS reductions and section assembly (R-8, R-10..R-14, R-16) | 3 | +| `quality/Dockerfile`, `quality/package.json`, `quality/package-lock.json`, `quality/run.mjs` | the pinned collector image (R-4, R-5) | 4 | +| `quality/test/fixture/**`, `quality/test/pages.json`, `quality/test/expected-shape.json`, `quality/test/check-shape.mjs` | collector test inside the image | 4 | +| `toolset/utils/docker_helper.py` | output export in `benchmark`, `build_quality_image`, `run_quality` | 5 | +| `toolset/quality/runner.py` | `eligible` (R-1), `archive_path`, `run_pass` (R-3, R-16) | 6 | +| `toolset/run-tests.py`, `toolset/utils/benchmark_config.py` | `--quality` flag | 6 | +| `toolset/benchmark/benchmarker.py` | export arguments and the pass hook (R-2) | 6 | +| `toolset/utils/results.py` | `add_quality`, `quality` and `environment.quality` in `results.json`, kept by `--parse` (R-6, R-17) | 6 | +| `toolset/utils/summary.py` | `quality_rows`, `quality_csv`, `quality_markdown`; the section in `to_markdown` (R-18, R-19) | 7 | +| `.github/workflows/ci.yml`, `.github/workflows/benchmark-round.yml`, `README.md` | CI jobs, round input, docs (goal 4) | 8 | +| `tests/test_quality_site.py`, `tests/test_quality_seo.py`, `tests/test_quality_reduce.py`, `tests/test_quality_docker.py`, `tests/test_quality_runner.py`, `tests/test_quality_summary.py`, `tests/fixtures/quality/**` | tests | 1–7 | + +## Contracts + +### Collector CLI (`quality/run.mjs`) + +``` +node /quality/run.mjs --site <dir> --pages '<json>' --out <dir> +``` + +`--pages` is the object returned by `site.resolve_pages`, for example: + +```json +{"index": {"url": "/", "file": "index.html"}, + "post": {"url": "/post/hello/", "file": "post/hello/index.html"}, + "404": {"url": "/404.html", "file": "404.html"}} +``` + +The collector writes `<out>/raw.json` and exits 0, even when checks fail. It exits non-zero only if it cannot write `raw.json`. + +### `raw.json` (collector output, input of `reduce.build_quality`) + +Any top-level section may instead be `{"error": "<message>"}`, and any page entry may be `{"error": "<message>"}`. + +```json +{ + "tools": {"node": "v24.x", "lighthouse": "x.y.z", "chromium": "x.y", "axeCore": "x.y.z", + "htmlValidate": "x.y.z", "lychee": "x.y.z", "playwright": "x.y.z"}, + "pages": {"index": {"url": "/", "file": "index.html"}, "post": {...}, "404": {...}}, + "lighthouse": { + "mobile": {"index": [RUN, RUN, RUN], "post": [...], "404": [...]}, + "desktop": {"index": [...], "post": [...], "404": [...]} + }, + "axe": {"index": {"violations": [{"id": "color-contrast", "impact": "serious", "nodes": 3}]}, "post": {...}, "404": {...}}, + "rendered": {"post": {"title": "Post title", "firstParagraph": "First paragraph text"}}, + "html": {"index": {"messages": [{"ruleId": "void-style", "severity": 2}]}, "post": {...}, "404": {...}}, + "links": {"report": <lychee --format json output>}, + "files": [{"path": "index.html", "bytes": 1234, "gzip": 600, "br": 500}] +} +``` + +`RUN` is either `{"error": "<message>"}` or: + +```json +{"scores": {"performance": 0.98, "accessibility": 1, "best-practices": 1, "seo": 0.82}, + "metrics": {"fcp": 812.3, "lcp": 812.3, "tbt": 0, "cls": 0, "si": 812.3}, + "resources": {"totalBytes": 5321, "jsBytes": 0, "cssBytes": 1210, "requests": 3, "domSize": 54}, + "failedAudits": ["meta-description"]} +``` + +Metric units are Lighthouse's `numericValue`: milliseconds for FCP, LCP, TBT and SI, unitless for CLS. + +### `quality.json` (R-16) + +```json +{ + "status": "ok", + "pages": {"index": "/", "post": "/post/hello/", "404": "/404.html"}, + "tools": {...copied from raw.json...}, + "lighthouse": {"status": "ok", + "mobile": {"post": {"status": "ok", "runs": 3, + "scores": {"performance": {"median": 0.98, "min": 0.97, "max": 0.99}, ...}, + "metrics": {"lcp": {"median": 812.3, "min": ..., "max": ...}, ...}, + "resources": {"jsBytes": {"median": 0, ...}, ...}, + "failedAudits": ["meta-description"]}, "index": {...}, "404": {...}}, + "desktop": {...}}, + "seo": {"status": "ok", "total": 12, + "pages": {"index": {...signals...}, "post": {...}, "404": {...}}, + "present": {"index": 2, "post": 2, "404": 2}, + "site": {"robotsTxt": false, "sitemap": {"present": false, "valid": null, "urls": null}, + "feed": false, "favicon": false}}, + "html": {"status": "ok", "pages": {"index": {"errors": 0, "warnings": 1, "rules": ["..."]}, ...}}, + "a11y": {"status": "ok", "pages": {"index": {"violations": {"critical": 0, "serious": 1, "moderate": 0, "minor": 0}, + "total": 1, "rules": ["color-contrast"]}, ...}}, + "links": {"status": "ok", "broken": 0, "byKind": {"link": 0, "anchor": 0, "asset": 0}, "items": []}, + "weight": {"status": "ok", "bytes": 0, "gzipBytes": 0, "brotliBytes": 0, "bytesPerPost": 0, + "generatorAddedBytes": 0}, + "js": {"status": "ok", "pages": {"index": 0, "post": 0, "404": 0}, "diskBytes": 0, + "jsClass": "none", "worksWithoutJs": true}, + "extra": {"status": "ok", "files": 0, "bytes": 0, "byExtension": {}} +} +``` + +When the whole pass fails, `quality.json` is `{"status": "error", "error": "<Type>: <message>"}`. + +### Python interfaces + +- **`toolset/quality/site.py`:** + - `REFERENCE_ASSETS = ('assets/ssgberk.css', 'assets/ssgberk.png')` + - `generator_config(test_directory) -> (output_folder: str, output_glob: str)` + - `extract(tar_path, dest) -> str`: extracts the archive and strips its top directory; returns `dest`. + - `url_to_file(site_dir, url) -> str | None` + - `resolve_pages(site_dir) -> dict` (shape above); raises `ValueError`. + - `list_files(site_dir) -> list[str]`: sorted POSIX relative paths. + - `extra_output(site_dir, output_glob) -> {"files", "bytes", "byExtension"}` + - `works_without_js(site_dir, post_file, rendered) -> bool | None` +- **`toolset/quality/seo.py`:** + - `PAGE_SIGNALS` (tuple of 12 keys) + - `page_signals(html: str) -> dict` + - `site_signals(site_dir, page_signals_list) -> dict` + - `seo_section(site_dir, pages) -> {"total", "pages", "present", "site"}` +- **`toolset/quality/reduce.py`:** + - `SECTIONS`, `spread(values)` + - `reduce_lighthouse(raw_lh)`, `reduce_axe(raw_axe)`, `reduce_html(raw_html)`, `reduce_links(raw_links)` + - `weight(files, output_glob)`, `js_section(lighthouse, files, works)` + - `build_quality(raw, site_dir, pages, output_glob) -> dict` +- **`toolset/quality/runner.py`:** + - `QUALITY_CELL = (50, 5.0)` + - `eligible(config) -> bool` + - `skipped(config) -> bool`: `--quality` was given but this cell is not eligible. + - `archive_path(results_dir, name) -> str` + - `run_pass(docker_helper, test, results_dir) -> dict`, which never raises. +- **`toolset/utils/docker_helper.py`:** + - `QUALITY_IMAGE = 'ssgberk/quality'`, `QUALITY_TIMEOUT_SECONDS = 1800` + - `DockerHelper.benchmark(..., export=None)`, where `export` is `(output_folder, dest_tar_path)` + - `DockerHelper.build_quality_image() -> str` (image id) + - `DockerHelper.run_quality(tar_path, site_name, pages, timeout_seconds=QUALITY_TIMEOUT_SECONDS) -> dict` (raw.json) +- **`toolset/utils/results.py`:** `Results.add_quality(name, quality, image_id=None)` +- **`toolset/utils/summary.py`:** + - `QUALITY_COLUMNS` + - `quality_rows(results) -> list[dict]` + - `quality_csv(rows) -> str` + - `quality_markdown(rows) -> list[str]` + +## Decisions + +1. **The collector writes raw data, and Python reduces it.** Medians, counts, the SEO signals and the page set live in Python so they are covered by pytest without Docker or Chrome (spec "Testing"). The collector only does what needs a browser or a binary. +2. **One Chromium, two drivers.** Lighthouse drives Playwright's Chromium through `chrome-launcher` (`chromePath = chromium.executablePath()`). axe-core and the rendered-text check use Playwright on the same binary. Chrome runs with `--headless=new --no-sandbox --disable-gpu --disable-dev-shm-usage`, because the container runs as root and `/dev/shm` is small. +3. **The static server is a few lines of Node `http`.** It has gzip, fixed headers and directory index, and it serves `404.html` with status 404, so no server package version can change the results. +4. **The summary uses the post page.** The 4 scores and LCP in the `summary.md` row come from the mobile preset on the post page, the page type that scales with the site. `quality.json` keeps every page and both presets. +5. **The image builds outside the time logger.** `build_quality_image` streams its own `APIClient.build`, so the generator's build-time log lines and `imageBuild` are not touched. It builds once per process, the first time a pass runs. +6. **No new dependency in the toolset image.** brotli is computed by the collector (Node `zlib`), not by Python. +7. **The export is skipped when the build did not end `ok` with exit code 0.** For a noisy re-run, the export of the re-run overwrites the first one, so the audited output is always the last build. +8. **The sitemap is parsed with `xml.etree.ElementTree`, not `defusedxml`.** The toolset stays stdlib plus `docker`. `sitemap.xml` comes from the generator under test, not from the network. ElementTree never fetches external entities, and the expat bundled with Python 3.12 limits entity expansion (billion laughs). A parse error is recorded as `valid: false`. +9. **What happens to the exported files.** The extracted site stays in `results/<ts>/quality/<fw>/site/` for inspection. `site.tar` is deleted after the pass. diff --git a/docs/specs/009-output-quality/spec.md b/docs/specs/009-output-quality/spec.md index d22694b..3a2a5aa 100644 --- a/docs/specs/009-output-quality/spec.md +++ b/docs/specs/009-output-quality/spec.md @@ -21,9 +21,9 @@ This spec adds an **untimed quality pass** over the output of one build per gene - **R-1** `--quality` enables the pass. Without `--suite`, it runs in the cell given by `-nf`/`-cs`. With `--suite`, it runs only in the cell `numberOfFiles=50, contentSize=5` (suite P, cell 0), and in every other cell it only logs that the pass was skipped. - **R-2** The pass runs after the timed runs of a generator succeed (status `ok`). It never runs for `failed`, `timeout`, `oom`, `nonconformant` or `unsupported` results, and it never changes a timing, a status or a ranking. - **R-3** Before the build container is removed, `DockerHelper` copies `<WorkingDir>/<output_folder>` out of it with `container.get_archive()` into `results/<ts>/quality/<generator>/site/`. `WorkingDir` comes from the container config, and `output_folder` comes from `benchmark_config.json`. That folder holds the output of the last timed build. -- **R-4** The checks run in a BT-owned image built from `quality/Dockerfile`, with pinned versions of Node, Chrome for Testing, Lighthouse, axe-core, html-validate and lychee. Its container runs with `--network none`, with the site mounted read-only and an output directory mounted read-write. +- **R-4** The checks run in a BT-owned image built from `quality/Dockerfile`, with pinned versions of Node, Playwright's Chromium build, Lighthouse, axe-core, html-validate and lychee. Chrome for Testing has no Linux arm64 build, but Playwright ships Chromium for both architectures. Its container runs with `network_mode="none"`. The site goes in with `put_archive` and `raw.json` comes out with `get_archive`. Nothing is bind-mounted, because the toolset itself runs in a container and host paths differ. - **R-5** Inside that container, one static server serves the site on `127.0.0.1` with gzip on and fixed headers. `quality/run.mjs` runs every check against it and writes `quality.json`. -- **R-6** The pinned tool versions (Lighthouse, Chrome, axe-core, html-validate, lychee, Node) and the quality image id are recorded in `results.json` `environment.quality`. +- **R-6** The pinned tool versions (Lighthouse, Chromium, axe-core, html-validate, lychee, Node) and the quality image id are recorded in `results.json` `environment.quality`. ### URLs @@ -76,7 +76,7 @@ This spec adds an **untimed quality pass** over the output of one build per gene ## Goals and success criteria 1. **One command audits the output.** *Measured by:* `./ssgberk --test hugo -nf 50 -cs 5 --quality` writes `results/<ts>/quality/hugo/quality.json` with all 8 sections and `status: ok`. A unit test asserts that `rawData`, `succeeded`, `failed` and the rankings in `summary.md` are identical with and without a `quality` section for the same parsed build output. -2. **The pass is isolated.** *Measured by:* a unit test asserts that the quality container runs with `network_mode="none"` and the site mount `ro`. Another asserts that a raised error in the pass leaves the build result unchanged and records `quality.<fw>.status = "error"`. +2. **The pass is isolated.** *Measured by:* a unit test asserts that the quality container runs with `network_mode="none"` and no volumes or mounts. Another asserts that a raised error in the pass leaves the build result unchanged and records `quality.<fw>.status = "error"`. 3. **Checks are pinned and recorded.** *Measured by:* `environment.quality` lists every tool version from R-6, and `quality/Dockerfile` names an exact version for each one. 4. **The matrix runs in CI.** *Measured by:* a BT CI job runs the pass on hugo for `ubuntu-24.04` and `ubuntu-24.04-arm`, and checks that every section is present and every score is between 0 and 1. A `workflow_dispatch` input runs it on every generator, one job each, and the job succeeds even when a check reports failures. diff --git a/docs/specs/009-output-quality/tasks.md b/docs/specs/009-output-quality/tasks.md new file mode 100644 index 0000000..ef3557e --- /dev/null +++ b/docs/specs/009-output-quality/tasks.md @@ -0,0 +1,2647 @@ +# 009 Output Quality: Tasks + +Read `spec.md` and `plan.md` (Global Constraints, Contracts, Decisions) first. Every task's requirements include the Global Constraints. Run commands from the repository root. Commit with `git commit -S` as configured; never pass `--no-gpg-sign` or `--author`. End every commit message with: + +``` +Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> +``` + +Test command (no Docker): `python3 -m pytest -q` and `ruff check toolset tests`. + +--- + +### Task 1: Site helpers (`toolset/quality/site.py`) + +**Files:** +- Create: `toolset/quality/__init__.py` (empty), `toolset/quality/site.py` +- Create: `tests/fixtures/quality/site/index.html`, `tests/fixtures/quality/site/post/hello/index.html`, `tests/fixtures/quality/site/404.html`, `tests/fixtures/quality/site/assets/ssgberk.css`, `tests/fixtures/quality/site/assets/ssgberk.png`, `tests/fixtures/quality/site/tags/a/index.html`, `tests/fixtures/quality/site/feed.xml` +- Test: `tests/test_quality_site.py` + +**Interfaces:** +- Consumes: nothing. +- Produces: + - `REFERENCE_ASSETS` + - `generator_config(test_directory) -> (str, str)` + - `extract(tar_path, dest) -> str` + - `url_to_file(site_dir, url) -> str | None` + - `resolve_pages(site_dir) -> dict` + - `list_files(site_dir) -> list[str]` + - `extra_output(site_dir, output_glob) -> dict` (no `status` key) + - `works_without_js(site_dir, post_file, rendered) -> bool | None` + - `text_of(html) -> str`, which collapses the visible text of an HTML string. + +- [ ] **Step 1: Write the fixture site** + +`tests/fixtures/quality/site/index.html`: + +```html +<!doctype html> +<html lang="en"> +<head><meta charset="utf-8"><title>SSGBerk Reference + + +
SSGBerk Reference +
+
+

Built for the SSGBerk build-time benchmark.

+ + +``` + +`tests/fixtures/quality/site/post/hello/index.html`: + +```html + + +Hello + + +
SSGBerk Reference +
+
+

Hello

+

SSGBerk

+
  • a
  • b
  • c
+

First paragraph of the post.

Second paragraph.

+
+

Built for the SSGBerk build-time benchmark.

+ + +``` + +`tests/fixtures/quality/site/404.html`: + +```html + + +Not found + + +
SSGBerk Reference +
+

Page not found

+

Built for the SSGBerk build-time benchmark.

+ + +``` + +`tests/fixtures/quality/site/assets/ssgberk.css`: + +```css +body { font-family: sans-serif; margin: 0 auto; max-width: 40rem; } +``` + +`tests/fixtures/quality/site/tags/a/index.html` (extra output): + +```html + +Tag a

a

+``` + +`tests/fixtures/quality/site/feed.xml` (extra output): + +```xml + +SSGBerk Reference +``` + +Write the 1×1 PNG with Python: + +```bash +python3 - <<'EOF' +import struct, zlib +def chunk(t, d): + return struct.pack('>I', len(d)) + t + d + struct.pack('>I', zlib.crc32(t + d) & 0xffffffff) +png = (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 1, 1, 8, 2, 0, 0, 0)) + + chunk(b'IDAT', zlib.compress(b'\x00\xff\xff\xff')) + chunk(b'IEND', b'')) +open('tests/fixtures/quality/site/assets/ssgberk.png', 'wb').write(png) +EOF +``` + +- [ ] **Step 2: Write the failing tests** + +`tests/test_quality_site.py`: + +```python +import io +import json +import pathlib +import tarfile + +import pytest + +from toolset.quality import site + +SITE = pathlib.Path(__file__).parent / "fixtures" / "quality" / "site" + + +def test_generator_config(tmp_path): + (tmp_path / "benchmark_config.json").write_text(json.dumps( + {"framework": "hugo", "config": [{"output_folder": "public", "output_glob": "post/*/index.html"}]})) + assert site.generator_config(str(tmp_path)) == ("public", "post/*/index.html") + + +def _tar(entries): + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w") as tar: + for name, data in entries: + info = tarfile.TarInfo(name) + if data is None: + info.type = tarfile.DIRTYPE + tar.addfile(info) + else: + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + return buf.getvalue() + + +def test_extract_strips_top_dir(tmp_path): + tar_path = tmp_path / "site.tar" + tar_path.write_bytes(_tar([("public", None), ("public/index.html", b"

i

"), + ("public/post/a/index.html", b"

a

")])) + dest = tmp_path / "site" + assert site.extract(str(tar_path), str(dest)) == str(dest) + assert (dest / "index.html").read_bytes() == b"

i

" + assert (dest / "post" / "a" / "index.html").exists() + + +def test_extract_rejects_path_traversal(tmp_path): + tar_path = tmp_path / "site.tar" + tar_path.write_bytes(_tar([("public/../../evil.html", b"x")])) + with pytest.raises(tarfile.TarError): + site.extract(str(tar_path), str(tmp_path / "site")) + assert not (tmp_path.parent / "evil.html").exists() + + +def test_resolve_pages_fixture(): + assert site.resolve_pages(str(SITE)) == { + "index": {"url": "/", "file": "index.html"}, + "post": {"url": "/post/hello/", "file": "post/hello/index.html"}, + "404": {"url": "/404.html", "file": "404.html"}, + } + + +@pytest.mark.parametrize("href,url,file", [ + ("/post/a/", "/post/a/", "post/a/index.html"), + ("post/a/", "/post/a/", "post/a/index.html"), + ("./post/a/index.html", "/post/a/index.html", "post/a/index.html"), + ("/post/a?x=1#t", "/post/a", "post/a/index.html"), + ("/post/a", "/post/a", "post/a/index.html"), + ("/post/b", "/post/b", "post/b.html"), +]) +def test_resolve_pages_variants(tmp_path, href, url, file): + (tmp_path / "post" / "a").mkdir(parents=True) + (tmp_path / "post" / "a" / "index.html").write_text("a") + (tmp_path / "post" / "b.html").write_text("b") + (tmp_path / "index.html").write_text( + '' % href) + pages = site.resolve_pages(str(tmp_path)) + assert pages["post"] == {"url": url, "file": file} + + +def test_resolve_pages_without_post_link(tmp_path): + (tmp_path / "index.html").write_text('x') + with pytest.raises(ValueError, match="post-item"): + site.resolve_pages(str(tmp_path)) + + +def test_resolve_pages_link_to_missing_file(tmp_path): + (tmp_path / "index.html").write_text('
  • g
    • ') + with pytest.raises(ValueError, match="no file"): + site.resolve_pages(str(tmp_path)) + + +def test_list_files_is_sorted_posix(): + files = site.list_files(str(SITE)) + assert files == sorted(files) and "post/hello/index.html" in files + + +def test_extra_output_fixture(): + extra = site.extra_output(str(SITE), "post/*/index.html") + assert extra["files"] == 2 + assert set(extra["byExtension"]) == {".html", ".xml"} + assert extra["byExtension"][".html"]["files"] == 1 + assert extra["bytes"] == sum(e["bytes"] for e in extra["byExtension"].values()) + + +def test_works_without_js(): + post = "post/hello/index.html" + rendered = {"title": "Hello", "firstParagraph": "First paragraph of the post."} + assert site.works_without_js(str(SITE), post, rendered) is True + assert site.works_without_js(str(SITE), post, {"title": "Hello", "firstParagraph": "Only after JS"}) is False + assert site.works_without_js(str(SITE), post, None) is None + assert site.works_without_js(str(SITE), post, {"error": "boom"}) is None + + +def test_text_of_collapses_whitespace(): + assert site.text_of("

    a\n b

    ") == "a b" +``` + +- [ ] **Step 3: Run them and see them fail** + +Run: `python3 -m pytest -q tests/test_quality_site.py` +Expected: FAIL with `ModuleNotFoundError: No module named 'toolset.quality'`. + +- [ ] **Step 4: Implement `toolset/quality/site.py`** + +```python +''' +Pages and files of a generated site (spec 009 R-3, R-7, R-14, R-15). +''' +import fnmatch +import json +import os +import posixpath +import re +import tarfile +from html.parser import HTMLParser +from urllib.parse import unquote, urlsplit + +# SF 005 reference assets: every generator ships them unchanged (R-13, R-15) +REFERENCE_ASSETS = ('assets/ssgberk.css', 'assets/ssgberk.png') + + +def generator_config(test_directory): + '''(output_folder, output_glob) of benchmark_config.json config[0].''' + with open(os.path.join(test_directory, 'benchmark_config.json')) as f: + config = json.load(f)['config'][0] + return config['output_folder'], config['output_glob'] + + +def extract(tar_path, dest): + ''' + Extracts the archive DockerHelper.benchmark exported (a single top + directory named after output_folder) into dest, without that directory. + ''' + os.makedirs(dest, exist_ok=True) + with tarfile.open(tar_path) as tar: + members = [] + for member in tar.getmembers(): + parts = member.name.split('/', 1) + if len(parts) < 2 or not parts[1]: + continue + member.name = parts[1] + members.append(member) + # filter='data' rejects absolute paths, '..' and links out of dest + tar.extractall(dest, members=members, filter='data') + return dest + + +def url_to_file(site_dir, url): + '''The file a static server would return for url, or None.''' + p = unquote(urlsplit(url).path).lstrip('/') + if p == '' or p.endswith('/'): + candidates = [posixpath.join(p, 'index.html')] + else: + candidates = [p, p + '/index.html', p + '.html'] + for c in candidates: + if os.path.isfile(os.path.join(site_dir, c)): + return posixpath.normpath(c) + return None + + +class _PostLink(HTMLParser): + '''First a[href] after the first element whose class list has post-item.''' + + def __init__(self): + super().__init__(convert_charrefs=True) + self.in_item = False + self.href = None + + def handle_starttag(self, tag, attrs): + a = {k: (v or '') for k, v in attrs} + if 'post-item' in a.get('class', '').split(): + self.in_item = True + if tag == 'a' and self.in_item and self.href is None and a.get('href'): + self.href = a['href'] + + +def _read(path): + with open(path, encoding='utf-8', errors='replace') as f: + return f.read() + + +def resolve_pages(site_dir): + '''The three audited pages (R-7): index, first post link, 404.''' + parser = _PostLink() + parser.feed(_read(os.path.join(site_dir, 'index.html'))) + if not parser.href: + raise ValueError('no a[href] inside .post-item on the index') + path = urlsplit(parser.href).path + url = posixpath.normpath(posixpath.join('/', path)) + if path.endswith('/') and url != '/': + url += '/' + post_file = url_to_file(site_dir, url) + if post_file is None: + raise ValueError('post link %s matches no file' % parser.href) + return { + 'index': {'url': '/', 'file': 'index.html'}, + 'post': {'url': url, 'file': post_file}, + '404': {'url': '/404.html', 'file': '404.html'}, + } + + +def list_files(site_dir): + out = [] + for root, _dirs, files in os.walk(site_dir): + for name in files: + rel = os.path.relpath(os.path.join(root, name), site_dir) + out.append(rel.replace(os.sep, '/')) + return sorted(out) + + +def extra_output(site_dir, output_glob): + '''R-15: files outside the SF 005 page set, by extension.''' + by_ext = {} + total_files = total_bytes = 0 + for f in list_files(site_dir): + if f in ('index.html', '404.html') or f in REFERENCE_ASSETS \ + or fnmatch.fnmatch(f, output_glob): + continue + size = os.path.getsize(os.path.join(site_dir, f)) + ext = os.path.splitext(f)[1].lower() or '(none)' + entry = by_ext.setdefault(ext, {'files': 0, 'bytes': 0}) + entry['files'] += 1 + entry['bytes'] += size + total_files += 1 + total_bytes += size + return {'files': total_files, 'bytes': total_bytes, + 'byExtension': dict(sorted(by_ext.items()))} + + +class _Text(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.parts = [] + self.skip = 0 + + def handle_starttag(self, tag, attrs): + if tag in ('script', 'style', 'template'): + self.skip += 1 + + def handle_endtag(self, tag): + if tag in ('script', 'style', 'template') and self.skip: + self.skip -= 1 + + def handle_data(self, data): + if not self.skip: + self.parts.append(data) + + +def text_of(html): + parser = _Text() + parser.feed(html) + parser.close() + return re.sub(r'\s+', ' ', ' '.join(parser.parts)).strip() + + +def works_without_js(site_dir, post_file, rendered): + ''' + R-14: True when the static post HTML already holds the title and first + body paragraph that the browser shows after scripts ran. + ''' + if not isinstance(rendered, dict) or 'error' in rendered: + return None + title = re.sub(r'\s+', ' ', rendered.get('title') or '').strip() + first = re.sub(r'\s+', ' ', rendered.get('firstParagraph') or '').strip() + if not title or not first: + return None + static = text_of(_read(os.path.join(site_dir, post_file))) + return title in static and first in static +``` + +- [ ] **Step 5: Run the tests** + +Run: `python3 -m pytest -q tests/test_quality_site.py && ruff check toolset tests` +Expected: all pass, no ruff findings. + +- [ ] **Step 6: Commit** + +```bash +git add toolset/quality/__init__.py toolset/quality/site.py tests/test_quality_site.py tests/fixtures/quality/site +git commit -m "feat(quality): site helpers for the output quality pass (spec 009) + +Co-Authored-By: Claude Opus 5.5 (1M context) " +``` + +--- + +### Task 2: SEO signals (`toolset/quality/seo.py`) + +**Files:** +- Create: `toolset/quality/seo.py` +- Test: `tests/test_quality_seo.py` + +**Interfaces:** +- Consumes: `site.list_files`, `site._read` (Task 1); the fixture site. +- Produces: + - `PAGE_SIGNALS` + - `page_signals(html) -> dict` + - `site_signals(site_dir, page_signals_list) -> dict` + - `seo_section(site_dir, pages) -> {"total", "pages", "present", "site"}` (no `status` key) + +- [ ] **Step 1: Write the failing tests** + +`tests/test_quality_seo.py`: + +```python +import pathlib + +from toolset.quality import seo, site + +SITE = pathlib.Path(__file__).parent / "fixtures" / "quality" / "site" + +RICH = """ +Post title + + + + + + + + + + +

    T

    S

    a""" + + +def test_reference_page_has_only_title_and_lang(): + s = seo.page_signals((SITE / "post" / "hello" / "index.html").read_text()) + assert s["title"] is True and s["titleLength"] == 5 and s["lang"] == "en" + assert s["description"] is False and s["viewport"] is False and s["canonical"] is False + assert s["jsonLd"] is False and s["jsonLdValid"] is None and s["robotsMeta"] is None + assert s["h1"] == 1 and s["headingSkip"] is False and s["imgWithoutAlt"] == 0 + assert s["present"] == 2 + + +def test_rich_page_has_every_signal(): + s = seo.page_signals(RICH) + assert s["present"] == len(seo.PAGE_SIGNALS) == 12 + assert s["descriptionLength"] == len("A post about things") + assert s["robotsMeta"] == "index,follow" and s["hreflang"] == 2 + assert s["jsonLdBlocks"] == 1 and s["jsonLdValid"] is True + assert s["faviconLink"] is True and s["imgWithoutAlt"] == 0 + + +def test_broken_json_ld(): + s = seo.page_signals('') + assert s["jsonLd"] is True and s["jsonLdValid"] is False + + +def test_heading_skip_images_and_empty_links(): + html = ('

    a

    b

    ' + '' + 'logo no href') + s = seo.page_signals(html) + assert s["headingSkip"] is True + assert s["imgWithoutAlt"] == 1 + assert s["emptyLinks"] == 2 + + +def test_page_signals_malformed_html(): + html = b"x

    unclosed \xff\xfe".decode("utf-8", "replace") + s = seo.page_signals(html) + assert s["title"] is True and s["lang"] == "en" + + +def test_site_signals(tmp_path): + (tmp_path / "robots.txt").write_text("User-agent: *\n") + (tmp_path / "sitemap.xml").write_text( + '' + '/a//b/') + (tmp_path / "feed.xml").write_text("") + (tmp_path / "favicon.ico").write_bytes(b"\x00") + s = seo.site_signals(str(tmp_path), []) + assert s == {"robotsTxt": True, "sitemap": {"present": True, "valid": True, "urls": 2}, + "feed": True, "favicon": True} + + +def test_site_signals_invalid_sitemap_and_link_favicon(tmp_path): + (tmp_path / "sitemap.xml").write_text("") + s = seo.site_signals(str(tmp_path), [{"faviconLink": True}]) + assert s["sitemap"] == {"present": True, "valid": False, "urls": None} + assert s["favicon"] is True and s["robotsTxt"] is False and s["feed"] is False + + +def test_seo_section_fixture(): + pages = site.resolve_pages(str(SITE)) + section = seo.seo_section(str(SITE), pages) + assert section["total"] == 12 + assert section["present"] == {"index": 2, "post": 2, "404": 2} + assert section["site"]["feed"] is True and section["site"]["sitemap"]["present"] is False + assert set(section["pages"]) == {"index", "post", "404"} +``` + +- [ ] **Step 2: Run them and see them fail** + +Run: `python3 -m pytest -q tests/test_quality_seo.py` +Expected: FAIL with `ImportError: cannot import name 'seo'`. + +- [ ] **Step 3: Implement `toolset/quality/seo.py`** + +```python +''' +SEO signals of a generated page and site (spec 009 R-9). Reported, not scored. +''' +import json +import os +import re +import xml.etree.ElementTree as ET +from html.parser import HTMLParser + +from toolset.quality import site + +# The per-page signals counted in seo.present (out of len(PAGE_SIGNALS)) +PAGE_SIGNALS = ('title', 'lang', 'description', 'viewport', 'canonical', 'ogTitle', + 'ogDescription', 'ogType', 'ogUrl', 'twitterCard', 'jsonLd', 'robotsMeta') +META = {('name', 'description'): 'description', ('name', 'viewport'): 'viewport', + ('property', 'og:title'): 'ogTitle', ('property', 'og:description'): 'ogDescription', + ('property', 'og:type'): 'ogType', ('property', 'og:url'): 'ogUrl', + ('name', 'twitter:card'): 'twitterCard', ('name', 'robots'): 'robotsMeta'} +FEEDS = ('feed.xml', 'atom.xml', 'rss.xml', 'index.xml', 'feed/index.xml', 'rss/index.xml') +FAVICONS = ('favicon.ico', 'favicon.svg', 'favicon.png') + + +class _Collector(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.lang = None + self.title = None + self.meta = {} + self.canonical = None + self.hreflang = 0 + self.favicon_link = False + self.jsonld = [] + self.headings = [] + self.img_without_alt = 0 + self.empty_links = 0 + self._capture = None # 'title' or 'jsonld' + self._buf = [] + self._links = [] # one bool per open : has an accessible name + + def handle_starttag(self, tag, attrs): + a = {k: (v or '') for k, v in attrs} + if tag == 'html' and a.get('lang'): + self.lang = a['lang'] + elif tag == 'title' and self.title is None: + self._capture, self._buf = 'title', [] + elif tag == 'meta': + for (attr, value), key in META.items(): + if a.get(attr, '').lower() == value: + self.meta[key] = a.get('content', '') + elif tag == 'link': + rel = a.get('rel', '').lower().split() + if 'canonical' in rel and a.get('href'): + self.canonical = a['href'] + if 'alternate' in rel and 'hreflang' in a: + self.hreflang += 1 + if 'icon' in rel: + self.favicon_link = True + elif tag == 'script' and a.get('type', '').lower() == 'application/ld+json': + self._capture, self._buf = 'jsonld', [] + elif re.fullmatch(r'h[1-6]', tag): + self.headings.append(int(tag[1])) + elif tag == 'img': + if 'alt' not in a: + self.img_without_alt += 1 + elif a['alt'].strip() and self._links: + self._links[-1] = True + elif tag == 'a' and 'href' in a: + self._links.append(bool(a.get('aria-label', '').strip() or a.get('title', '').strip())) + + def handle_endtag(self, tag): + if tag == 'title' and self._capture == 'title': + self.title = ''.join(self._buf).strip() + self._capture = None + elif tag == 'script' and self._capture == 'jsonld': + self.jsonld.append(''.join(self._buf)) + self._capture = None + elif tag == 'a' and self._links: + if not self._links.pop(): + self.empty_links += 1 + + def handle_data(self, data): + if self._capture: + self._buf.append(data) + if self._links and data.strip(): + self._links[-1] = True + + +def _skips(levels): + return any(b > a + 1 for a, b in zip(levels, levels[1:])) + + +def page_signals(html): + c = _Collector() + c.feed(html) + c.close() + valid = [] + for block in c.jsonld: + try: + json.loads(block) + valid.append(True) + except ValueError: + valid.append(False) + s = { + 'title': bool(c.title), + 'titleLength': len(c.title or ''), + 'lang': c.lang, + 'description': bool(c.meta.get('description')), + 'descriptionLength': len(c.meta.get('description', '')), + 'viewport': 'viewport' in c.meta, + 'canonical': bool(c.canonical), + 'ogTitle': 'ogTitle' in c.meta, + 'ogDescription': 'ogDescription' in c.meta, + 'ogType': 'ogType' in c.meta, + 'ogUrl': 'ogUrl' in c.meta, + 'twitterCard': 'twitterCard' in c.meta, + 'jsonLd': bool(c.jsonld), + 'jsonLdBlocks': len(c.jsonld), + 'jsonLdValid': all(valid) if valid else None, + 'robotsMeta': c.meta.get('robotsMeta'), + 'hreflang': c.hreflang, + 'h1': c.headings.count(1), + 'headingSkip': _skips(c.headings), + 'imgWithoutAlt': c.img_without_alt, + 'emptyLinks': c.empty_links, + 'faviconLink': c.favicon_link, + } + s['present'] = sum(1 for k in PAGE_SIGNALS if s[k] not in (False, None, '')) + return s + + +def _sitemap(site_dir): + path = os.path.join(site_dir, 'sitemap.xml') + if not os.path.isfile(path): + return {'present': False, 'valid': None, 'urls': None} + try: + root = ET.parse(path).getroot() + except ET.ParseError: + return {'present': True, 'valid': False, 'urls': None} + if not root.tag.endswith(('urlset', 'sitemapindex')): + return {'present': True, 'valid': False, 'urls': None} + return {'present': True, 'valid': True, + 'urls': sum(1 for el in root.iter() if el.tag.endswith('loc'))} + + +def site_signals(site_dir, page_signals_list): + exists = lambda rel: os.path.isfile(os.path.join(site_dir, rel)) # noqa: E731 + return { + 'robotsTxt': exists('robots.txt'), + 'sitemap': _sitemap(site_dir), + 'feed': any(exists(f) for f in FEEDS), + 'favicon': any(exists(f) for f in FAVICONS) + or any(s.get('faviconLink') for s in page_signals_list), + } + + +def seo_section(site_dir, pages): + signals = {key: page_signals(site._read(os.path.join(site_dir, page['file']))) + for key, page in pages.items()} + return { + 'total': len(PAGE_SIGNALS), + 'pages': signals, + 'present': {key: s['present'] for key, s in signals.items()}, + 'site': site_signals(site_dir, list(signals.values())), + } +``` + +- [ ] **Step 4: Run the tests** + +Run: `python3 -m pytest -q tests/test_quality_seo.py && ruff check toolset tests` +Expected: all pass. If ruff flags E731 despite the `noqa`, turn `exists` into a nested `def`. + +- [ ] **Step 5: Commit** + +```bash +git add toolset/quality/seo.py tests/test_quality_seo.py +git commit -m "feat(quality): SEO signals per page and site (spec 009 R-9) + +Co-Authored-By: Claude Opus 5.5 (1M context) " +``` + +--- + +### Task 3: Reduce the collector output (`toolset/quality/reduce.py`) + +**Files:** +- Create: `toolset/quality/reduce.py` +- Create: `tests/fixtures/quality/raw.json` (a full collector output for the fixture site) +- Test: `tests/test_quality_reduce.py` + +**Interfaces:** +- Consumes: + - `site.extra_output`, `site.works_without_js`, `site.REFERENCE_ASSETS` (Task 1); + - `seo.seo_section` (Task 2); + - the `raw.json` contract in `plan.md`. +- Produces: + - `SECTIONS`, `spread` + - `reduce_lighthouse`, `reduce_axe`, `reduce_html`, `reduce_links`, `weight`, `js_section` + - `build_quality(raw, site_dir, pages, output_glob) -> dict` (the `quality.json` contract) + +- [ ] **Step 1: Write the raw fixture** + +Generate `tests/fixtures/quality/raw.json`, so it matches the fixture site byte for byte: + +```bash +python3 - <<'EOF' +import gzip, json, os, pathlib +site = pathlib.Path('tests/fixtures/quality/site') +def run(perf, lcp, failed): + return {"scores": {"performance": perf, "accessibility": 1, "best-practices": 1, "seo": 0.82}, + "metrics": {"fcp": lcp, "lcp": lcp, "tbt": 0, "cls": 0, "si": lcp}, + "resources": {"totalBytes": 2400, "jsBytes": 0, "cssBytes": 300, "requests": 2, "domSize": 40}, + "failedAudits": failed} +pages = {"index": {"url": "/", "file": "index.html"}, + "post": {"url": "/post/hello/", "file": "post/hello/index.html"}, + "404": {"url": "/404.html", "file": "404.html"}} +lh = {p: [run(0.97, 900, ["a"]), run(0.99, 700, ["c"]), run(0.98, 800, ["b"])] for p in pages} +files = [] +for f in sorted(str(p.relative_to(site)).replace(os.sep, '/') for p in site.rglob('*') if p.is_file()): + data = (site / f).read_bytes() + files.append({"path": f, "bytes": len(data), "gzip": len(gzip.compress(data, 9)), "br": len(gzip.compress(data, 9)) - 1}) +raw = {"tools": {"node": "v24.21.0", "lighthouse": "1.0.0", "playwright": "1.0.0", "chromium": "1.0", + "axeCore": "1.0.0", "htmlValidate": "1.0.0", "chromeLauncher": "1.0.0", "lychee": "1.0.0"}, + "pages": pages, + "lighthouse": {"mobile": lh, "desktop": lh}, + "axe": {"index": {"violations": []}, + "post": {"violations": [{"id": "color-contrast", "impact": "serious", "nodes": 3}, + {"id": "region", "impact": "moderate", "nodes": 1}]}, + "404": {"violations": []}}, + "rendered": {"post": {"title": "Hello", "firstParagraph": "First paragraph of the post."}}, + "html": {"index": {"messages": []}, + "post": {"messages": [{"ruleId": "no-inline-style", "severity": 2}, + {"ruleId": "prefer-native-element", "severity": 1}]}, + "404": {"messages": []}}, + "links": {"report": {"fail_map": {str(site / "index.html"): [ + {"url": "file:///x/missing.css", "status": {"text": "Cannot find file"}}, + {"url": "file:///x/post/hello/#nope", "status": {"text": "Cannot find fragment"}}]}}}, + "files": files} +pathlib.Path('tests/fixtures/quality/raw.json').write_text(json.dumps(raw, indent=2) + "\n") +EOF +``` + +- [ ] **Step 2: Write the failing tests** + +`tests/test_quality_reduce.py`: + +```python +import json +import pathlib + +from toolset.quality import reduce, site + +FIX = pathlib.Path(__file__).parent / "fixtures" / "quality" +SITE = FIX / "site" +GLOB = "post/*/index.html" + + +def _raw(): + return json.loads((FIX / "raw.json").read_text()) + + +def test_spread(): + assert reduce.spread([3, 1, 2]) == {"median": 2, "min": 1, "max": 3} + assert reduce.spread([None, 4]) == {"median": 4, "min": 4, "max": 4} + assert reduce.spread([None]) is None + + +def test_reduce_lighthouse_median_and_median_run_audits(): + lh = reduce.reduce_lighthouse(_raw()["lighthouse"]) + post = lh["mobile"]["post"] + assert post["status"] == "ok" and post["runs"] == 3 + assert post["scores"]["performance"] == {"median": 0.98, "min": 0.97, "max": 0.99} + assert post["metrics"]["lcp"] == {"median": 800, "min": 700, "max": 900} + # the run whose performance is the median (0.98) failed audit "b" + assert post["failedAudits"] == ["b"] + assert set(lh) == {"mobile", "desktop"} + + +def test_reduce_lighthouse_page_errors(): + raw = {"mobile": {"index": {"error": "chrome died"}, + "post": [{"error": "timeout"}, {"error": "timeout"}], + "404": [{"error": "x"}, _raw()["lighthouse"]["mobile"]["post"][0]]}, + "desktop": {}} + lh = reduce.reduce_lighthouse(raw) + assert lh["mobile"]["index"] == {"status": "error", "error": "chrome died"} + assert lh["mobile"]["post"] == {"status": "error", "error": "timeout"} + assert lh["mobile"]["404"]["status"] == "ok" and lh["mobile"]["404"]["runs"] == 1 + + +def test_reduce_axe_counts_by_impact(): + a = reduce.reduce_axe(_raw()["axe"]) + assert a["post"] == {"violations": {"critical": 0, "serious": 1, "moderate": 1, "minor": 0}, + "total": 2, "rules": ["color-contrast", "region"]} + assert a["index"]["total"] == 0 + assert reduce.reduce_axe({"index": {"error": "boom"}})["index"] == {"status": "error", "error": "boom"} + + +def test_reduce_html_counts(): + h = reduce.reduce_html(_raw()["html"]) + assert h["post"] == {"errors": 1, "warnings": 1, "rules": ["no-inline-style", "prefer-native-element"]} + + +def test_reduce_links_kinds(): + links = reduce.reduce_links(_raw()["links"]) + assert links["broken"] == 2 + assert links["byKind"] == {"link": 0, "anchor": 1, "asset": 1} + assert links["items"][0]["status"] == "Cannot find file" + + +def test_reduce_links_caps_items(): + fail = {"s": [{"url": "/p%d/" % i, "status": {"text": "x"}} for i in range(30)]} + links = reduce.reduce_links({"report": {"fail_map": fail}}) + assert links["broken"] == 30 and len(links["items"]) == reduce.MAX_BROKEN_LISTED == 20 + + +def test_weight(): + files = [{"path": "index.html", "bytes": 100, "gzip": 50, "br": 40}, + {"path": "post/a/index.html", "bytes": 300, "gzip": 100, "br": 90}, + {"path": "post/b/index.html", "bytes": 100, "gzip": 60, "br": 50}, + {"path": "assets/ssgberk.css", "bytes": 1000, "gzip": 400, "br": 300}] + assert reduce.weight(files, GLOB) == {"bytes": 1500, "gzipBytes": 610, "brotliBytes": 480, + "bytesPerPost": 200, "generatorAddedBytes": 500} + + +def test_js_section_classes(): + lh = {"mobile": {"index": {"status": "ok", "resources": {"jsBytes": {"median": 0}}}, + "post": {"status": "ok", "resources": {"jsBytes": {"median": 0}}}}} + files = [{"path": "a.js", "bytes": 10, "gzip": 5, "br": 4}] + js = reduce.js_section(lh, files, True) + assert js == {"pages": {"index": 0, "post": 0}, "diskBytes": 10, "jsClass": "none", "worksWithoutJs": True} + lh["mobile"]["post"]["resources"]["jsBytes"]["median"] = 5000 + assert reduce.js_section(lh, files, False)["jsClass"] == "hydrated" + assert reduce.js_section({"mobile": {}}, [], None)["jsClass"] is None + + +def test_build_quality_fixture_all_ok(): + pages = site.resolve_pages(str(SITE)) + q = reduce.build_quality(_raw(), str(SITE), pages, GLOB) + assert q["status"] == "ok" + assert q["pages"] == {"index": "/", "post": "/post/hello/", "404": "/404.html"} + for section in reduce.SECTIONS: + assert q[section]["status"] == "ok", (section, q[section]) + assert q["js"]["worksWithoutJs"] is True and q["js"]["jsClass"] == "none" + assert q["extra"]["files"] == 2 and q["seo"]["present"]["post"] == 2 + + +def test_build_quality_partial_errors(): + raw = _raw() + raw["lighthouse"] = {"error": "lighthouse crashed"} + del raw["links"] + raw["files"] = {"error": "EACCES"} + pages = site.resolve_pages(str(SITE)) + q = reduce.build_quality(raw, str(SITE), pages, GLOB) + assert q["lighthouse"] == {"status": "error", "error": "RuntimeError: lighthouse crashed"} + assert q["links"]["status"] == "error" and "missing" in q["links"]["error"] + assert q["weight"]["status"] == "error" and q["js"]["status"] == "error" + for section in ("seo", "html", "a11y", "extra"): + assert q[section]["status"] == "ok" + assert q["status"] == "ok" +``` + +- [ ] **Step 3: Run them and see them fail** + +Run: `python3 -m pytest -q tests/test_quality_reduce.py` +Expected: FAIL with `ImportError: cannot import name 'reduce'`. + +- [ ] **Step 4: Implement `toolset/quality/reduce.py`** + +```python +''' +Reduces the collector output (quality/run.mjs raw.json) into quality.json +(spec 009 R-8, R-10..R-16). Pure functions; the contracts are in plan.md. +''' +import fnmatch +import posixpath +import statistics + +from toolset.quality import seo, site + +SECTIONS = ('lighthouse', 'seo', 'html', 'a11y', 'links', 'weight', 'js', 'extra') +PRESETS = ('mobile', 'desktop') +SCORES = ('performance', 'accessibility', 'best-practices', 'seo') +METRICS = ('fcp', 'lcp', 'tbt', 'cls', 'si') +RESOURCES = ('totalBytes', 'jsBytes', 'cssBytes', 'requests', 'domSize') +IMPACTS = ('critical', 'serious', 'moderate', 'minor') +MAX_BROKEN_LISTED = 20 + + +def spread(values): + vs = [v for v in values if v is not None] + if not vs: + return None + return {'median': statistics.median(vs), 'min': min(vs), 'max': max(vs)} + + +def _is_error(part): + return isinstance(part, dict) and 'error' in part + + +def _error(part): + return {'status': 'error', 'error': part['error']} + + +def _median_run(runs): + ranked = sorted(runs, key=lambda r: (r['scores'].get('performance') is None, + r['scores'].get('performance') or 0)) + return ranked[(len(ranked) - 1) // 2] + + +def _lighthouse_page(runs): + if _is_error(runs): + return _error(runs) + good = [r for r in runs if not _is_error(r)] + if not good: + return {'status': 'error', 'error': runs[0]['error'] if runs else 'no runs'} + return { + 'status': 'ok', + 'runs': len(good), + 'scores': {k: spread([r['scores'].get(k) for r in good]) for k in SCORES}, + 'metrics': {k: spread([r['metrics'].get(k) for r in good]) for k in METRICS}, + 'resources': {k: spread([r['resources'].get(k) for r in good]) for k in RESOURCES}, + 'failedAudits': _median_run(good)['failedAudits'], + } + + +def reduce_lighthouse(raw_lh): + return {preset: {page: _lighthouse_page(runs) + for page, runs in (raw_lh.get(preset) or {}).items()} + for preset in PRESETS} + + +def reduce_axe(raw_axe): + out = {} + for page, data in raw_axe.items(): + if _is_error(data): + out[page] = _error(data) + continue + counts = {impact: 0 for impact in IMPACTS} + for v in data.get('violations') or []: + if v.get('impact') in counts: + counts[v['impact']] += 1 + out[page] = {'violations': counts, 'total': len(data.get('violations') or []), + 'rules': sorted({v['id'] for v in data.get('violations') or []})} + return out + + +def reduce_html(raw_html): + out = {} + for page, data in raw_html.items(): + if _is_error(data): + out[page] = _error(data) + continue + messages = data.get('messages') or [] + out[page] = {'errors': sum(1 for m in messages if m.get('severity') == 2), + 'warnings': sum(1 for m in messages if m.get('severity') == 1), + 'rules': sorted({m['ruleId'] for m in messages if m.get('ruleId')})} + return out + + +def _kind(url): + if '#' in url: + return 'anchor' + path = url.split('?', 1)[0] + if path.endswith('/') or path.endswith('.html') or not posixpath.splitext(path)[1]: + return 'link' + return 'asset' + + +def reduce_links(raw_links): + report = raw_links.get('report') or {} + fail_map = report.get('fail_map') or report.get('error_map') or {} + items = [] + for source, failures in sorted(fail_map.items()): + for f in failures: + status = f.get('status') + items.append({'source': source, 'url': f.get('url'), + 'status': status.get('text') if isinstance(status, dict) else status}) + by_kind = {'link': 0, 'anchor': 0, 'asset': 0} + for item in items: + by_kind[_kind(item['url'] or '')] += 1 + return {'broken': len(items), 'byKind': by_kind, 'items': items[:MAX_BROKEN_LISTED]} + + +def weight(files, output_glob): + posts = [f['bytes'] for f in files if fnmatch.fnmatch(f['path'], output_glob)] + total = sum(f['bytes'] for f in files) + reference = sum(f['bytes'] for f in files if f['path'] in site.REFERENCE_ASSETS) + return {'bytes': total, + 'gzipBytes': sum(f['gzip'] for f in files), + 'brotliBytes': sum(f['br'] for f in files), + 'bytesPerPost': round(sum(posts) / len(posts)) if posts else None, + 'generatorAddedBytes': total - reference} + + +def js_section(lighthouse, files, works): + pages = {} + for page, data in (lighthouse.get('mobile') or {}).items(): + js = (data.get('resources') or {}).get('jsBytes') if data.get('status') == 'ok' else None + pages[page] = js['median'] if js else None + known = [v for v in pages.values() if v is not None] + js_class = None if not known else ('none' if all(v == 0 for v in known) else 'hydrated') + return {'pages': pages, + 'diskBytes': sum(f['bytes'] for f in files if f['path'].endswith(('.js', '.mjs'))), + 'jsClass': js_class, + 'worksWithoutJs': works} + + +def _raw(raw, key): + part = raw.get(key) + if part is None: + raise RuntimeError('%s missing in collector output' % key) + if _is_error(part): + raise RuntimeError(part['error']) + return part + + +def _section(fn): + try: + return dict({'status': 'ok'}, **fn()) + except Exception as e: + return {'status': 'error', 'error': '%s: %s' % (type(e).__name__, e)} + + +def build_quality(raw, site_dir, pages, output_glob): + '''quality.json (R-16): one section per check, each ok or error.''' + q = {'status': 'ok', 'pages': {k: v['url'] for k, v in pages.items()}, + 'tools': raw.get('tools')} + q['lighthouse'] = _section(lambda: reduce_lighthouse(_raw(raw, 'lighthouse'))) + q['seo'] = _section(lambda: seo.seo_section(site_dir, pages)) + q['html'] = _section(lambda: {'pages': reduce_html(_raw(raw, 'html'))}) + q['a11y'] = _section(lambda: {'pages': reduce_axe(_raw(raw, 'axe'))}) + q['links'] = _section(lambda: reduce_links(_raw(raw, 'links'))) + q['weight'] = _section(lambda: weight(_raw(raw, 'files'), output_glob)) + + def js(): + if q['lighthouse']['status'] != 'ok': + raise RuntimeError('lighthouse section failed') + rendered = (raw.get('rendered') or {}).get('post') if not _is_error(raw.get('rendered')) else None + works = site.works_without_js(site_dir, pages['post']['file'], rendered) + return js_section(q['lighthouse'], _raw(raw, 'files'), works) + + q['js'] = _section(js) + q['extra'] = _section(lambda: site.extra_output(site_dir, output_glob)) + return q +``` + +- [ ] **Step 5: Run the tests** + +Run: `python3 -m pytest -q tests/test_quality_reduce.py && ruff check toolset tests` +Expected: all pass. + +- [ ] **Step 6: Commit** + +```bash +git add toolset/quality/reduce.py tests/test_quality_reduce.py tests/fixtures/quality/raw.json +git commit -m "feat(quality): reduce collector output into quality.json (spec 009) + +Co-Authored-By: Claude Opus 5.5 (1M context) " +``` + +--- + +### Task 4: Collector image (`quality/`) + +**Files:** +- Create: `quality/Dockerfile`, `quality/package.json`, `quality/package-lock.json`, `quality/run.mjs`, `quality/.dockerignore` +- Create: `quality/test/pages.json`, `quality/test/expected-shape.json`, `quality/test/check-shape.mjs` +- Test: the image itself (Step 7). It needs Docker; take the Docker lock in `/Users/jobs/Dev/ssgberk/.docker-smoke.lock` with `mkdir`/`rmdir` if working on the shared machine, and check for ≥5 GiB free disk first. + +**Interfaces:** +- Consumes: the fixture site (Task 1). +- Produces: the image `ssgberk/quality` and the collector CLI and `raw.json` contracts in `plan.md`. + +- [ ] **Step 1: Pin the versions** + +Resolve the latest stable versions, and record them in the commit message: + +```bash +for p in lighthouse chrome-launcher playwright axe-core html-validate; do echo "$p $(npm view $p version)"; done +gh release view --repo lycheeverse/lychee --json tagName,assets --jq '.tagName, [.assets[].name]' +``` + +`quality/package.json` (replace each `X.Y.Z` with the resolved version, exact, no `^`): + +```json +{ + "name": "ssgberk-quality", + "private": true, + "type": "module", + "description": "SSGBerk output quality collector (benchmark-tool spec 009)", + "dependencies": { + "axe-core": "X.Y.Z", + "chrome-launcher": "X.Y.Z", + "html-validate": "X.Y.Z", + "lighthouse": "X.Y.Z", + "playwright": "X.Y.Z" + } +} +``` + +Then generate the lock: `cd quality && npm install --package-lock-only --ignore-scripts && cd ..`. If `npm` is not on the host, run the same command in `docker run --rm -v "$PWD/quality:/q" -w /q node:24 ...`. + +`quality/.dockerignore`: + +``` +node_modules +test +``` + +- [ ] **Step 2: Write `quality/Dockerfile`** + +Set `LYCHEE_VERSION` to the resolved release tag without the `lychee-v` or `v` prefix. Check that the asset names match `lychee--unknown-linux-gnu.tar.gz`, and adjust the URL if the release uses another name. + +```dockerfile +# SSGBerk quality collector (benchmark-tool spec 009): Lighthouse, axe-core, +# html-validate and lychee against Playwright's Chromium. Runs with no network. +FROM ubuntu:24.04 + +ARG DEBIAN_FRONTEND=noninteractive +ARG NODE_VERSION=24.21.0 +ARG LYCHEE_VERSION=X.Y.Z + +RUN apt-get -yqq update \ + && apt-get -yqq install --no-install-recommends ca-certificates curl xz-utils \ + && rm -rf /var/lib/apt/lists/* + +RUN ARCH="$(dpkg --print-architecture)" \ + && case "$ARCH" in amd64) NARCH=x64 ;; arm64) NARCH=arm64 ;; *) echo "unsupported $ARCH"; exit 1 ;; esac \ + && curl -fsSL "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${NARCH}.tar.xz" \ + | tar -xJ -C /usr/local --strip-components=1 \ + && node --version + +RUN ARCH="$(dpkg --print-architecture)" \ + && case "$ARCH" in amd64) LARCH=x86_64 ;; arm64) LARCH=aarch64 ;; *) echo "unsupported $ARCH"; exit 1 ;; esac \ + && curl -fsSL "https://github.com/lycheeverse/lychee/releases/download/lychee-v${LYCHEE_VERSION}/lychee-${LARCH}-unknown-linux-gnu.tar.gz" \ + | tar -xz -C /usr/local/bin lychee \ + && lychee --version + +ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright NPM_CONFIG_UPDATE_NOTIFIER=false +WORKDIR /quality +COPY package.json package-lock.json ./ +RUN npm ci --omit=dev \ + && npx playwright install --with-deps chromium \ + && rm -rf /root/.npm /var/lib/apt/lists/* +COPY run.mjs ./ +RUN mkdir -p /work/out +WORKDIR /work +``` + +- [ ] **Step 3: Write `quality/run.mjs`** + +```js +// SSGBerk quality collector (benchmark-tool spec 009). Writes raw data only: +// toolset/quality/reduce.py turns raw.json into quality.json. +import { execFile } from 'node:child_process'; +import fs from 'node:fs/promises'; +import http from 'node:http'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs, promisify } from 'node:util'; +import zlib from 'node:zlib'; + +import * as chromeLauncher from 'chrome-launcher'; +import { HtmlValidate } from 'html-validate'; +import lighthouse from 'lighthouse'; +import desktopConfig from 'lighthouse/core/config/desktop-config.js'; +import { chromium } from 'playwright'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const RUNS = 3; +const PRESETS = { mobile: undefined, desktop: desktopConfig }; +const CATEGORIES = ['performance', 'accessibility', 'best-practices', 'seo']; +const CHROME_FLAGS = ['--headless=new', '--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage']; +const NOT_SCORED = new Set(['informative', 'manual', 'notApplicable', 'error']); +const TYPES = { + '.html': 'text/html; charset=utf-8', '.css': 'text/css; charset=utf-8', + '.js': 'text/javascript; charset=utf-8', '.mjs': 'text/javascript; charset=utf-8', + '.json': 'application/json', '.xml': 'application/xml', '.txt': 'text/plain; charset=utf-8', + '.svg': 'image/svg+xml', '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', + '.webp': 'image/webp', '.gif': 'image/gif', '.ico': 'image/x-icon', '.woff': 'font/woff', + '.woff2': 'font/woff2', '.wasm': 'application/wasm', +}; +const COMPRESSIBLE = /^(text\/|application\/(json|xml|wasm)|image\/svg\+xml)/; +const execFileP = promisify(execFile); + +async function guard(fn) { + try { + return await fn(); + } catch (e) { + return { error: String(e && e.message ? e.message : e) }; + } +} + +async function isFile(p) { + try { + return (await fs.stat(p)).isFile(); + } catch { + return false; + } +} + +async function resolveFile(root, pathname) { + const rel = decodeURIComponent(pathname).replace(/^\/+/, ''); + const base = path.resolve(root, rel); + if (base !== root && !base.startsWith(root + path.sep)) return null; + const candidates = rel === '' || rel.endsWith('/') + ? [path.join(base, 'index.html')] + : [base, path.join(base, 'index.html'), `${base}.html`]; + for (const c of candidates) if (await isFile(c)) return c; + return null; +} + +// Fixed static server: gzip, no-cache, directory index, 404.html with status 404. +function serve(root) { + const server = http.createServer(async (req, res) => { + const { pathname } = new URL(req.url, 'http://127.0.0.1'); + let file = await resolveFile(root, pathname); + let status = 200; + if (!file) { + status = 404; + file = (await isFile(path.join(root, '404.html'))) ? path.join(root, '404.html') : null; + } + let body = file ? await fs.readFile(file) : Buffer.from('Not found'); + const type = file ? TYPES[path.extname(file).toLowerCase()] || 'application/octet-stream' + : 'text/plain; charset=utf-8'; + const headers = { 'content-type': type, 'cache-control': 'no-cache' }; + if (COMPRESSIBLE.test(type) && /\bgzip\b/.test(req.headers['accept-encoding'] || '')) { + body = zlib.gzipSync(body); + headers['content-encoding'] = 'gzip'; + headers.vary = 'accept-encoding'; + } + headers['content-length'] = body.length; + res.writeHead(status, headers); + res.end(req.method === 'HEAD' ? undefined : body); + }); + return new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve({ + server, origin: `http://127.0.0.1:${server.address().port}`, + }))); +} + +function summarize(lhr) { + const audits = lhr.audits; + const value = (id) => audits[id]?.numericValue ?? null; + const requests = audits['network-requests']?.details?.items ?? []; + const bytes = (type) => requests.filter((r) => r.resourceType === type) + .reduce((sum, r) => sum + (r.transferSize || 0), 0); + return { + scores: Object.fromEntries(CATEGORIES.map((c) => [c, lhr.categories[c]?.score ?? null])), + metrics: { + fcp: value('first-contentful-paint'), + lcp: value('largest-contentful-paint'), + tbt: value('total-blocking-time'), + cls: value('cumulative-layout-shift'), + si: value('speed-index'), + }, + resources: { + totalBytes: value('total-byte-weight'), + jsBytes: bytes('Script'), + cssBytes: bytes('Stylesheet'), + requests: requests.length, + domSize: value('dom-size') ?? value('dom-size-insight'), + }, + failedAudits: Object.values(audits) + .filter((a) => !NOT_SCORED.has(a.scoreDisplayMode) && a.score !== null && a.score < 1) + .map((a) => a.id).sort(), + }; +} + +async function runLighthouse(origin, pages) { + const chrome = await chromeLauncher.launch({ + chromePath: chromium.executablePath(), chromeFlags: CHROME_FLAGS, + }); + try { + const out = {}; + for (const [preset, config] of Object.entries(PRESETS)) { + out[preset] = {}; + for (const [key, page] of Object.entries(pages)) { + const runs = []; + for (let i = 0; i < RUNS; i += 1) { + runs.push(await guard(async () => { + const result = await lighthouse(origin + page.url, { + port: chrome.port, output: 'json', logLevel: 'error', onlyCategories: CATEGORIES, + }, config); + if (!result?.lhr) throw new Error('lighthouse returned no report'); + if (result.lhr.runtimeError) throw new Error(result.lhr.runtimeError.message); + return summarize(result.lhr); + })); + } + out[preset][key] = runs; + } + } + return out; + } finally { + await chrome.kill(); + } +} + +async function runBrowserChecks(origin, pages) { + const axeSource = await fs.readFile(path.join(HERE, 'node_modules', 'axe-core', 'axe.min.js'), 'utf8'); + const browser = await chromium.launch({ args: ['--no-sandbox', '--disable-dev-shm-usage'] }); + try { + const axe = {}; + for (const [key, page] of Object.entries(pages)) { + axe[key] = await guard(async () => { + const tab = await browser.newPage(); + try { + await tab.goto(origin + page.url, { waitUntil: 'load' }); + await tab.addScriptTag({ content: axeSource }); + const result = await tab.evaluate(() => window.axe.run(document, { resultTypes: ['violations'] })); + return { + violations: result.violations.map((v) => ({ id: v.id, impact: v.impact, nodes: v.nodes.length })), + }; + } finally { + await tab.close(); + } + }); + } + const post = await guard(async () => { + const tab = await browser.newPage(); + try { + await tab.goto(origin + pages.post.url, { waitUntil: 'networkidle' }); + const text = async (selector) => ((await tab.locator(selector).first() + .textContent({ timeout: 5000 })) || '').trim(); + return { title: await text('h1.post-title'), firstParagraph: await text('.post-body p') }; + } finally { + await tab.close(); + } + }); + return { axe, rendered: { post }, chromium: browser.version() }; + } finally { + await browser.close(); + } +} + +async function runHtmlValidate(site, pages) { + const validator = new HtmlValidate({ extends: ['html-validate:recommended'] }); + const out = {}; + for (const [key, page] of Object.entries(pages)) { + out[key] = await guard(async () => { + const report = await validator.validateFile(path.join(site, page.file)); + return { + messages: report.results.flatMap((r) => r.messages) + .map((m) => ({ ruleId: m.ruleId, severity: m.severity })), + }; + }); + } + return out; +} + +async function runLychee(site, outDir) { + const report = path.join(outDir, 'lychee.json'); + try { + await execFileP('lychee', ['--offline', '--no-progress', '--include-fragments', '--format', 'json', + '--output', report, '--root-dir', site, site], { maxBuffer: 64 * 1024 * 1024 }); + } catch (e) { + // exit code 2 means broken links; the report is still written + if (!(await isFile(report))) throw e; + } + return { report: JSON.parse(await fs.readFile(report, 'utf8')) }; +} + +async function listFiles(dir) { + const out = []; + for (const entry of await fs.readdir(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) out.push(...await listFiles(full)); + else if (entry.isFile()) out.push(full); + } + return out; +} + +async function sizes(site) { + const files = []; + for (const full of (await listFiles(site)).sort()) { + const data = await fs.readFile(full); + files.push({ + path: path.relative(site, full).split(path.sep).join('/'), + bytes: data.length, + gzip: zlib.gzipSync(data, { level: 9 }).length, + br: zlib.brotliCompressSync(data, { + params: { [zlib.constants.BROTLI_PARAM_QUALITY]: 11 }, + }).length, + }); + } + return files; +} + +async function packageVersion(name) { + const file = path.join(HERE, 'node_modules', name, 'package.json'); + return JSON.parse(await fs.readFile(file, 'utf8')).version; +} + +async function tools(chromiumVersion) { + const lychee = await guard(async () => (await execFileP('lychee', ['--version'])).stdout.trim() + .split(/\s+/).pop()); + return { + node: process.version, + lighthouse: await packageVersion('lighthouse'), + playwright: await packageVersion('playwright'), + chromium: chromiumVersion ?? null, + axeCore: await packageVersion('axe-core'), + htmlValidate: await packageVersion('html-validate'), + chromeLauncher: await packageVersion('chrome-launcher'), + lychee: typeof lychee === 'string' ? lychee : null, + }; +} + +async function main() { + const { values } = parseArgs({ + options: { site: { type: 'string' }, pages: { type: 'string' }, out: { type: 'string' } }, + }); + const site = path.resolve(values.site); + const pages = JSON.parse(values.pages); + const outDir = path.resolve(values.out); + await fs.mkdir(outDir, { recursive: true }); + const raw = { pages }; + const { server, origin } = await serve(site); + try { + const browser = await guard(() => runBrowserChecks(origin, pages)); + raw.axe = browser.error ? browser : browser.axe; + raw.rendered = browser.error ? browser : browser.rendered; + raw.lighthouse = await guard(() => runLighthouse(origin, pages)); + raw.html = await guard(() => runHtmlValidate(site, pages)); + raw.links = await guard(() => runLychee(site, outDir)); + raw.files = await guard(() => sizes(site)); + raw.tools = await guard(() => tools(browser.chromium)); + } finally { + server.close(); + } + await fs.writeFile(path.join(outDir, 'raw.json'), JSON.stringify(raw, null, 2)); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); +``` + +- [ ] **Step 4: Write the collector test files** + +`quality/test/pages.json`: + +```json +{"index": {"url": "/", "file": "index.html"}, "post": {"url": "/post/hello/", "file": "post/hello/index.html"}, "404": {"url": "/404.html", "file": "404.html"}} +``` + +`quality/test/expected-shape.json`. A string is a type (`"a|b"` allows either), an object is checked key by key, and a one-element array means every element must match that element: + +```json +{ + "tools": {"node": "string", "lighthouse": "string", "playwright": "string", "chromium": "string", + "axeCore": "string", "htmlValidate": "string", "chromeLauncher": "string", "lychee": "string"}, + "pages": "object", + "lighthouse": { + "mobile": { + "index": [{"scores": {"performance": "number", "accessibility": "number", "best-practices": "number", "seo": "number"}, + "metrics": {"fcp": "number", "lcp": "number", "tbt": "number", "cls": "number", "si": "number"}, + "resources": {"totalBytes": "number", "jsBytes": "number", "cssBytes": "number", "requests": "number", "domSize": "number|null"}, + "failedAudits": "array"}], + "post": [{"scores": {"performance": "number", "accessibility": "number", "best-practices": "number", "seo": "number"}, + "metrics": {"fcp": "number", "lcp": "number", "tbt": "number", "cls": "number", "si": "number"}, + "resources": {"totalBytes": "number", "jsBytes": "number", "cssBytes": "number", "requests": "number", "domSize": "number|null"}, + "failedAudits": "array"}], + "404": [{"scores": {"performance": "number", "accessibility": "number", "best-practices": "number", "seo": "number"}, + "metrics": {"fcp": "number", "lcp": "number", "tbt": "number", "cls": "number", "si": "number"}, + "resources": {"totalBytes": "number", "jsBytes": "number", "cssBytes": "number", "requests": "number", "domSize": "number|null"}, + "failedAudits": "array"}] + }, + "desktop": { + "index": [{"scores": {"performance": "number", "accessibility": "number", "best-practices": "number", "seo": "number"}, + "metrics": {"fcp": "number", "lcp": "number", "tbt": "number", "cls": "number", "si": "number"}, + "resources": {"totalBytes": "number", "jsBytes": "number", "cssBytes": "number", "requests": "number", "domSize": "number|null"}, + "failedAudits": "array"}], + "post": [{"scores": {"performance": "number", "accessibility": "number", "best-practices": "number", "seo": "number"}, + "metrics": {"fcp": "number", "lcp": "number", "tbt": "number", "cls": "number", "si": "number"}, + "resources": {"totalBytes": "number", "jsBytes": "number", "cssBytes": "number", "requests": "number", "domSize": "number|null"}, + "failedAudits": "array"}], + "404": [{"scores": {"performance": "number", "accessibility": "number", "best-practices": "number", "seo": "number"}, + "metrics": {"fcp": "number", "lcp": "number", "tbt": "number", "cls": "number", "si": "number"}, + "resources": {"totalBytes": "number", "jsBytes": "number", "cssBytes": "number", "requests": "number", "domSize": "number|null"}, + "failedAudits": "array"}] + } + }, + "axe": {"index": {"violations": "array"}, "post": {"violations": "array"}, "404": {"violations": "array"}}, + "rendered": {"post": {"title": "string", "firstParagraph": "string"}}, + "html": {"index": {"messages": "array"}, "post": {"messages": "array"}, "404": {"messages": "array"}}, + "links": {"report": "object"}, + "files": [{"path": "string", "bytes": "number", "gzip": "number", "br": "number"}] +} +``` + +`quality/test/check-shape.mjs`: + +```js +// Checks a collector raw.json against expected-shape.json (keys and types, not values). +import fs from 'node:fs'; + +const [rawPath, shapePath] = process.argv.slice(2); +const raw = JSON.parse(fs.readFileSync(rawPath, 'utf8')); +const shape = JSON.parse(fs.readFileSync(shapePath, 'utf8')); +const errors = []; + +function typeOf(v) { + if (v === null) return 'null'; + if (Array.isArray(v)) return 'array'; + return typeof v; +} + +function check(value, expected, where) { + if (typeof expected === 'string') { + if (!expected.split('|').includes(typeOf(value))) errors.push(`${where}: ${typeOf(value)} is not ${expected}`); + } else if (Array.isArray(expected)) { + if (!Array.isArray(value) || value.length === 0) errors.push(`${where}: expected a non-empty array`); + else value.forEach((v, i) => check(v, expected[0], `${where}[${i}]`)); + } else { + if (typeOf(value) !== 'object') { + errors.push(`${where}: expected an object, got ${JSON.stringify(value).slice(0, 200)}`); + return; + } + for (const [k, e] of Object.entries(expected)) check(value[k], e, `${where}.${k}`); + } +} + +check(raw, shape, 'raw'); +for (const preset of ['mobile', 'desktop']) { + for (const page of ['index', 'post', '404']) { + if ((raw.lighthouse?.[preset]?.[page] || []).length !== 3) errors.push(`lighthouse.${preset}.${page}: expected 3 runs`); + } +} +if (raw.rendered?.post?.title !== 'Hello') errors.push('rendered.post.title is not "Hello"'); +if (raw.lighthouse?.mobile?.post?.[0]?.resources?.jsBytes !== 0) errors.push('fixture post page should request 0 JS bytes'); +if (errors.length) { + console.error(errors.join('\n')); + process.exit(1); +} +console.log('raw.json shape ok'); +``` + +- [ ] **Step 5: Build the image** + +Run: `docker build -t ssgberk/quality quality` +Expected: the build succeeds, and both `node --version` and `lychee --version` print. + +- [ ] **Step 6: Run the collector on the fixture and check the shape** + +```bash +docker run --rm --network none \ + -v "$PWD/tests/fixtures/quality/site:/fixture:ro" -v "$PWD/quality/test:/test:ro" \ + ssgberk/quality sh -c 'node /quality/run.mjs --site /fixture --pages "$(cat /test/pages.json)" --out /tmp/out \ + && node /test/check-shape.mjs /tmp/out/raw.json /test/expected-shape.json' +``` + +Expected: `raw.json shape ok`. If a check fails because the pinned version renamed something (a Lighthouse audit id such as `network-requests` or `dom-size`, a lychee flag or its JSON keys, or the `html-validate` API), fix `run.mjs` for the pinned version. If the lychee JSON uses keys other than `fail_map` or `error_map`, update `reduce.reduce_links` and the `links` part of `tests/fixtures/quality/raw.json` to match. Re-run until it passes. Keep the fixture site unchanged. + +- [ ] **Step 7: Remove the local image and commit** + +```bash +docker rmi ssgberk/quality +git add quality +git commit -m "feat(quality): pinned collector image with Lighthouse, axe-core, html-validate, lychee (spec 009 R-4, R-5) + +Pinned: lighthouse X, chrome-launcher X, playwright X (Chromium X), axe-core X, +html-validate X, lychee X, Node 24.21.0. + +Co-Authored-By: Claude Opus 5.5 (1M context) " +``` + +--- + +### Task 5: Docker plumbing (`toolset/utils/docker_helper.py`) + +**Files:** +- Modify: `toolset/utils/docker_helper.py` + - imports at the top: `io`, `json`, `posixpath`, `tarfile`; + - class attributes next to `last_build`; + - `benchmark()` (signature and the end of its `try` block); + - new methods after `benchmark()`. +- Test: `tests/test_quality_docker.py` + +**Interfaces:** +- Consumes: nothing from earlier tasks (pure Docker plumbing). +- Produces: + - `QUALITY_IMAGE`, `QUALITY_TIMEOUT_SECONDS` + - `DockerHelper.benchmark(framework_test, script, variables, raw_file, resources, timeout_seconds, export=None)` + - `DockerHelper.build_quality_image() -> str` + - `DockerHelper.run_quality(tar_path, site_name, pages, timeout_seconds=QUALITY_TIMEOUT_SECONDS) -> dict` + +- [ ] **Step 1: Write the failing tests** + +`tests/test_quality_docker.py`: + +```python +import io +import json +import tarfile +import types +from unittest import mock + +import docker +import pytest + +from toolset.utils import docker_helper + +RES = {"cpus": 4.0, "memoryBytes": 8589934592, "swap": False, "cpuset": "4-7"} + + +def _helper(): + helper = docker_helper.DockerHelper.__new__(docker_helper.DockerHelper) + helper.benchmarker = mock.Mock() + helper.server = mock.Mock() + return helper + + +def _container(exit_code=0, oom=False): + c = mock.Mock() + c.logs.return_value = iter(()) + c.wait.return_value = {"StatusCode": exit_code} + c.attrs = {"State": {"OOMKilled": oom, "ExitCode": exit_code}, + "Config": {"WorkingDir": "/opt/hugo/src"}} + c.get_archive.return_value = (iter([b"tar", b"bytes"]), {}) + return c + + +def test_benchmark_exports_output_on_success(tmp_path): + helper, c = _helper(), _container() + helper.server.containers.run.return_value = c + dest = tmp_path / "quality" / "hugo" / "site.tar" + out = helper.benchmark(types.SimpleNamespace(name="hugo"), "build.sh", {}, str(tmp_path / "raw.txt"), + RES, 60, export=("public", str(dest))) + assert out["status"] == "ok" + c.get_archive.assert_called_once_with("/opt/hugo/src/public") + assert dest.read_bytes() == b"tarbytes" + c.remove.assert_called_once_with(force=True) + + +@pytest.mark.parametrize("exit_code,oom", [(1, False), (137, True)]) +def test_benchmark_no_export_on_failure(tmp_path, exit_code, oom): + helper, c = _helper(), _container(exit_code, oom) + helper.server.containers.run.return_value = c + helper.benchmark(types.SimpleNamespace(name="hugo"), "build.sh", {}, str(tmp_path / "raw.txt"), + RES, 60, export=("public", str(tmp_path / "site.tar"))) + c.get_archive.assert_not_called() + + +def test_benchmark_export_error_keeps_status(tmp_path): + helper, c = _helper(), _container() + c.get_archive.side_effect = docker.errors.NotFound("no such path") + helper.server.containers.run.return_value = c + out = helper.benchmark(types.SimpleNamespace(name="hugo"), "build.sh", {}, str(tmp_path / "raw.txt"), + RES, 60, export=("public", str(tmp_path / "site.tar"))) + assert out == {"status": "ok", "exitCode": 0} + + +def test_benchmark_without_export_never_reads_archive(tmp_path): + helper, c = _helper(), _container() + helper.server.containers.run.return_value = c + helper.benchmark(types.SimpleNamespace(name="hugo"), "build.sh", {}, str(tmp_path / "raw.txt"), RES, 60) + c.get_archive.assert_not_called() + + +def _raw_tar(payload): + buf = io.BytesIO() + data = json.dumps(payload).encode() + with tarfile.open(fileobj=buf, mode="w") as tar: + info = tarfile.TarInfo("raw.json") + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + return buf.getvalue() + + +def test_run_quality_isolated_and_cleaned_up(tmp_path): + helper = _helper() + helper._quality_image = "sha256:q" + c = mock.Mock() + c.wait.return_value = {"StatusCode": 0} + c.logs.return_value = b"" + c.get_archive.return_value = (iter([_raw_tar({"tools": {"node": "v24"}})]), {}) + helper.server.containers.create.return_value = c + tar_path = tmp_path / "site.tar" + tar_path.write_bytes(b"site-tar") + pages = {"index": {"url": "/", "file": "index.html"}} + raw = helper.run_quality(str(tar_path), "public", pages) + assert raw == {"tools": {"node": "v24"}} + args, kwargs = helper.server.containers.create.call_args + assert args[0] == "sha256:q" + assert kwargs["network_mode"] == "none" + assert "volumes" not in kwargs and "mounts" not in kwargs + assert kwargs["command"][:4] == ["node", "/quality/run.mjs", "--site", "/work/public"] + assert json.loads(kwargs["command"][kwargs["command"].index("--pages") + 1]) == pages + c.put_archive.assert_called_once_with("/work", b"site-tar") + c.get_archive.assert_called_once_with("/work/out/raw.json") + c.remove.assert_called_once_with(force=True) + + +def test_run_quality_without_raw_json_raises_and_cleans_up(tmp_path): + helper = _helper() + helper._quality_image = "sha256:q" + c = mock.Mock() + c.wait.return_value = {"StatusCode": 1} + c.logs.return_value = b"Error: boom" + c.get_archive.side_effect = docker.errors.NotFound("missing") + helper.server.containers.create.return_value = c + (tmp_path / "site.tar").write_bytes(b"x") + with pytest.raises(RuntimeError, match="exit 1.*boom"): + helper.run_quality(str(tmp_path / "site.tar"), "public", {}) + c.remove.assert_called_once_with(force=True) + + +def test_build_quality_image_once(monkeypatch, tmp_path): + builds = [] + + class FakeAPIClient: + def __init__(self, base_url=None): + pass + + def build(self, **kw): + builds.append(kw) + return iter([{"stream": "Step 1/2 : FROM ubuntu:24.04\n"}, {"stream": "Successfully built abc\n"}]) + + monkeypatch.setattr(docker_helper.docker, "APIClient", FakeAPIClient) + helper = _helper() + helper._quality_image = None + helper.benchmarker.config = types.SimpleNamespace(fw_root=str(tmp_path), server_docker_host=None) + helper.server.images.get.return_value = types.SimpleNamespace(id="sha256:abc") + assert helper.build_quality_image() == "sha256:abc" + assert helper.build_quality_image() == "sha256:abc" + assert len(builds) == 1 + assert builds[0]["path"] == str(tmp_path / "quality") and builds[0]["tag"] == docker_helper.QUALITY_IMAGE + + +def test_build_quality_image_error(monkeypatch, tmp_path): + class FakeAPIClient: + def __init__(self, base_url=None): + pass + + def build(self, **kw): + return iter([{"errorDetail": {"message": "npm ci failed"}}]) + + monkeypatch.setattr(docker_helper.docker, "APIClient", FakeAPIClient) + helper = _helper() + helper._quality_image = None + helper.benchmarker.config = types.SimpleNamespace(fw_root=str(tmp_path), server_docker_host=None) + with pytest.raises(RuntimeError, match="npm ci failed"): + helper.build_quality_image() +``` + +- [ ] **Step 2: Run them and see them fail** + +Run: `python3 -m pytest -q tests/test_quality_docker.py` +Expected: FAIL with `TypeError: ... unexpected keyword argument 'export'` and `AttributeError: ... run_quality`. + +- [ ] **Step 3: Implement the changes** + +At the top of `toolset/utils/docker_helper.py`, add `import io`, `import json`, `import posixpath` and `import tarfile` with the other stdlib imports. Next to `RUN_LABEL`, add: + +```python +QUALITY_IMAGE = 'ssgberk/quality' +QUALITY_TIMEOUT_SECONDS = 1800 +QUALITY_MEMORY = '4g' +``` + +In the class body, next to `last_build = None`, add `_quality_image = None`. + +Change the signature of `benchmark` to: + +```python + def benchmark(self, framework_test, script, variables, raw_file, + resources, timeout_seconds, export=None): + ''' + Runs the generator container with fixed resource limits and returns + {"status": "ok"|"timeout"|"oom", "exitCode": int}. The container is + always stopped and removed before returning. With export + (output_folder, dest_tar), a successful build's output is copied to + dest_tar first (spec 009 R-3). + ''' +``` + +At the end of its `try:` block, right after the OOM check and before `finally:`, insert: + +```python + if export and status == 'ok' and exit_code == 0: + DockerHelper.__export_output(container, *export) +``` + +Add these methods after `benchmark`: + +```python + @staticmethod + def __export_output(container, output_folder, dest): + '''Copies / to dest as a tar. Never raises.''' + try: + workdir = (container.attrs.get('Config') or {}).get('WorkingDir') or '/' + bits, _ = container.get_archive(posixpath.join(workdir, output_folder)) + os.makedirs(os.path.dirname(dest), exist_ok=True) + with open(dest, 'wb') as f: + for chunk in bits: + f.write(chunk) + except Exception as e: + log("quality: could not export %s: %s" % (output_folder, e), color=Fore.YELLOW) + + def build_quality_image(self): + ''' + Builds ssgberk/quality from /quality once per process + (spec 009 R-4), outside the generator build-time accounting. + ''' + if self._quality_image: + return self._quality_image + client = docker.APIClient(base_url=self.benchmarker.config.server_docker_host) + for token in client.build(path=os.path.join(self.benchmarker.config.fw_root, 'quality'), + dockerfile='Dockerfile', tag=QUALITY_IMAGE, forcerm=True, + pull=True, timeout=3600, decode=True): + if 'errorDetail' in token: + raise RuntimeError(token['errorDetail']['message']) + if token.get('stream', '').strip(): + log(token['stream'].rstrip(), prefix='quality: ') + self._quality_image = self.server.images.get(QUALITY_IMAGE).id + return self._quality_image + + def run_quality(self, tar_path, site_name, pages, + timeout_seconds=QUALITY_TIMEOUT_SECONDS): + ''' + Runs quality/run.mjs on the exported site with no network and no + mounts (spec 009 R-4, R-5) and returns its raw.json. + ''' + image = self.build_quality_image() + command = ['node', '/quality/run.mjs', '--site', '/work/' + site_name, + '--pages', json.dumps(pages), '--out', '/work/out'] + container = self.server.containers.create( + image, command=command, labels=self._run_labels(), network_mode='none', + mem_limit=QUALITY_MEMORY, working_dir='/work') + try: + with open(tar_path, 'rb') as f: + container.put_archive('/work', f.read()) + container.start() + result = container.wait(timeout=timeout_seconds) + try: + bits, _ = container.get_archive('/work/out/raw.json') + except docker.errors.NotFound: + logs = container.logs().decode('utf-8', 'replace') + raise RuntimeError('collector wrote no raw.json (exit %s): %s' % ( + result.get('StatusCode'), logs[-2000:])) + with tarfile.open(fileobj=io.BytesIO(b''.join(bits))) as tar: + return json.load(tar.extractfile(tar.getmembers()[0])) + finally: + try: + container.remove(force=True) + except docker.errors.NotFound: + pass +``` + +- [ ] **Step 4: Run all tests** + +Run: `python3 -m pytest -q && ruff check toolset tests` +Expected: all pass, including the existing `tests/test_docker_helper.py`. + +- [ ] **Step 5: Commit** + +```bash +git add toolset/utils/docker_helper.py tests/test_quality_docker.py +git commit -m "feat(quality): export build output and run the collector without network (spec 009 R-3..R-5) + +Co-Authored-By: Claude Opus 5.5 (1M context) " +``` + +--- + +### Task 6: The pass in a run (runner, CLI, Benchmarker, Results) + +**Files:** +- Create: `toolset/quality/runner.py` +- Modify: + - `toolset/run-tests.py`: a new `--quality` argument after `--no-cache`; + - `toolset/utils/benchmark_config.py`: `self.quality` next to `self.no_cache`; + - `toolset/benchmark/benchmarker.py`: `benchmark_type` in `__benchmark`, `__noise_control`, and two new private methods; + - `toolset/utils/results.py`: `STORED_RUN_KEYS`, `__init__`, `add_quality`, `__to_jsonable`, `write_summary`. +- Test: `tests/test_quality_runner.py` + +**Interfaces:** +- Consumes: + - `site.generator_config`, `site.extract`, `site.resolve_pages` (Task 1); + - `reduce.build_quality` (Task 3); + - `DockerHelper.run_quality`, `DockerHelper.benchmark(..., export=...)`, `DockerHelper._quality_image` (Task 5). +- Produces: + - `runner.eligible(config)`, `runner.skipped(config)`, `runner.archive_path(results_dir, name)`, `runner.run_pass(docker_helper, test, results_dir)` + - `Results.add_quality(name, quality, image_id=None)` + - `results.json` `quality` and `environment.quality` + - `config.quality` + +- [ ] **Step 1: Write the failing tests** + +`tests/test_quality_runner.py`: + +```python +import json +import pathlib +import tarfile +import types +from unittest import mock + +from toolset.quality import runner +from toolset.utils.results import STORED_RUN_KEYS, Results + +FIX = pathlib.Path(__file__).parent / "fixtures" / "quality" + + +def _cfg(**kw): + return types.SimpleNamespace(**kw) + + +def test_eligible_and_skipped(): + assert not runner.eligible(_cfg()) + assert not runner.skipped(_cfg()) + assert runner.eligible(_cfg(quality=True, suite_info=None, number_of_files="10", content_size="0.500")) + p0 = _cfg(quality=True, suite_info={"name": "P"}, number_of_files="50", content_size="5") + p1 = _cfg(quality=True, suite_info={"name": "P"}, number_of_files="50", content_size="50") + assert runner.eligible(p0) and not runner.skipped(p0) + assert not runner.eligible(p1) and runner.skipped(p1) + + +def _generator(tmp_path): + d = tmp_path / "frameworks" / "Go" / "hugo" + d.mkdir(parents=True) + (d / "benchmark_config.json").write_text(json.dumps( + {"framework": "hugo", "config": [{"output_folder": "public", "output_glob": "post/*/index.html"}]})) + return types.SimpleNamespace(name="hugo", directory=str(d)) + + +def _archive(results_dir): + tar_path = pathlib.Path(runner.archive_path(str(results_dir), "hugo")) + tar_path.parent.mkdir(parents=True) + with tarfile.open(tar_path, "w") as tar: + tar.add(FIX / "site", arcname="public") + return tar_path + + +def test_run_pass_ok(tmp_path): + test, results_dir = _generator(tmp_path), tmp_path / "results" + tar_path = _archive(results_dir) + helper = mock.Mock() + helper.run_quality.return_value = json.loads((FIX / "raw.json").read_text()) + q = runner.run_pass(helper, test, str(results_dir)) + assert q["status"] == "ok" and q["lighthouse"]["status"] == "ok" + args = helper.run_quality.call_args[0] + assert args[0] == str(tar_path) and args[1] == "public" + assert args[2]["post"] == {"url": "/post/hello/", "file": "post/hello/index.html"} + out = results_dir / "quality" / "hugo" + assert json.loads((out / "quality.json").read_text()) == q + assert (out / "raw.json").exists() and (out / "site" / "index.html").exists() + assert not tar_path.exists() + + +def test_run_pass_missing_archive(tmp_path): + test, results_dir = _generator(tmp_path), tmp_path / "results" + q = runner.run_pass(mock.Mock(), test, str(results_dir)) + assert q["status"] == "error" and "no exported output" in q["error"] + assert json.loads((results_dir / "quality" / "hugo" / "quality.json").read_text()) == q + + +def test_run_pass_collector_error(tmp_path): + test, results_dir = _generator(tmp_path), tmp_path / "results" + _archive(results_dir) + helper = mock.Mock() + helper.run_quality.side_effect = RuntimeError("collector wrote no raw.json (exit 1)") + q = runner.run_pass(helper, test, str(results_dir)) + assert q == {"status": "error", "error": "RuntimeError: collector wrote no raw.json (exit 1)"} + + +def _bm(tmp_path, quality=True, results=None, failure=None): + from toolset.benchmark.benchmarker import Benchmarker + tmp_path.mkdir(parents=True, exist_ok=True) + b = Benchmarker.__new__(Benchmarker) + b.config = types.SimpleNamespace(min_runs="1", run_test_timeout_seconds=10, resources={"cpus": 1}, + quality=quality, suite_info=None, number_of_files="50", + content_size="5") + test_type = mock.Mock() + test_type.get_script_name.return_value = "build.sh" + test_type.get_script_variables.return_value = {"min_runs": "1"} + b.config.types = {"datarate": test_type} + b.results = mock.Mock() + b.results.directory = str(tmp_path / "results") + b.results.get_raw_file.return_value = str(tmp_path / "raw.txt") + exports = [] + + def bench(ft, script, variables, raw_file, res, timeout, export=None): + exports.append(export) + return {"status": "ok", "exitCode": 0} + + b.docker_helper = mock.Mock() + b.docker_helper.benchmark.side_effect = bench + b.docker_helper._quality_image = "sha256:q" + b.results.parse_test.return_value = { + "results": [{"cv": None, "median": 1.0}] if results is None else results, + "status": "ok", "failureReason": failure, "unsupported": False} + b._Benchmarker__begin_logging = mock.Mock() + b._Benchmarker__end_logging = mock.Mock() + ft = _generator(tmp_path) + ft.runTests = {"datarate": mock.Mock(failed=False)} + return b, ft, exports + + +def test_quality_pass_runs_after_ok(tmp_path, monkeypatch): + b, ft, exports = _bm(tmp_path) + monkeypatch.setattr(runner, "run_pass", lambda helper, test, d: {"status": "ok", "test": test.name}) + b._Benchmarker__benchmark(ft, open(tmp_path / "log", "w")) + assert exports == [("public", runner.archive_path(b.results.directory, "hugo"))] + b.results.add_quality.assert_called_once_with("hugo", {"status": "ok", "test": "hugo"}, "sha256:q") + + +def test_quality_pass_missing_archive_keeps_result(tmp_path): + reported = [] + for quality in (False, True): + b, ft, _ = _bm(tmp_path / str(quality), quality=quality) + b._Benchmarker__benchmark(ft, open(tmp_path / ("log%s" % quality), "w")) + # everything but the generator object (its directory differs per run) + reported.append(b.results.report_benchmark_results.call_args[0][1:]) + if quality: + name, q, _image = b.results.add_quality.call_args[0] + assert name == "hugo" and q["status"] == "error" + assert reported[0] == reported[1] + + +def test_no_pass_without_results_or_with_failure(tmp_path, monkeypatch): + called = [] + monkeypatch.setattr(runner, "run_pass", lambda *a: called.append(a) or {"status": "ok"}) + for i, kw in enumerate(({"results": []}, {"failure": "nonconformant: index"})): + b, ft, _ = _bm(tmp_path / str(i), **kw) + b._Benchmarker__benchmark(ft, open(tmp_path / "log", "w")) + b.results.add_quality.assert_not_called() + assert called == [] + + +def test_skipped_cell_exports_nothing(tmp_path): + b, ft, exports = _bm(tmp_path) + b.config.suite_info = {"name": "P"} + b.config.content_size = "50" + b._Benchmarker__benchmark(ft, open(tmp_path / "log", "w")) + assert exports == [None] + b.results.add_quality.assert_not_called() + + +def test_results_add_quality(fake_benchmarker): + res = Results(fake_benchmarker) + res._environment = {"fingerprint": "f"} # what environment.capture would have cached + before = res._Results__to_jsonable() + assert "quality" not in before and "quality" not in before["environment"] + res.add_quality("hugo", {"status": "ok", "tools": {"lighthouse": "1.0.0"}}, "sha256:q") + after = res._Results__to_jsonable() + assert after["quality"] == {"hugo": {"status": "ok", "tools": {"lighthouse": "1.0.0"}}} + assert after["rawData"] == before["rawData"] and after["succeeded"] == before["succeeded"] + assert after["environment"] == {"fingerprint": "f", + "quality": {"lighthouse": "1.0.0", "image": "sha256:q"}} + + +def test_reparse_keeps_quality(fake_benchmarker, monkeypatch): + assert "quality" in STORED_RUN_KEYS + monkeypatch.setattr(Results, "_Results__count_commits", lambda self: None) + monkeypatch.setattr(Results, "_Results__count_sloc", lambda self: None) + res = Results(fake_benchmarker) + with open(res.file, "w") as f: + json.dump({"quality": {"hugo": {"status": "ok"}}, "rawData": {"datarate": {}}}, f) + res.reparse([]) + with open(res.file) as f: + assert json.load(f)["quality"] == {"hugo": {"status": "ok"}} +``` + +- [ ] **Step 2: Run them and see them fail** + +Run: `python3 -m pytest -q tests/test_quality_runner.py` +Expected: FAIL with `ImportError: cannot import name 'runner'`. + +- [ ] **Step 3: Implement `toolset/quality/runner.py`** + +```python +''' +The untimed quality pass of one generator (spec 009 R-1..R-3, R-16). +''' +import json +import os + +from toolset.quality import reduce, site + +# R-1: with --suite, the pass runs only in this (numberOfFiles, contentSize) cell +QUALITY_CELL = (50, 5.0) + + +def _cell(config): + return int(config.number_of_files), float(config.content_size) + + +def eligible(config): + if not getattr(config, 'quality', False): + return False + if getattr(config, 'suite_info', None) is None: + return True + return _cell(config) == QUALITY_CELL + + +def skipped(config): + return bool(getattr(config, 'quality', False)) and not eligible(config) + + +def archive_path(results_dir, name): + return os.path.join(results_dir, 'quality', name, 'site.tar') + + +def run_pass(docker_helper, test, results_dir): + '''Runs the pass, writes quality.json and returns it. Never raises.''' + out_dir = os.path.join(results_dir, 'quality', test.name) + tar_path = archive_path(results_dir, test.name) + try: + output_folder, output_glob = site.generator_config(test.directory) + if not os.path.exists(tar_path): + raise RuntimeError('no exported output (%s)' % output_folder) + site_dir = site.extract(tar_path, os.path.join(out_dir, 'site')) + pages = site.resolve_pages(site_dir) + site_name = os.path.basename(output_folder.rstrip('/')) + raw = docker_helper.run_quality(tar_path, site_name, pages) + with open(os.path.join(out_dir, 'raw.json'), 'w') as f: + json.dump(raw, f, indent=2) + quality = reduce.build_quality(raw, site_dir, pages, output_glob) + except Exception as e: + quality = {'status': 'error', 'error': '%s: %s' % (type(e).__name__, e)} + os.makedirs(out_dir, exist_ok=True) + with open(os.path.join(out_dir, 'quality.json'), 'w') as f: + json.dump(quality, f, indent=2) + if os.path.exists(tar_path): + os.remove(tar_path) + return quality +``` + +- [ ] **Step 4: Add the flag** + +In `toolset/run-tests.py`, after the `--no-cache` argument: + +```python + parser.add_argument( + '--quality', + action='store_true', + default=False, + help='After the timed runs, audit the output of the last build (Lighthouse, SEO ' + 'signals, HTML, accessibility, links, weight); untimed. With --suite, only in ' + 'the cell nf50-cs5 (spec 009)') +``` + +In `toolset/utils/benchmark_config.py`, after `self.no_cache = ...`: + +```python + self.quality = bool(getattr(args, 'quality', False)) +``` + +- [ ] **Step 5: Hook the pass into `Benchmarker`** + +In `toolset/benchmark/benchmarker.py`, add the imports `from toolset.quality import runner as quality_runner` and `from toolset.quality import site as quality_site` with the other `toolset` imports. + +In `__benchmark.benchmark_type`, before `if not test.failed:`, add: + +```python + export = self.__quality_export(framework_test) + self._quality_kwargs = {'export': export} if export else {} +``` + +Change both `self.docker_helper.benchmark(...)` calls, in `benchmark_type` and in `__noise_control`, to pass `**self._quality_kwargs` after `self.config.run_test_timeout_seconds`. In `__noise_control`, use `**getattr(self, '_quality_kwargs', {})`, because older tests call it directly. + +After `self.results.report_benchmark_results(...)` in `benchmark_type`, add: + +```python + if export and results['results'] and not results.get('failureReason') \ + and not results.get('unsupported'): + self.__quality_pass(framework_test, benchmark_log) +``` + +Add the two methods to the class: + +```python + def __quality_export(self, framework_test): + '''(output_folder, dest_tar) when the quality pass runs in this cell (spec 009 R-1).''' + if quality_runner.skipped(self.config): + log("quality: skipped (only cell nf50-cs5 of a suite)") + return None + if not quality_runner.eligible(self.config): + return None + try: + output_folder, _glob = quality_site.generator_config(framework_test.directory) + except Exception as e: + log("quality: cannot read output_folder: %s" % e) + return None + return (output_folder, + quality_runner.archive_path(self.results.directory, framework_test.name)) + + def __quality_pass(self, framework_test, benchmark_log): + '''Untimed; never changes the build result (spec 009 R-2).''' + log("QUALITY PASS %s (untimed)" % framework_test.name, file=benchmark_log, border='*') + quality = quality_runner.run_pass(self.docker_helper, framework_test, self.results.directory) + self.results.add_quality(framework_test.name, quality, + getattr(self.docker_helper, '_quality_image', None)) + log("quality: %s" % quality.get('status'), file=benchmark_log) +``` + +- [ ] **Step 6: Record quality in `Results`** + +In `toolset/utils/results.py`: +- Add `'quality'` to the end of `STORED_RUN_KEYS`. +- In `__init__`, next to the other run attributes, set `self.quality = {}` and `self.quality_environment = None`. +- Add: + +```python + def add_quality(self, name, quality, image_id=None): + '''results.json quality. and environment.quality (spec 009 R-6, R-17).''' + self.quality[name] = quality + tools = quality.get('tools') + if self.quality_environment is None and isinstance(tools, dict) \ + and not tools.get('error'): + self.quality_environment = dict(tools, image=image_id) +``` + +- In `__to_jsonable`, replace `toRet['environment'] = self.__environment()` with: + +```python + env = self.__environment() + if env is not None and self.quality_environment: + env = dict(env, quality=self.quality_environment) + toRet['environment'] = env +``` + + Before the `toRet.update(getattr(self, '_stored_run', None) or {})` line, add: + +```python + if getattr(self, 'quality', None): + toRet['quality'] = self.quality +``` + +- [ ] **Step 7: Run all tests** + +Run: `python3 -m pytest -q && ruff check toolset tests` +Expected: all pass. That includes `tests/test_noise.py` and `tests/test_schema.py`, whose `bench` side effects take 6 positional arguments; without `--quality`, no `export` keyword is passed. If `test_schema.py` lists the allowed top-level keys of `results.json`, add `"quality": dict` to that mapping, because the key is only present after a quality pass. + +- [ ] **Step 8: Commit** + +```bash +git add toolset/quality/runner.py toolset/run-tests.py toolset/utils/benchmark_config.py \ + toolset/benchmark/benchmarker.py toolset/utils/results.py tests/test_quality_runner.py tests/test_schema.py +git commit -m "feat(quality): --quality runs the untimed pass after a successful build (spec 009 R-1, R-2, R-6, R-17) + +Co-Authored-By: Claude Opus 5.5 (1M context) " +``` + +--- + +### Task 7: "Qualidade" in `summary.md` and `quality-summary.csv` + +**Files:** +- Modify: + - `toolset/utils/summary.py`: new functions after `to_markdown`, and one hook at the end of `to_markdown`; + - `toolset/utils/results.py`: `write_summary`. +- Test: `tests/test_quality_summary.py` + +**Interfaces:** +- Consumes: the `quality.json` contract (Task 3); `results.json` `quality` (Task 6). +- Produces: `QUALITY_COLUMNS`, `quality_rows(results)`, `quality_csv(rows)`, `quality_markdown(rows)`. + +- [ ] **Step 1: Write the failing tests** + +`tests/test_quality_summary.py`: + +```python +import csv +import io +import json +import pathlib + +from toolset.quality import reduce, site +from toolset.utils import summary +from toolset.utils.results import parse_build_output + +FIX = pathlib.Path(__file__).parent / "fixtures" / "quality" +RAW_OK = pathlib.Path(__file__).parent / "fixtures" / "raw_ok.txt" + + +def _quality(): + raw = json.loads((FIX / "raw.json").read_text()) + pages = site.resolve_pages(str(FIX / "site")) + return reduce.build_quality(raw, str(FIX / "site"), pages, "post/*/index.html") + + +def _data(with_quality): + [entry] = parse_build_output(RAW_OK.read_text(), "50", "5", "1") + data = {"name": "r", "frameworks": ["hugo", "gatsby"], + "rawData": {"datarate": {"hugo": [entry]}}, + "succeeded": {"datarate": ["hugo"]}, "failed": {"datarate": ["gatsby"]}, "excluded": []} + if with_quality: + data["quality"] = {"hugo": _quality(), "gatsby": {"status": "error", "error": "RuntimeError: boom"}} + return data + + +def test_quality_rows(): + rows = summary.quality_rows(_data(True)) + assert [r["framework"] for r in rows] == ["gatsby", "hugo"] + hugo = rows[1] + assert hugo["performance"] == 0.98 and hugo["lcpMs"] == 800 + assert hugo["postJsBytes"] == 0 and hugo["jsClass"] == "none" + assert hugo["htmlErrors"] == 1 and hugo["a11yViolations"] == 2 and hugo["brokenLinks"] == 2 + assert (hugo["seoPresent"], hugo["seoTotal"]) == (2, 12) + assert rows[0]["status"] == "error" and rows[0]["performance"] is None + + +def test_quality_markdown_section(): + lines = summary.quality_markdown(summary.quality_rows(_data(True))) + text = "\n".join(lines) + assert lines[0] == "## Qualidade" + assert "| hugo | 98 | 100 | 100 | 82 | 800 | 0.0 | none |" in text + assert "2/12" in text and "no meta description and no viewport" in text + assert "gatsby: RuntimeError: boom" in text + assert "Rank" not in text + + +def test_quality_csv_has_every_numeric_field(): + rows = summary.quality_rows(_data(True)) + parsed = list(csv.DictReader(io.StringIO(summary.quality_csv(rows)))) + assert parsed[1]["framework"] == "hugo" + header = list(parsed[0].keys()) + assert header[:len(summary.QUALITY_COLUMNS)] == summary.QUALITY_COLUMNS + assert "lighthouse.desktop.index.scores.seo.median" in header + assert "weight.brotliBytes" in header + + +def test_quality_never_changes_timing_summary(): + without, with_q = _data(False), _data(True) + assert summary.build_rows(with_q, {}) == summary.build_rows(without, {}) + md_without = summary.to_markdown(summary.build_rows(without, {}), without) + md_with = summary.to_markdown(summary.build_rows(with_q, {}), with_q) + assert md_with.startswith(md_without.rstrip("\n")) + assert "## Qualidade" in md_with and "## Qualidade" not in md_without +``` + +- [ ] **Step 2: Run them and see them fail** + +Run: `python3 -m pytest -q tests/test_quality_summary.py` +Expected: FAIL with `AttributeError: module 'toolset.utils.summary' has no attribute 'quality_rows'`. + +- [ ] **Step 3: Implement in `toolset/utils/summary.py`** + +Add after `to_markdown`: + +```python +QUALITY_COLUMNS = ['framework', 'status', 'performance', 'accessibility', 'bestPractices', 'seo', + 'lcpMs', 'postJsBytes', 'jsClass', 'gzipBytes', 'htmlErrors', + 'a11yViolations', 'brokenLinks', 'seoPresent', 'seoTotal'] +QUALITY_NOTE = ('The SF 005 templates carry no meta description and no viewport, so every ' + 'generator loses the same Lighthouse SEO points; that is not a difference ' + 'between generators.') + + +def _get(data, *path): + for key in path: + if not isinstance(data, dict): + return None + data = data.get(key) + return data + + +def _sum_pages(q, section, key): + pages = _get(q, section, 'pages') or {} + values = [p.get(key) for p in pages.values() if isinstance(p, dict) and p.get(key) is not None] + return sum(values) if values else None + + +def _flatten(data, prefix=''): + for key, value in (data or {}).items(): + name = prefix + str(key) + if isinstance(value, dict): + yield from _flatten(value, name + '.') + elif isinstance(value, (int, float)) and not isinstance(value, bool): + yield name, value + + +def quality_rows(results): + '''One row per generator with a quality pass (spec 009 R-18), sorted by name.''' + rows = [] + for name in sorted(results.get('quality') or {}): + q = results['quality'][name] + post = _get(q, 'lighthouse', 'mobile', 'post') or {} + rows.append({ + 'framework': name, + 'status': q.get('status'), + 'error': q.get('error'), + 'performance': _get(post, 'scores', 'performance', 'median'), + 'accessibility': _get(post, 'scores', 'accessibility', 'median'), + 'bestPractices': _get(post, 'scores', 'best-practices', 'median'), + 'seo': _get(post, 'scores', 'seo', 'median'), + 'lcpMs': _get(post, 'metrics', 'lcp', 'median'), + 'postJsBytes': _get(q, 'js', 'pages', 'post'), + 'jsClass': _get(q, 'js', 'jsClass'), + 'gzipBytes': _get(q, 'weight', 'gzipBytes'), + 'htmlErrors': _sum_pages(q, 'html', 'errors'), + 'a11yViolations': _sum_pages(q, 'a11y', 'total'), + 'brokenLinks': _get(q, 'links', 'broken'), + 'seoPresent': _get(q, 'seo', 'present', 'post'), + 'seoTotal': _get(q, 'seo', 'total'), + 'numeric': dict(_flatten({k: v for k, v in q.items() if k != 'tools'})), + }) + return rows + + +def quality_csv(rows): + extra = sorted({k for r in rows for k in r['numeric']}) + out = io.StringIO() + writer = csv.writer(out, lineterminator='\n') + writer.writerow(QUALITY_COLUMNS + extra) + for r in rows: + writer.writerow([_blank(r[c]) for c in QUALITY_COLUMNS] + + [_blank(r['numeric'].get(k)) for k in extra]) + return out.getvalue() + + +def _score(value): + return '—' if value is None else '%d' % round(value * 100) + + +def _kb(value): + return '—' if value is None else '%.1f' % (value / 1024) + + +def _int(value): + return '—' if value is None else '%d' % round(value) + + +def quality_markdown(rows): + lines = ['## Qualidade', '', + 'Untimed pass over the output of the last build (spec 009). Scores and LCP: ' + 'Lighthouse mobile preset on the post page, median of 3 runs. No combined score ' + 'and no ranking.', '', + '| Framework | Performance | Accessibility | Best practices | SEO | LCP (ms) ' + '| Post JS (KB) | JS | Gzip (KB) | HTML errors | axe violations | Broken links ' + '| SEO signals |', + '|' + '---|' * 13] + for r in rows: + signals = '—' if r['seoPresent'] is None else '%s/%s' % (r['seoPresent'], r['seoTotal']) + cells = [r['framework'], _score(r['performance']), _score(r['accessibility']), + _score(r['bestPractices']), _score(r['seo']), _int(r['lcpMs']), + _kb(r['postJsBytes']), r['jsClass'] or '—', _kb(r['gzipBytes']), + _int(r['htmlErrors']), _int(r['a11yViolations']), _int(r['brokenLinks']), signals] + lines.append('| ' + ' | '.join(str(c) for c in cells) + ' |') + lines += ['', '- ' + QUALITY_NOTE] + for r in rows: + if r['status'] == 'error': + lines.append('- %s: %s' % (r['framework'], r['error'])) + return lines +``` + +Check that `summary.py` already imports `csv` and `io`, which `to_csv` needs; add them if not. Check that `_blank` maps `None` to `''`; if its behavior differs, use `'' if v is None else v` inline. At the end of `to_markdown`, replace `return '\n'.join(lines) + '\n'` with: + +```python + if meta.get('quality'): + lines += [''] + quality_markdown(quality_rows(meta)) + return '\n'.join(lines) + '\n' +``` + +- [ ] **Step 4: Write `quality-summary.csv` in `Results.write_summary`** + +In `toolset/utils/results.py` `write_summary`, after the `summary.md` write: + +```python + if data.get('quality'): + with open(os.path.join(self.directory, 'quality-summary.csv'), 'w', newline='') as f: + f.write(summary.quality_csv(summary.quality_rows(data))) +``` + +- [ ] **Step 5: Run all tests** + +Run: `python3 -m pytest -q && ruff check toolset tests` +Expected: all pass. + +- [ ] **Step 6: Commit** + +```bash +git add toolset/utils/summary.py toolset/utils/results.py tests/test_quality_summary.py +git commit -m "feat(quality): Qualidade section in summary.md and quality-summary.csv (spec 009 R-18, R-19) + +Co-Authored-By: Claude Opus 5.5 (1M context) " +``` + +--- + +### Task 8: CI, the round workflow and docs + +**Files:** +- Modify: `.github/workflows/ci.yml` (two new jobs), `.github/workflows/benchmark-round.yml` (a `quality` input), `README.md` (a "Quality pass" section after "Results") +- Test: CI itself, run on the PR. + +**Interfaces:** +- Consumes: everything above. +- Produces: goal 4 of the spec. + +- [ ] **Step 1: Add the CI jobs** + +Append to `.github/workflows/ci.yml` under `jobs:`: + +```yaml + quality-image: + needs: toolset + strategy: + matrix: + runner: [ubuntu-24.04, ubuntu-24.04-arm] + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + - run: docker build -t ssgberk/quality quality + - name: Collector on the fixture site + run: | + docker run --rm --network none \ + -v "$PWD/tests/fixtures/quality/site:/fixture:ro" -v "$PWD/quality/test:/test:ro" \ + ssgberk/quality sh -c 'node /quality/run.mjs --site /fixture --pages "$(cat /test/pages.json)" --out /tmp/out \ + && node /test/check-shape.mjs /tmp/out/raw.json /test/expected-shape.json' + + quality: + needs: quality-image + strategy: + matrix: + runner: [ubuntu-24.04, ubuntu-24.04-arm] + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - run: ./ssgberk --test hugo -nf 50 -cs 5 -mr 1 --quality + - run: | + R=$(ls -td results/*/ | head -1) + jq -e '.rawData.datarate.hugo[0].mean > 0' "$R/results.json" + jq -e '.quality.hugo.status == "ok"' "$R/results.json" + for s in lighthouse seo html a11y links weight js extra; do + jq -e --arg s "$s" '.quality.hugo[$s].status == "ok"' "$R/results.json" + done + jq -e '[.quality.hugo.lighthouse.mobile[], .quality.hugo.lighthouse.desktop[] + | .scores[] | .median | select(. < 0 or . > 1)] | length == 0' "$R/results.json" + jq -e '.environment.quality.lighthouse and .environment.quality.chromium' "$R/results.json" + test -s "$R/quality-summary.csv" + grep -q '^## Qualidade' "$R/summary.md" +``` + +- [ ] **Step 2: Add the round input** + +In `.github/workflows/benchmark-round.yml`, add to `workflow_dispatch.inputs`: + +```yaml + quality: + description: Run the untimed quality pass (spec 009); with a suite, only in cell nf50-cs5 + type: boolean + default: false +``` + +In the `run` job's "Run benchmark" step, add `QUALITY: ${{ inputs.quality }}` to `env`, and change the script to: + +```bash + Q="" + if [ "$QUALITY" = "true" ]; then Q="--quality"; fi + if [ -n "$SUITE" ]; then + ./ssgberk --test "$TEST" --suite "$SUITE" --cell "$CELL" $Q + else + ./ssgberk --test "$TEST" -nf "$NF" -cs "$CS" -mr "$MR" $Q + fi +``` + +This job already runs one job per generator. A failing check is a result, so the job still succeeds (spec goal 4). + +- [ ] **Step 3: Document it** + +Add to `README.md` after the "Results" section: + +```markdown +## Quality pass + +`--quality` audits the output of the last timed build of each generator, without timing it (spec `docs/specs/009-output-quality`): + + $ ./ssgberk --test hugo -nf 50 -cs 5 --quality + +With `--suite`, the pass runs only in the cell `nf50-cs5` (suite P, cell 0). It builds the `ssgberk/quality` image from `quality/` once. That image pins Lighthouse, Playwright's Chromium, axe-core, html-validate and lychee. The image runs with no network and with the site copied in, and it audits the index, the first post and `404.html`: +- Lighthouse, mobile and desktop presets, median of 3 runs; +- SEO signals (title, lang, description, viewport, canonical, Open Graph, Twitter card, JSON-LD, robots, hreflang, headings, `alt`, plus `robots.txt`, `sitemap.xml`, feed and favicon); +- HTML validity, axe-core violations and broken internal links; +- weight on disk, gzip and brotli; +- JS bytes, and whether the post works without JS; +- files outside the reference page set. + +Results go to `results//quality//quality.json`, to `quality` in `results.json`, to the "Qualidade" section of `summary.md` and to `quality-summary.csv`. They have no combined score and no ranking. A failing check is a result and never fails the run. +``` + +- [ ] **Step 4: Run all tests** + +Run: `python3 -m pytest -q && ruff check toolset tests`, then validate the workflow files with `python3 -c "import yaml,sys; [yaml.safe_load(open(f)) for f in sys.argv[1:]]" .github/workflows/ci.yml .github/workflows/benchmark-round.yml`. If PyYAML is not installed, use `ruby -ryaml -e 'ARGV.each { |f| YAML.load_file(f) }' .github/workflows/*.yml`. +Expected: all pass, and both files parse. + +- [ ] **Step 5: Commit and push** + +```bash +git add .github/workflows/ci.yml .github/workflows/benchmark-round.yml README.md +git commit -m "ci(quality): quality image and hugo quality pass on amd64 and arm64; round input (spec 009 goal 4) + +Co-Authored-By: Claude Opus 5.5 (1M context) " +git push +``` + +Expected: the `quality-image` and `quality` CI jobs pass on both runners. If a Lighthouse audit id or lychee key differs on CI, fix it as described in Task 4 Step 6. From 20b221562007fdc5542de9712c279fc3abbd17cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matheus=20Bregu=C3=AAz?= Date: Sat, 10 Oct 2026 14:43:56 -0300 Subject: [PATCH 03/15] feat(quality): site helpers for the output quality pass (spec 009) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/fixtures/quality/site/404.html | 11 ++ .../fixtures/quality/site/assets/ssgberk.css | 1 + .../fixtures/quality/site/assets/ssgberk.png | Bin 0 -> 69 bytes tests/fixtures/quality/site/feed.xml | 2 + tests/fixtures/quality/site/index.html | 13 ++ .../quality/site/post/hello/index.html | 16 ++ tests/fixtures/quality/site/tags/a/index.html | 2 + tests/test_quality_site.py | 112 ++++++++++++ toolset/quality/__init__.py | 0 toolset/quality/site.py | 163 ++++++++++++++++++ 10 files changed, 320 insertions(+) create mode 100644 tests/fixtures/quality/site/404.html create mode 100644 tests/fixtures/quality/site/assets/ssgberk.css create mode 100644 tests/fixtures/quality/site/assets/ssgberk.png create mode 100644 tests/fixtures/quality/site/feed.xml create mode 100644 tests/fixtures/quality/site/index.html create mode 100644 tests/fixtures/quality/site/post/hello/index.html create mode 100644 tests/fixtures/quality/site/tags/a/index.html create mode 100644 tests/test_quality_site.py create mode 100644 toolset/quality/__init__.py create mode 100644 toolset/quality/site.py diff --git a/tests/fixtures/quality/site/404.html b/tests/fixtures/quality/site/404.html new file mode 100644 index 0000000..5656812 --- /dev/null +++ b/tests/fixtures/quality/site/404.html @@ -0,0 +1,11 @@ + + +Not found + + +

    SSGBerk Reference +
    +

    Page not found

    +

    Built for the SSGBerk build-time benchmark.

    + + diff --git a/tests/fixtures/quality/site/assets/ssgberk.css b/tests/fixtures/quality/site/assets/ssgberk.css new file mode 100644 index 0000000..3ce4b15 --- /dev/null +++ b/tests/fixtures/quality/site/assets/ssgberk.css @@ -0,0 +1 @@ +body { font-family: sans-serif; margin: 0 auto; max-width: 40rem; } diff --git a/tests/fixtures/quality/site/assets/ssgberk.png b/tests/fixtures/quality/site/assets/ssgberk.png new file mode 100644 index 0000000000000000000000000000000000000000..94381b429d7f7fe87e1bade52d893ab348ae29cc GIT binary patch literal 69 zcmeAS@N?(olHy`uVBq!ia0vp^j3CUx1SBVv2j2ryJf1F&Ar*6yfBgS%&%pYR=^yWV Rw;e!n22WQ%mvv4FO#tpo5$gZ| literal 0 HcmV?d00001 diff --git a/tests/fixtures/quality/site/feed.xml b/tests/fixtures/quality/site/feed.xml new file mode 100644 index 0000000..e51b66e --- /dev/null +++ b/tests/fixtures/quality/site/feed.xml @@ -0,0 +1,2 @@ + +SSGBerk Reference diff --git a/tests/fixtures/quality/site/index.html b/tests/fixtures/quality/site/index.html new file mode 100644 index 0000000..c9439e9 --- /dev/null +++ b/tests/fixtures/quality/site/index.html @@ -0,0 +1,13 @@ + + +SSGBerk Reference + + +
    SSGBerk Reference +
    +
    +

    Built for the SSGBerk build-time benchmark.

    + + diff --git a/tests/fixtures/quality/site/post/hello/index.html b/tests/fixtures/quality/site/post/hello/index.html new file mode 100644 index 0000000..10b77e4 --- /dev/null +++ b/tests/fixtures/quality/site/post/hello/index.html @@ -0,0 +1,16 @@ + + +Hello + + +
    SSGBerk Reference +
    +
    +

    Hello

    +

    SSGBerk

    +
    • a
    • b
    • c
    +

    First paragraph of the post.

    Second paragraph.

    +
    +

    Built for the SSGBerk build-time benchmark.

    + + diff --git a/tests/fixtures/quality/site/tags/a/index.html b/tests/fixtures/quality/site/tags/a/index.html new file mode 100644 index 0000000..ca28365 --- /dev/null +++ b/tests/fixtures/quality/site/tags/a/index.html @@ -0,0 +1,2 @@ + +Tag a

    a

    diff --git a/tests/test_quality_site.py b/tests/test_quality_site.py new file mode 100644 index 0000000..42bd1ac --- /dev/null +++ b/tests/test_quality_site.py @@ -0,0 +1,112 @@ +import io +import json +import pathlib +import tarfile + +import pytest + +from toolset.quality import site + +SITE = pathlib.Path(__file__).parent / "fixtures" / "quality" / "site" + + +def test_generator_config(tmp_path): + (tmp_path / "benchmark_config.json").write_text(json.dumps( + {"framework": "hugo", "config": [{"output_folder": "public", "output_glob": "post/*/index.html"}]})) + assert site.generator_config(str(tmp_path)) == ("public", "post/*/index.html") + + +def _tar(entries): + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w") as tar: + for name, data in entries: + info = tarfile.TarInfo(name) + if data is None: + info.type = tarfile.DIRTYPE + tar.addfile(info) + else: + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + return buf.getvalue() + + +def test_extract_strips_top_dir(tmp_path): + tar_path = tmp_path / "site.tar" + tar_path.write_bytes(_tar([("public", None), ("public/index.html", b"

    i

    "), + ("public/post/a/index.html", b"

    a

    ")])) + dest = tmp_path / "site" + assert site.extract(str(tar_path), str(dest)) == str(dest) + assert (dest / "index.html").read_bytes() == b"

    i

    " + assert (dest / "post" / "a" / "index.html").exists() + + +def test_extract_rejects_path_traversal(tmp_path): + tar_path = tmp_path / "site.tar" + tar_path.write_bytes(_tar([("public/../../evil.html", b"x")])) + with pytest.raises(tarfile.TarError): + site.extract(str(tar_path), str(tmp_path / "site")) + assert not (tmp_path.parent / "evil.html").exists() + + +def test_resolve_pages_fixture(): + assert site.resolve_pages(str(SITE)) == { + "index": {"url": "/", "file": "index.html"}, + "post": {"url": "/post/hello/", "file": "post/hello/index.html"}, + "404": {"url": "/404.html", "file": "404.html"}, + } + + +@pytest.mark.parametrize("href,url,file", [ + ("/post/a/", "/post/a/", "post/a/index.html"), + ("post/a/", "/post/a/", "post/a/index.html"), + ("./post/a/index.html", "/post/a/index.html", "post/a/index.html"), + ("/post/a?x=1#t", "/post/a", "post/a/index.html"), + ("/post/a", "/post/a", "post/a/index.html"), + ("/post/b", "/post/b", "post/b.html"), +]) +def test_resolve_pages_variants(tmp_path, href, url, file): + (tmp_path / "post" / "a").mkdir(parents=True) + (tmp_path / "post" / "a" / "index.html").write_text("a") + (tmp_path / "post" / "b.html").write_text("b") + (tmp_path / "index.html").write_text( + '' % href) + pages = site.resolve_pages(str(tmp_path)) + assert pages["post"] == {"url": url, "file": file} + + +def test_resolve_pages_without_post_link(tmp_path): + (tmp_path / "index.html").write_text('x') + with pytest.raises(ValueError, match="post-item"): + site.resolve_pages(str(tmp_path)) + + +def test_resolve_pages_link_to_missing_file(tmp_path): + (tmp_path / "index.html").write_text('
  • g
    • ') + with pytest.raises(ValueError, match="no file"): + site.resolve_pages(str(tmp_path)) + + +def test_list_files_is_sorted_posix(): + files = site.list_files(str(SITE)) + assert files == sorted(files) and "post/hello/index.html" in files + + +def test_extra_output_fixture(): + extra = site.extra_output(str(SITE), "post/*/index.html") + assert extra["files"] == 2 + assert set(extra["byExtension"]) == {".html", ".xml"} + assert extra["byExtension"][".html"]["files"] == 1 + assert extra["bytes"] == sum(e["bytes"] for e in extra["byExtension"].values()) + + +def test_works_without_js(): + post = "post/hello/index.html" + rendered = {"title": "Hello", "firstParagraph": "First paragraph of the post."} + assert site.works_without_js(str(SITE), post, rendered) is True + assert site.works_without_js(str(SITE), post, {"title": "Hello", "firstParagraph": "Only after JS"}) is False + assert site.works_without_js(str(SITE), post, None) is None + assert site.works_without_js(str(SITE), post, {"error": "boom"}) is None + + +def test_text_of_collapses_whitespace(): + assert site.text_of("

    a\n b

    ") == "a b" diff --git a/toolset/quality/__init__.py b/toolset/quality/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/toolset/quality/site.py b/toolset/quality/site.py new file mode 100644 index 0000000..195cc1f --- /dev/null +++ b/toolset/quality/site.py @@ -0,0 +1,163 @@ +''' +Pages and files of a generated site (spec 009 R-3, R-7, R-14, R-15). +''' +import fnmatch +import json +import os +import posixpath +import re +import tarfile +from html.parser import HTMLParser +from urllib.parse import unquote, urlsplit + +# SF 005 reference assets: every generator ships them unchanged (R-13, R-15) +REFERENCE_ASSETS = ('assets/ssgberk.css', 'assets/ssgberk.png') + + +def generator_config(test_directory): + '''(output_folder, output_glob) of benchmark_config.json config[0].''' + with open(os.path.join(test_directory, 'benchmark_config.json')) as f: + config = json.load(f)['config'][0] + return config['output_folder'], config['output_glob'] + + +def extract(tar_path, dest): + ''' + Extracts the archive DockerHelper.benchmark exported (a single top + directory named after output_folder) into dest, without that directory. + ''' + os.makedirs(dest, exist_ok=True) + with tarfile.open(tar_path) as tar: + members = [] + for member in tar.getmembers(): + parts = member.name.split('/', 1) + if len(parts) < 2 or not parts[1]: + continue + member.name = parts[1] + members.append(member) + # filter='data' rejects absolute paths, '..' and links out of dest + tar.extractall(dest, members=members, filter='data') + return dest + + +def url_to_file(site_dir, url): + '''The file a static server would return for url, or None.''' + p = unquote(urlsplit(url).path).lstrip('/') + if p == '' or p.endswith('/'): + candidates = [posixpath.join(p, 'index.html')] + else: + candidates = [p, p + '/index.html', p + '.html'] + for c in candidates: + if os.path.isfile(os.path.join(site_dir, c)): + return posixpath.normpath(c) + return None + + +class _PostLink(HTMLParser): + '''First a[href] after the first element whose class list has post-item.''' + + def __init__(self): + super().__init__(convert_charrefs=True) + self.in_item = False + self.href = None + + def handle_starttag(self, tag, attrs): + a = {k: (v or '') for k, v in attrs} + if 'post-item' in a.get('class', '').split(): + self.in_item = True + if tag == 'a' and self.in_item and self.href is None and a.get('href'): + self.href = a['href'] + + +def _read(path): + with open(path, encoding='utf-8', errors='replace') as f: + return f.read() + + +def resolve_pages(site_dir): + '''The three audited pages (R-7): index, first post link, 404.''' + parser = _PostLink() + parser.feed(_read(os.path.join(site_dir, 'index.html'))) + if not parser.href: + raise ValueError('no a[href] inside .post-item on the index') + path = urlsplit(parser.href).path + url = posixpath.normpath(posixpath.join('/', path)) + if path.endswith('/') and url != '/': + url += '/' + post_file = url_to_file(site_dir, url) + if post_file is None: + raise ValueError('post link %s matches no file' % parser.href) + return { + 'index': {'url': '/', 'file': 'index.html'}, + 'post': {'url': url, 'file': post_file}, + '404': {'url': '/404.html', 'file': '404.html'}, + } + + +def list_files(site_dir): + out = [] + for root, _dirs, files in os.walk(site_dir): + for name in files: + rel = os.path.relpath(os.path.join(root, name), site_dir) + out.append(rel.replace(os.sep, '/')) + return sorted(out) + + +def extra_output(site_dir, output_glob): + '''R-15: files outside the SF 005 page set, by extension.''' + by_ext = {} + total_files = total_bytes = 0 + for f in list_files(site_dir): + if f in ('index.html', '404.html') or f in REFERENCE_ASSETS \ + or fnmatch.fnmatch(f, output_glob): + continue + size = os.path.getsize(os.path.join(site_dir, f)) + ext = os.path.splitext(f)[1].lower() or '(none)' + entry = by_ext.setdefault(ext, {'files': 0, 'bytes': 0}) + entry['files'] += 1 + entry['bytes'] += size + total_files += 1 + total_bytes += size + return {'files': total_files, 'bytes': total_bytes, + 'byExtension': dict(sorted(by_ext.items()))} + + +class _Text(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.parts = [] + self.skip = 0 + + def handle_starttag(self, tag, attrs): + if tag in ('script', 'style', 'template'): + self.skip += 1 + + def handle_endtag(self, tag): + if tag in ('script', 'style', 'template') and self.skip: + self.skip -= 1 + + def handle_data(self, data): + if not self.skip: + self.parts.append(data) + + +def text_of(html): + parser = _Text() + parser.feed(html) + parser.close() + return re.sub(r'\s+', ' ', ' '.join(parser.parts)).strip() + + +def works_without_js(site_dir, post_file, rendered): + ''' + R-14: True when the static post HTML already holds the title and first + body paragraph that the browser shows after scripts ran. + ''' + if not isinstance(rendered, dict) or 'error' in rendered: + return None + title = re.sub(r'\s+', ' ', rendered.get('title') or '').strip() + first = re.sub(r'\s+', ' ', rendered.get('firstParagraph') or '').strip() + if not title or not first: + return None + static = text_of(_read(os.path.join(site_dir, post_file))) + return title in static and first in static From c1a72d864832d5c863434bc33e4336faba023d05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matheus=20Bregu=C3=AAz?= Date: Sat, 10 Oct 2026 14:44:52 -0300 Subject: [PATCH 04/15] feat(quality): SEO signals per page and site (spec 009 R-9) Co-Authored-By: Claude Haiku 5.5 --- tests/test_quality_seo.py | 86 ++++++++++++++++++++ toolset/quality/seo.py | 164 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 250 insertions(+) create mode 100644 tests/test_quality_seo.py create mode 100644 toolset/quality/seo.py diff --git a/tests/test_quality_seo.py b/tests/test_quality_seo.py new file mode 100644 index 0000000..48154d1 --- /dev/null +++ b/tests/test_quality_seo.py @@ -0,0 +1,86 @@ +import pathlib + +from toolset.quality import seo, site + +SITE = pathlib.Path(__file__).parent / "fixtures" / "quality" / "site" + +RICH = """ +Post title + + + + + + + + + + +

    T

    S

    a""" + + +def test_reference_page_has_only_title_and_lang(): + s = seo.page_signals((SITE / "post" / "hello" / "index.html").read_text()) + assert s["title"] is True and s["titleLength"] == 5 and s["lang"] == "en" + assert s["description"] is False and s["viewport"] is False and s["canonical"] is False + assert s["jsonLd"] is False and s["jsonLdValid"] is None and s["robotsMeta"] is None + assert s["h1"] == 1 and s["headingSkip"] is False and s["imgWithoutAlt"] == 0 + assert s["present"] == 2 + + +def test_rich_page_has_every_signal(): + s = seo.page_signals(RICH) + assert s["present"] == len(seo.PAGE_SIGNALS) == 12 + assert s["descriptionLength"] == len("A post about things") + assert s["robotsMeta"] == "index,follow" and s["hreflang"] == 2 + assert s["jsonLdBlocks"] == 1 and s["jsonLdValid"] is True + assert s["faviconLink"] is True and s["imgWithoutAlt"] == 0 + + +def test_broken_json_ld(): + s = seo.page_signals('') + assert s["jsonLd"] is True and s["jsonLdValid"] is False + + +def test_heading_skip_images_and_empty_links(): + html = ('

    a

    b

    ' + '' + 'logo no href') + s = seo.page_signals(html) + assert s["headingSkip"] is True + assert s["imgWithoutAlt"] == 1 + assert s["emptyLinks"] == 2 + + +def test_page_signals_malformed_html(): + html = b"x

    unclosed \xff\xfe".decode("utf-8", "replace") + s = seo.page_signals(html) + assert s["title"] is True and s["lang"] == "en" + + +def test_site_signals(tmp_path): + (tmp_path / "robots.txt").write_text("User-agent: *\n") + (tmp_path / "sitemap.xml").write_text( + '' + '/a//b/') + (tmp_path / "feed.xml").write_text("") + (tmp_path / "favicon.ico").write_bytes(b"\x00") + s = seo.site_signals(str(tmp_path), []) + assert s == {"robotsTxt": True, "sitemap": {"present": True, "valid": True, "urls": 2}, + "feed": True, "favicon": True} + + +def test_site_signals_invalid_sitemap_and_link_favicon(tmp_path): + (tmp_path / "sitemap.xml").write_text("") + s = seo.site_signals(str(tmp_path), [{"faviconLink": True}]) + assert s["sitemap"] == {"present": True, "valid": False, "urls": None} + assert s["favicon"] is True and s["robotsTxt"] is False and s["feed"] is False + + +def test_seo_section_fixture(): + pages = site.resolve_pages(str(SITE)) + section = seo.seo_section(str(SITE), pages) + assert section["total"] == 12 + assert section["present"] == {"index": 2, "post": 2, "404": 2} + assert section["site"]["feed"] is True and section["site"]["sitemap"]["present"] is False + assert set(section["pages"]) == {"index", "post", "404"} diff --git a/toolset/quality/seo.py b/toolset/quality/seo.py new file mode 100644 index 0000000..a605b2b --- /dev/null +++ b/toolset/quality/seo.py @@ -0,0 +1,164 @@ +''' +SEO signals of a generated page and site (spec 009 R-9). Reported, not scored. +''' +import json +import os +import re +import xml.etree.ElementTree as ET +from html.parser import HTMLParser + +from toolset.quality import site + +# The per-page signals counted in seo.present (out of len(PAGE_SIGNALS)) +PAGE_SIGNALS = ('title', 'lang', 'description', 'viewport', 'canonical', 'ogTitle', + 'ogDescription', 'ogType', 'ogUrl', 'twitterCard', 'jsonLd', 'robotsMeta') +META = {('name', 'description'): 'description', ('name', 'viewport'): 'viewport', + ('property', 'og:title'): 'ogTitle', ('property', 'og:description'): 'ogDescription', + ('property', 'og:type'): 'ogType', ('property', 'og:url'): 'ogUrl', + ('name', 'twitter:card'): 'twitterCard', ('name', 'robots'): 'robotsMeta'} +FEEDS = ('feed.xml', 'atom.xml', 'rss.xml', 'index.xml', 'feed/index.xml', 'rss/index.xml') +FAVICONS = ('favicon.ico', 'favicon.svg', 'favicon.png') + + +class _Collector(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.lang = None + self.title = None + self.meta = {} + self.canonical = None + self.hreflang = 0 + self.favicon_link = False + self.jsonld = [] + self.headings = [] + self.img_without_alt = 0 + self.empty_links = 0 + self._capture = None # 'title' or 'jsonld' + self._buf = [] + self._links = [] # one bool per open : has an accessible name + + def handle_starttag(self, tag, attrs): + a = {k: (v or '') for k, v in attrs} + if tag == 'html' and a.get('lang'): + self.lang = a['lang'] + elif tag == 'title' and self.title is None: + self._capture, self._buf = 'title', [] + elif tag == 'meta': + for (attr, value), key in META.items(): + if a.get(attr, '').lower() == value: + self.meta[key] = a.get('content', '') + elif tag == 'link': + rel = a.get('rel', '').lower().split() + if 'canonical' in rel and a.get('href'): + self.canonical = a['href'] + if 'alternate' in rel and 'hreflang' in a: + self.hreflang += 1 + if 'icon' in rel: + self.favicon_link = True + elif tag == 'script' and a.get('type', '').lower() == 'application/ld+json': + self._capture, self._buf = 'jsonld', [] + elif re.fullmatch(r'h[1-6]', tag): + self.headings.append(int(tag[1])) + elif tag == 'img': + if 'alt' not in a: + self.img_without_alt += 1 + elif a['alt'].strip() and self._links: + self._links[-1] = True + elif tag == 'a' and 'href' in a: + self._links.append(bool(a.get('aria-label', '').strip() or a.get('title', '').strip())) + + def handle_endtag(self, tag): + if tag == 'title' and self._capture == 'title': + self.title = ''.join(self._buf).strip() + self._capture = None + elif tag == 'script' and self._capture == 'jsonld': + self.jsonld.append(''.join(self._buf)) + self._capture = None + elif tag == 'a' and self._links: + if not self._links.pop(): + self.empty_links += 1 + + def handle_data(self, data): + if self._capture: + self._buf.append(data) + if self._links and data.strip(): + self._links[-1] = True + + +def _skips(levels): + return any(b > a + 1 for a, b in zip(levels, levels[1:])) + + +def page_signals(html): + c = _Collector() + c.feed(html) + c.close() + valid = [] + for block in c.jsonld: + try: + json.loads(block) + valid.append(True) + except ValueError: + valid.append(False) + s = { + 'title': bool(c.title), + 'titleLength': len(c.title or ''), + 'lang': c.lang, + 'description': bool(c.meta.get('description')), + 'descriptionLength': len(c.meta.get('description', '')), + 'viewport': 'viewport' in c.meta, + 'canonical': bool(c.canonical), + 'ogTitle': 'ogTitle' in c.meta, + 'ogDescription': 'ogDescription' in c.meta, + 'ogType': 'ogType' in c.meta, + 'ogUrl': 'ogUrl' in c.meta, + 'twitterCard': 'twitterCard' in c.meta, + 'jsonLd': bool(c.jsonld), + 'jsonLdBlocks': len(c.jsonld), + 'jsonLdValid': all(valid) if valid else None, + 'robotsMeta': c.meta.get('robotsMeta'), + 'hreflang': c.hreflang, + 'h1': c.headings.count(1), + 'headingSkip': _skips(c.headings), + 'imgWithoutAlt': c.img_without_alt, + 'emptyLinks': c.empty_links, + 'faviconLink': c.favicon_link, + } + s['present'] = sum(1 for k in PAGE_SIGNALS if s[k] not in (False, None, '')) + return s + + +def _sitemap(site_dir): + path = os.path.join(site_dir, 'sitemap.xml') + if not os.path.isfile(path): + return {'present': False, 'valid': None, 'urls': None} + try: + root = ET.parse(path).getroot() + except ET.ParseError: + return {'present': True, 'valid': False, 'urls': None} + if not root.tag.endswith(('urlset', 'sitemapindex')): + return {'present': True, 'valid': False, 'urls': None} + return {'present': True, 'valid': True, + 'urls': sum(1 for el in root.iter() if el.tag.endswith('loc'))} + + +def site_signals(site_dir, page_signals_list): + exists = lambda rel: os.path.isfile(os.path.join(site_dir, rel)) # noqa: E731 + return { + 'robotsTxt': exists('robots.txt'), + 'sitemap': _sitemap(site_dir), + 'feed': any(exists(f) for f in FEEDS), + 'favicon': any(exists(f) for f in FAVICONS) + or any(s.get('faviconLink') for s in page_signals_list), + } + + +def seo_section(site_dir, pages): + signals = {key: page_signals(site._read(os.path.join(site_dir, page['file']))) + for key, page in pages.items()} + return { + 'total': len(PAGE_SIGNALS), + 'pages': signals, + 'present': {key: s['present'] for key, s in signals.items()}, + 'site': site_signals(site_dir, list(signals.values())), + } From 00a499707d71fe2c380c0ff027244bb3fd31adf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matheus=20Bregu=C3=AAz?= Date: Sat, 10 Oct 2026 14:46:02 -0300 Subject: [PATCH 05/15] fix(quality): text_of keeps inline punctuation (spec 009 R-14) Join text nodes without a separator, as textContent does, so "

    See x.

    " yields "See x." instead of "See x .". Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_quality_site.py | 13 +++++++++++++ toolset/quality/site.py | 4 +++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/test_quality_site.py b/tests/test_quality_site.py index 42bd1ac..038b1cc 100644 --- a/tests/test_quality_site.py +++ b/tests/test_quality_site.py @@ -110,3 +110,16 @@ def test_works_without_js(): def test_text_of_collapses_whitespace(): assert site.text_of("

    a\n b

    ") == "a b" + + +def test_text_of_keeps_inline_punctuation(): + assert site.text_of("

    See x.

    ") == "See x." + + +def test_works_without_js_inline_markup(tmp_path): + (tmp_path / "post.html").write_text( + 'T' + '

    T

    See x.

    ' + '') + rendered = {"title": "T", "firstParagraph": "See x."} + assert site.works_without_js(str(tmp_path), "post.html", rendered) is True diff --git a/toolset/quality/site.py b/toolset/quality/site.py index 195cc1f..a82a4ae 100644 --- a/toolset/quality/site.py +++ b/toolset/quality/site.py @@ -145,7 +145,9 @@ def text_of(html): parser = _Text() parser.feed(html) parser.close() - return re.sub(r'\s+', ' ', ' '.join(parser.parts)).strip() + # join without a separator, as textContent does, so inline markup does not + # insert spaces ("See x." -> "See x."); whitespace is collapsed after + return re.sub(r'\s+', ' ', ''.join(parser.parts)).strip() def works_without_js(site_dir, post_file, rendered): From fba51cd5ce9b67f3f51ebe1f1c94bc3144dd631b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matheus=20Bregu=C3=AAz?= Date: Sat, 10 Oct 2026 14:47:07 -0300 Subject: [PATCH 06/15] feat(quality): reduce collector output into quality.json (spec 009) Co-Authored-By: Claude Haiku 5.5 --- tests/fixtures/quality/raw.json | 606 ++++++++++++++++++++++++++++++++ tests/test_quality_reduce.py | 112 ++++++ toolset/quality/reduce.py | 175 +++++++++ 3 files changed, 893 insertions(+) create mode 100644 tests/fixtures/quality/raw.json create mode 100644 tests/test_quality_reduce.py create mode 100644 toolset/quality/reduce.py diff --git a/tests/fixtures/quality/raw.json b/tests/fixtures/quality/raw.json new file mode 100644 index 0000000..18b90f4 --- /dev/null +++ b/tests/fixtures/quality/raw.json @@ -0,0 +1,606 @@ +{ + "tools": { + "node": "v24.21.0", + "lighthouse": "1.0.0", + "playwright": "1.0.0", + "chromium": "1.0", + "axeCore": "1.0.0", + "htmlValidate": "1.0.0", + "chromeLauncher": "1.0.0", + "lychee": "1.0.0" + }, + "pages": { + "index": { + "url": "/", + "file": "index.html" + }, + "post": { + "url": "/post/hello/", + "file": "post/hello/index.html" + }, + "404": { + "url": "/404.html", + "file": "404.html" + } + }, + "lighthouse": { + "mobile": { + "index": [ + { + "scores": { + "performance": 0.97, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 900, + "lcp": 900, + "tbt": 0, + "cls": 0, + "si": 900 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "a" + ] + }, + { + "scores": { + "performance": 0.99, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 700, + "lcp": 700, + "tbt": 0, + "cls": 0, + "si": 700 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "c" + ] + }, + { + "scores": { + "performance": 0.98, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 800, + "lcp": 800, + "tbt": 0, + "cls": 0, + "si": 800 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "b" + ] + } + ], + "post": [ + { + "scores": { + "performance": 0.97, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 900, + "lcp": 900, + "tbt": 0, + "cls": 0, + "si": 900 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "a" + ] + }, + { + "scores": { + "performance": 0.99, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 700, + "lcp": 700, + "tbt": 0, + "cls": 0, + "si": 700 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "c" + ] + }, + { + "scores": { + "performance": 0.98, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 800, + "lcp": 800, + "tbt": 0, + "cls": 0, + "si": 800 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "b" + ] + } + ], + "404": [ + { + "scores": { + "performance": 0.97, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 900, + "lcp": 900, + "tbt": 0, + "cls": 0, + "si": 900 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "a" + ] + }, + { + "scores": { + "performance": 0.99, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 700, + "lcp": 700, + "tbt": 0, + "cls": 0, + "si": 700 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "c" + ] + }, + { + "scores": { + "performance": 0.98, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 800, + "lcp": 800, + "tbt": 0, + "cls": 0, + "si": 800 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "b" + ] + } + ] + }, + "desktop": { + "index": [ + { + "scores": { + "performance": 0.97, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 900, + "lcp": 900, + "tbt": 0, + "cls": 0, + "si": 900 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "a" + ] + }, + { + "scores": { + "performance": 0.99, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 700, + "lcp": 700, + "tbt": 0, + "cls": 0, + "si": 700 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "c" + ] + }, + { + "scores": { + "performance": 0.98, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 800, + "lcp": 800, + "tbt": 0, + "cls": 0, + "si": 800 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "b" + ] + } + ], + "post": [ + { + "scores": { + "performance": 0.97, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 900, + "lcp": 900, + "tbt": 0, + "cls": 0, + "si": 900 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "a" + ] + }, + { + "scores": { + "performance": 0.99, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 700, + "lcp": 700, + "tbt": 0, + "cls": 0, + "si": 700 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "c" + ] + }, + { + "scores": { + "performance": 0.98, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 800, + "lcp": 800, + "tbt": 0, + "cls": 0, + "si": 800 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "b" + ] + } + ], + "404": [ + { + "scores": { + "performance": 0.97, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 900, + "lcp": 900, + "tbt": 0, + "cls": 0, + "si": 900 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "a" + ] + }, + { + "scores": { + "performance": 0.99, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 700, + "lcp": 700, + "tbt": 0, + "cls": 0, + "si": 700 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "c" + ] + }, + { + "scores": { + "performance": 0.98, + "accessibility": 1, + "best-practices": 1, + "seo": 0.82 + }, + "metrics": { + "fcp": 800, + "lcp": 800, + "tbt": 0, + "cls": 0, + "si": 800 + }, + "resources": { + "totalBytes": 2400, + "jsBytes": 0, + "cssBytes": 300, + "requests": 2, + "domSize": 40 + }, + "failedAudits": [ + "b" + ] + } + ] + } + }, + "axe": { + "index": { + "violations": [] + }, + "post": { + "violations": [ + { + "id": "color-contrast", + "impact": "serious", + "nodes": 3 + }, + { + "id": "region", + "impact": "moderate", + "nodes": 1 + } + ] + }, + "404": { + "violations": [] + } + }, + "rendered": { + "post": { + "title": "Hello", + "firstParagraph": "First paragraph of the post." + } + }, + "html": { + "index": { + "messages": [] + }, + "post": { + "messages": [ + { + "ruleId": "no-inline-style", + "severity": 2 + }, + { + "ruleId": "prefer-native-element", + "severity": 1 + } + ] + }, + "404": { + "messages": [] + } + }, + "links": { + "report": { + "fail_map": { + "tests/fixtures/quality/site/index.html": [ + { + "url": "file:///x/missing.css", + "status": { + "text": "Cannot find file" + } + }, + { + "url": "file:///x/post/hello/#nope", + "status": { + "text": "Cannot find fragment" + } + } + ] + } + } + }, + "files": [ + { + "path": "404.html", + "bytes": 559, + "gzip": 319, + "br": 318 + }, + { + "path": "assets/ssgberk.css", + "bytes": 68, + "gzip": 85, + "br": 84 + }, + { + "path": "assets/ssgberk.png", + "bytes": 69, + "gzip": 82, + "br": 81 + }, + { + "path": "feed.xml", + "bytes": 116, + "gzip": 114, + "br": 113 + }, + { + "path": "index.html", + "bytes": 705, + "gzip": 365, + "br": 364 + }, + { + "path": "post/hello/index.html", + "bytes": 927, + "gzip": 448, + "br": 447 + }, + { + "path": "tags/a/index.html", + "bytes": 116, + "gzip": 111, + "br": 110 + } + ] +} diff --git a/tests/test_quality_reduce.py b/tests/test_quality_reduce.py new file mode 100644 index 0000000..2b2cc17 --- /dev/null +++ b/tests/test_quality_reduce.py @@ -0,0 +1,112 @@ +import json +import pathlib + +from toolset.quality import reduce, site + +FIX = pathlib.Path(__file__).parent / "fixtures" / "quality" +SITE = FIX / "site" +GLOB = "post/*/index.html" + + +def _raw(): + return json.loads((FIX / "raw.json").read_text()) + + +def test_spread(): + assert reduce.spread([3, 1, 2]) == {"median": 2, "min": 1, "max": 3} + assert reduce.spread([None, 4]) == {"median": 4, "min": 4, "max": 4} + assert reduce.spread([None]) is None + + +def test_reduce_lighthouse_median_and_median_run_audits(): + lh = reduce.reduce_lighthouse(_raw()["lighthouse"]) + post = lh["mobile"]["post"] + assert post["status"] == "ok" and post["runs"] == 3 + assert post["scores"]["performance"] == {"median": 0.98, "min": 0.97, "max": 0.99} + assert post["metrics"]["lcp"] == {"median": 800, "min": 700, "max": 900} + # the run whose performance is the median (0.98) failed audit "b" + assert post["failedAudits"] == ["b"] + assert set(lh) == {"mobile", "desktop"} + + +def test_reduce_lighthouse_page_errors(): + raw = {"mobile": {"index": {"error": "chrome died"}, + "post": [{"error": "timeout"}, {"error": "timeout"}], + "404": [{"error": "x"}, _raw()["lighthouse"]["mobile"]["post"][0]]}, + "desktop": {}} + lh = reduce.reduce_lighthouse(raw) + assert lh["mobile"]["index"] == {"status": "error", "error": "chrome died"} + assert lh["mobile"]["post"] == {"status": "error", "error": "timeout"} + assert lh["mobile"]["404"]["status"] == "ok" and lh["mobile"]["404"]["runs"] == 1 + + +def test_reduce_axe_counts_by_impact(): + a = reduce.reduce_axe(_raw()["axe"]) + assert a["post"] == {"violations": {"critical": 0, "serious": 1, "moderate": 1, "minor": 0}, + "total": 2, "rules": ["color-contrast", "region"]} + assert a["index"]["total"] == 0 + assert reduce.reduce_axe({"index": {"error": "boom"}})["index"] == {"status": "error", "error": "boom"} + + +def test_reduce_html_counts(): + h = reduce.reduce_html(_raw()["html"]) + assert h["post"] == {"errors": 1, "warnings": 1, "rules": ["no-inline-style", "prefer-native-element"]} + + +def test_reduce_links_kinds(): + links = reduce.reduce_links(_raw()["links"]) + assert links["broken"] == 2 + assert links["byKind"] == {"link": 0, "anchor": 1, "asset": 1} + assert links["items"][0]["status"] == "Cannot find file" + + +def test_reduce_links_caps_items(): + fail = {"s": [{"url": "/p%d/" % i, "status": {"text": "x"}} for i in range(30)]} + links = reduce.reduce_links({"report": {"fail_map": fail}}) + assert links["broken"] == 30 and len(links["items"]) == reduce.MAX_BROKEN_LISTED == 20 + + +def test_weight(): + files = [{"path": "index.html", "bytes": 100, "gzip": 50, "br": 40}, + {"path": "post/a/index.html", "bytes": 300, "gzip": 100, "br": 90}, + {"path": "post/b/index.html", "bytes": 100, "gzip": 60, "br": 50}, + {"path": "assets/ssgberk.css", "bytes": 1000, "gzip": 400, "br": 300}] + assert reduce.weight(files, GLOB) == {"bytes": 1500, "gzipBytes": 610, "brotliBytes": 480, + "bytesPerPost": 200, "generatorAddedBytes": 500} + + +def test_js_section_classes(): + lh = {"mobile": {"index": {"status": "ok", "resources": {"jsBytes": {"median": 0}}}, + "post": {"status": "ok", "resources": {"jsBytes": {"median": 0}}}}} + files = [{"path": "a.js", "bytes": 10, "gzip": 5, "br": 4}] + js = reduce.js_section(lh, files, True) + assert js == {"pages": {"index": 0, "post": 0}, "diskBytes": 10, "jsClass": "none", "worksWithoutJs": True} + lh["mobile"]["post"]["resources"]["jsBytes"]["median"] = 5000 + assert reduce.js_section(lh, files, False)["jsClass"] == "hydrated" + assert reduce.js_section({"mobile": {}}, [], None)["jsClass"] is None + + +def test_build_quality_fixture_all_ok(): + pages = site.resolve_pages(str(SITE)) + q = reduce.build_quality(_raw(), str(SITE), pages, GLOB) + assert q["status"] == "ok" + assert q["pages"] == {"index": "/", "post": "/post/hello/", "404": "/404.html"} + for section in reduce.SECTIONS: + assert q[section]["status"] == "ok", (section, q[section]) + assert q["js"]["worksWithoutJs"] is True and q["js"]["jsClass"] == "none" + assert q["extra"]["files"] == 2 and q["seo"]["present"]["post"] == 2 + + +def test_build_quality_partial_errors(): + raw = _raw() + raw["lighthouse"] = {"error": "lighthouse crashed"} + del raw["links"] + raw["files"] = {"error": "EACCES"} + pages = site.resolve_pages(str(SITE)) + q = reduce.build_quality(raw, str(SITE), pages, GLOB) + assert q["lighthouse"] == {"status": "error", "error": "RuntimeError: lighthouse crashed"} + assert q["links"]["status"] == "error" and "missing" in q["links"]["error"] + assert q["weight"]["status"] == "error" and q["js"]["status"] == "error" + for section in ("seo", "html", "a11y", "extra"): + assert q[section]["status"] == "ok" + assert q["status"] == "ok" diff --git a/toolset/quality/reduce.py b/toolset/quality/reduce.py new file mode 100644 index 0000000..8b68e5e --- /dev/null +++ b/toolset/quality/reduce.py @@ -0,0 +1,175 @@ +''' +Reduces the collector output (quality/run.mjs raw.json) into quality.json +(spec 009 R-8, R-10..R-16). Pure functions; the contracts are in plan.md. +''' +import fnmatch +import posixpath +import statistics + +from toolset.quality import seo, site + +SECTIONS = ('lighthouse', 'seo', 'html', 'a11y', 'links', 'weight', 'js', 'extra') +PRESETS = ('mobile', 'desktop') +SCORES = ('performance', 'accessibility', 'best-practices', 'seo') +METRICS = ('fcp', 'lcp', 'tbt', 'cls', 'si') +RESOURCES = ('totalBytes', 'jsBytes', 'cssBytes', 'requests', 'domSize') +IMPACTS = ('critical', 'serious', 'moderate', 'minor') +MAX_BROKEN_LISTED = 20 + + +def spread(values): + vs = [v for v in values if v is not None] + if not vs: + return None + return {'median': statistics.median(vs), 'min': min(vs), 'max': max(vs)} + + +def _is_error(part): + return isinstance(part, dict) and 'error' in part + + +def _error(part): + return {'status': 'error', 'error': part['error']} + + +def _median_run(runs): + ranked = sorted(runs, key=lambda r: (r['scores'].get('performance') is None, + r['scores'].get('performance') or 0)) + return ranked[(len(ranked) - 1) // 2] + + +def _lighthouse_page(runs): + if _is_error(runs): + return _error(runs) + good = [r for r in runs if not _is_error(r)] + if not good: + return {'status': 'error', 'error': runs[0]['error'] if runs else 'no runs'} + return { + 'status': 'ok', + 'runs': len(good), + 'scores': {k: spread([r['scores'].get(k) for r in good]) for k in SCORES}, + 'metrics': {k: spread([r['metrics'].get(k) for r in good]) for k in METRICS}, + 'resources': {k: spread([r['resources'].get(k) for r in good]) for k in RESOURCES}, + 'failedAudits': _median_run(good)['failedAudits'], + } + + +def reduce_lighthouse(raw_lh): + return {preset: {page: _lighthouse_page(runs) + for page, runs in (raw_lh.get(preset) or {}).items()} + for preset in PRESETS} + + +def reduce_axe(raw_axe): + out = {} + for page, data in raw_axe.items(): + if _is_error(data): + out[page] = _error(data) + continue + counts = {impact: 0 for impact in IMPACTS} + for v in data.get('violations') or []: + if v.get('impact') in counts: + counts[v['impact']] += 1 + out[page] = {'violations': counts, 'total': len(data.get('violations') or []), + 'rules': sorted({v['id'] for v in data.get('violations') or []})} + return out + + +def reduce_html(raw_html): + out = {} + for page, data in raw_html.items(): + if _is_error(data): + out[page] = _error(data) + continue + messages = data.get('messages') or [] + out[page] = {'errors': sum(1 for m in messages if m.get('severity') == 2), + 'warnings': sum(1 for m in messages if m.get('severity') == 1), + 'rules': sorted({m['ruleId'] for m in messages if m.get('ruleId')})} + return out + + +def _kind(url): + if '#' in url: + return 'anchor' + path = url.split('?', 1)[0] + if path.endswith('/') or path.endswith('.html') or not posixpath.splitext(path)[1]: + return 'link' + return 'asset' + + +def reduce_links(raw_links): + report = raw_links.get('report') or {} + fail_map = report.get('fail_map') or report.get('error_map') or {} + items = [] + for source, failures in sorted(fail_map.items()): + for f in failures: + status = f.get('status') + items.append({'source': source, 'url': f.get('url'), + 'status': status.get('text') if isinstance(status, dict) else status}) + by_kind = {'link': 0, 'anchor': 0, 'asset': 0} + for item in items: + by_kind[_kind(item['url'] or '')] += 1 + return {'broken': len(items), 'byKind': by_kind, 'items': items[:MAX_BROKEN_LISTED]} + + +def weight(files, output_glob): + posts = [f['bytes'] for f in files if fnmatch.fnmatch(f['path'], output_glob)] + total = sum(f['bytes'] for f in files) + reference = sum(f['bytes'] for f in files if f['path'] in site.REFERENCE_ASSETS) + return {'bytes': total, + 'gzipBytes': sum(f['gzip'] for f in files), + 'brotliBytes': sum(f['br'] for f in files), + 'bytesPerPost': round(sum(posts) / len(posts)) if posts else None, + 'generatorAddedBytes': total - reference} + + +def js_section(lighthouse, files, works): + pages = {} + for page, data in (lighthouse.get('mobile') or {}).items(): + js = (data.get('resources') or {}).get('jsBytes') if data.get('status') == 'ok' else None + pages[page] = js['median'] if js else None + known = [v for v in pages.values() if v is not None] + js_class = None if not known else ('none' if all(v == 0 for v in known) else 'hydrated') + return {'pages': pages, + 'diskBytes': sum(f['bytes'] for f in files if f['path'].endswith(('.js', '.mjs'))), + 'jsClass': js_class, + 'worksWithoutJs': works} + + +def _raw(raw, key): + part = raw.get(key) + if part is None: + raise RuntimeError('%s missing in collector output' % key) + if _is_error(part): + raise RuntimeError(part['error']) + return part + + +def _section(fn): + try: + return dict({'status': 'ok'}, **fn()) + except Exception as e: + return {'status': 'error', 'error': '%s: %s' % (type(e).__name__, e)} + + +def build_quality(raw, site_dir, pages, output_glob): + '''quality.json (R-16): one section per check, each ok or error.''' + q = {'status': 'ok', 'pages': {k: v['url'] for k, v in pages.items()}, + 'tools': raw.get('tools')} + q['lighthouse'] = _section(lambda: reduce_lighthouse(_raw(raw, 'lighthouse'))) + q['seo'] = _section(lambda: seo.seo_section(site_dir, pages)) + q['html'] = _section(lambda: {'pages': reduce_html(_raw(raw, 'html'))}) + q['a11y'] = _section(lambda: {'pages': reduce_axe(_raw(raw, 'axe'))}) + q['links'] = _section(lambda: reduce_links(_raw(raw, 'links'))) + q['weight'] = _section(lambda: weight(_raw(raw, 'files'), output_glob)) + + def js(): + if q['lighthouse']['status'] != 'ok': + raise RuntimeError('lighthouse section failed') + rendered = (raw.get('rendered') or {}).get('post') if not _is_error(raw.get('rendered')) else None + works = site.works_without_js(site_dir, pages['post']['file'], rendered) + return js_section(q['lighthouse'], _raw(raw, 'files'), works) + + q['js'] = _section(js) + q['extra'] = _section(lambda: site.extra_output(site_dir, output_glob)) + return q From 0e8911a40eb113629cce92d36491f5268fa95702 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matheus=20Bregu=C3=AAz?= Date: Sat, 10 Oct 2026 14:49:07 -0300 Subject: [PATCH 07/15] feat(quality): export build output and run the collector without network (spec 009 R-3..R-5) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_quality_docker.py | 152 +++++++++++++++++++++++++++++++++ toolset/utils/docker_helper.py | 78 ++++++++++++++++- 2 files changed, 228 insertions(+), 2 deletions(-) create mode 100644 tests/test_quality_docker.py diff --git a/tests/test_quality_docker.py b/tests/test_quality_docker.py new file mode 100644 index 0000000..945e9df --- /dev/null +++ b/tests/test_quality_docker.py @@ -0,0 +1,152 @@ +import io +import json +import tarfile +import types +from unittest import mock + +import docker +import pytest + +from toolset.utils import docker_helper + +RES = {"cpus": 4.0, "memoryBytes": 8589934592, "swap": False, "cpuset": "4-7"} + + +def _helper(): + helper = docker_helper.DockerHelper.__new__(docker_helper.DockerHelper) + helper.benchmarker = mock.Mock() + helper.server = mock.Mock() + return helper + + +def _container(exit_code=0, oom=False): + c = mock.Mock() + c.logs.return_value = iter(()) + c.wait.return_value = {"StatusCode": exit_code} + c.attrs = {"State": {"OOMKilled": oom, "ExitCode": exit_code}, + "Config": {"WorkingDir": "/opt/hugo/src"}} + c.get_archive.return_value = (iter([b"tar", b"bytes"]), {}) + return c + + +def test_benchmark_exports_output_on_success(tmp_path): + helper, c = _helper(), _container() + helper.server.containers.run.return_value = c + dest = tmp_path / "quality" / "hugo" / "site.tar" + out = helper.benchmark(types.SimpleNamespace(name="hugo"), "build.sh", {}, str(tmp_path / "raw.txt"), + RES, 60, export=("public", str(dest))) + assert out["status"] == "ok" + c.get_archive.assert_called_once_with("/opt/hugo/src/public") + assert dest.read_bytes() == b"tarbytes" + c.remove.assert_called_once_with(force=True) + + +@pytest.mark.parametrize("exit_code,oom", [(1, False), (137, True)]) +def test_benchmark_no_export_on_failure(tmp_path, exit_code, oom): + helper, c = _helper(), _container(exit_code, oom) + helper.server.containers.run.return_value = c + helper.benchmark(types.SimpleNamespace(name="hugo"), "build.sh", {}, str(tmp_path / "raw.txt"), + RES, 60, export=("public", str(tmp_path / "site.tar"))) + c.get_archive.assert_not_called() + + +def test_benchmark_export_error_keeps_status(tmp_path): + helper, c = _helper(), _container() + c.get_archive.side_effect = docker.errors.NotFound("no such path") + helper.server.containers.run.return_value = c + out = helper.benchmark(types.SimpleNamespace(name="hugo"), "build.sh", {}, str(tmp_path / "raw.txt"), + RES, 60, export=("public", str(tmp_path / "site.tar"))) + assert out == {"status": "ok", "exitCode": 0} + + +def test_benchmark_without_export_never_reads_archive(tmp_path): + helper, c = _helper(), _container() + helper.server.containers.run.return_value = c + helper.benchmark(types.SimpleNamespace(name="hugo"), "build.sh", {}, str(tmp_path / "raw.txt"), RES, 60) + c.get_archive.assert_not_called() + + +def _raw_tar(payload): + buf = io.BytesIO() + data = json.dumps(payload).encode() + with tarfile.open(fileobj=buf, mode="w") as tar: + info = tarfile.TarInfo("raw.json") + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + return buf.getvalue() + + +def test_run_quality_isolated_and_cleaned_up(tmp_path): + helper = _helper() + helper._quality_image = "sha256:q" + c = mock.Mock() + c.wait.return_value = {"StatusCode": 0} + c.logs.return_value = b"" + c.get_archive.return_value = (iter([_raw_tar({"tools": {"node": "v24"}})]), {}) + helper.server.containers.create.return_value = c + tar_path = tmp_path / "site.tar" + tar_path.write_bytes(b"site-tar") + pages = {"index": {"url": "/", "file": "index.html"}} + raw = helper.run_quality(str(tar_path), "public", pages) + assert raw == {"tools": {"node": "v24"}} + args, kwargs = helper.server.containers.create.call_args + assert args[0] == "sha256:q" + assert kwargs["network_mode"] == "none" + assert "volumes" not in kwargs and "mounts" not in kwargs + assert kwargs["command"][:4] == ["node", "/quality/run.mjs", "--site", "/work/public"] + assert json.loads(kwargs["command"][kwargs["command"].index("--pages") + 1]) == pages + c.put_archive.assert_called_once_with("/work", b"site-tar") + c.get_archive.assert_called_once_with("/work/out/raw.json") + c.remove.assert_called_once_with(force=True) + + +def test_run_quality_without_raw_json_raises_and_cleans_up(tmp_path): + helper = _helper() + helper._quality_image = "sha256:q" + c = mock.Mock() + c.wait.return_value = {"StatusCode": 1} + c.logs.return_value = b"Error: boom" + c.get_archive.side_effect = docker.errors.NotFound("missing") + helper.server.containers.create.return_value = c + (tmp_path / "site.tar").write_bytes(b"x") + with pytest.raises(RuntimeError, match="exit 1.*boom"): + helper.run_quality(str(tmp_path / "site.tar"), "public", {}) + c.remove.assert_called_once_with(force=True) + + +def test_build_quality_image_once(monkeypatch, tmp_path): + builds = [] + + class FakeAPIClient: + def __init__(self, base_url=None): + pass + + def build(self, **kw): + builds.append(kw) + return iter([{"stream": "Step 1/2 : FROM ubuntu:24.04\n"}, {"stream": "Successfully built abc\n"}]) + + monkeypatch.setattr(docker_helper.docker, "APIClient", FakeAPIClient) + helper = _helper() + helper._quality_image = None + helper.benchmarker.config = types.SimpleNamespace(fw_root=str(tmp_path), server_docker_host=None) + helper.server.images.get.return_value = types.SimpleNamespace(id="sha256:abc") + assert helper.build_quality_image() == "sha256:abc" + assert helper.build_quality_image() == "sha256:abc" + assert len(builds) == 1 + assert builds[0]["path"] == str(tmp_path / "quality") and builds[0]["tag"] == docker_helper.QUALITY_IMAGE + + +def test_build_quality_image_error(monkeypatch, tmp_path): + class FakeAPIClient: + def __init__(self, base_url=None): + pass + + def build(self, **kw): + return iter([{"errorDetail": {"message": "npm ci failed"}}]) + + monkeypatch.setattr(docker_helper.docker, "APIClient", FakeAPIClient) + helper = _helper() + helper._quality_image = None + helper.benchmarker.config = types.SimpleNamespace(fw_root=str(tmp_path), server_docker_host=None) + with pytest.raises(RuntimeError, match="npm ci failed"): + helper.build_quality_image() diff --git a/toolset/utils/docker_helper.py b/toolset/utils/docker_helper.py index 8ebfdb8..0dd7814 100644 --- a/toolset/utils/docker_helper.py +++ b/toolset/utils/docker_helper.py @@ -1,5 +1,9 @@ +import io +import json import os +import posixpath import socket +import tarfile import time import re import traceback @@ -18,6 +22,9 @@ TEST_IMAGE_PREFIX = 'ssgberk/test.' RUN_LABEL = 'ssgberk.run' +QUALITY_IMAGE = 'ssgberk/quality' +QUALITY_TIMEOUT_SECONDS = 1800 +QUALITY_MEMORY = '4g' class DockerHelper: @@ -33,6 +40,7 @@ def __init__(self, benchmarker=None): base_url=self.benchmarker.config.server_docker_host) last_build = None + _quality_image = None def _run_labels(self): return {RUN_LABEL: str(self.benchmarker.config.run_id)} @@ -330,11 +338,13 @@ def server_container_exists(self, container_id_or_name): return False def benchmark(self, framework_test, script, variables, raw_file, - resources, timeout_seconds): + resources, timeout_seconds, export=None): ''' Runs the generator container with fixed resource limits and returns {"status": "ok"|"timeout"|"oom", "exitCode": int}. The container is - always stopped and removed before returning. + always stopped and removed before returning. With export + (output_folder, dest_tar), a successful build's output is copied to + dest_tar first (spec 009 R-3). ''' def watch_container(container): @@ -408,9 +418,73 @@ def watch_container(container): # stopping the container after a timeout if status == 'ok' and (state.get('OOMKilled') or exit_code == 137): status = 'oom' + if export and status == 'ok' and exit_code == 0: + DockerHelper.__export_output(container, *export) finally: try: container.remove(force=True) except docker.errors.NotFound: pass return {'status': status, 'exitCode': exit_code} + + @staticmethod + def __export_output(container, output_folder, dest): + '''Copies / to dest as a tar. Never raises.''' + try: + workdir = (container.attrs.get('Config') or {}).get('WorkingDir') or '/' + bits, _ = container.get_archive(posixpath.join(workdir, output_folder)) + os.makedirs(os.path.dirname(dest), exist_ok=True) + with open(dest, 'wb') as f: + for chunk in bits: + f.write(chunk) + except Exception as e: + log("quality: could not export %s: %s" % (output_folder, e), color=Fore.YELLOW) + + def build_quality_image(self): + ''' + Builds ssgberk/quality from /quality once per process + (spec 009 R-4), outside the generator build-time accounting. + ''' + if self._quality_image: + return self._quality_image + client = docker.APIClient(base_url=self.benchmarker.config.server_docker_host) + for token in client.build(path=os.path.join(self.benchmarker.config.fw_root, 'quality'), + dockerfile='Dockerfile', tag=QUALITY_IMAGE, forcerm=True, + pull=True, timeout=3600, decode=True): + if 'errorDetail' in token: + raise RuntimeError(token['errorDetail']['message']) + if token.get('stream', '').strip(): + log(token['stream'].rstrip(), prefix='quality: ') + self._quality_image = self.server.images.get(QUALITY_IMAGE).id + return self._quality_image + + def run_quality(self, tar_path, site_name, pages, + timeout_seconds=QUALITY_TIMEOUT_SECONDS): + ''' + Runs quality/run.mjs on the exported site with no network and no + mounts (spec 009 R-4, R-5) and returns its raw.json. + ''' + image = self.build_quality_image() + command = ['node', '/quality/run.mjs', '--site', '/work/' + site_name, + '--pages', json.dumps(pages), '--out', '/work/out'] + container = self.server.containers.create( + image, command=command, labels=self._run_labels(), network_mode='none', + mem_limit=QUALITY_MEMORY, working_dir='/work') + try: + with open(tar_path, 'rb') as f: + container.put_archive('/work', f.read()) + container.start() + result = container.wait(timeout=timeout_seconds) + try: + bits, _ = container.get_archive('/work/out/raw.json') + except docker.errors.NotFound: + logs = container.logs().decode('utf-8', 'replace') + raise RuntimeError('collector wrote no raw.json (exit %s): %s' % ( + result.get('StatusCode'), logs[-2000:])) + with tarfile.open(fileobj=io.BytesIO(b''.join(bits))) as tar: + return json.load(tar.extractfile(tar.getmembers()[0])) + finally: + try: + container.remove(force=True) + except docker.errors.NotFound: + pass From 1d494b02ca09dd1532f54d0a53ce81ac7a3e4870 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matheus=20Bregu=C3=AAz?= Date: Sat, 10 Oct 2026 14:50:32 -0300 Subject: [PATCH 08/15] feat(quality): --quality runs the untimed pass after a successful build (spec 009 R-1, R-2, R-6, R-17) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_quality_runner.py | 170 ++++++++++++++++++++++++++++++ toolset/benchmark/benchmarker.py | 36 ++++++- toolset/quality/runner.py | 55 ++++++++++ toolset/run-tests.py | 7 ++ toolset/utils/benchmark_config.py | 1 + toolset/utils/results.py | 19 +++- 6 files changed, 284 insertions(+), 4 deletions(-) create mode 100644 tests/test_quality_runner.py create mode 100644 toolset/quality/runner.py diff --git a/tests/test_quality_runner.py b/tests/test_quality_runner.py new file mode 100644 index 0000000..5cd3794 --- /dev/null +++ b/tests/test_quality_runner.py @@ -0,0 +1,170 @@ +import json +import pathlib +import tarfile +import types +from unittest import mock + +from toolset.quality import runner +from toolset.utils.results import STORED_RUN_KEYS, Results + +FIX = pathlib.Path(__file__).parent / "fixtures" / "quality" + + +def _cfg(**kw): + return types.SimpleNamespace(**kw) + + +def test_eligible_and_skipped(): + assert not runner.eligible(_cfg()) + assert not runner.skipped(_cfg()) + assert runner.eligible(_cfg(quality=True, suite_info=None, number_of_files="10", content_size="0.500")) + p0 = _cfg(quality=True, suite_info={"name": "P"}, number_of_files="50", content_size="5") + p1 = _cfg(quality=True, suite_info={"name": "P"}, number_of_files="50", content_size="50") + assert runner.eligible(p0) and not runner.skipped(p0) + assert not runner.eligible(p1) and runner.skipped(p1) + + +def _generator(tmp_path): + d = tmp_path / "frameworks" / "Go" / "hugo" + d.mkdir(parents=True) + (d / "benchmark_config.json").write_text(json.dumps( + {"framework": "hugo", "config": [{"output_folder": "public", "output_glob": "post/*/index.html"}]})) + return types.SimpleNamespace(name="hugo", directory=str(d)) + + +def _archive(results_dir): + tar_path = pathlib.Path(runner.archive_path(str(results_dir), "hugo")) + tar_path.parent.mkdir(parents=True) + with tarfile.open(tar_path, "w") as tar: + tar.add(FIX / "site", arcname="public") + return tar_path + + +def test_run_pass_ok(tmp_path): + test, results_dir = _generator(tmp_path), tmp_path / "results" + tar_path = _archive(results_dir) + helper = mock.Mock() + helper.run_quality.return_value = json.loads((FIX / "raw.json").read_text()) + q = runner.run_pass(helper, test, str(results_dir)) + assert q["status"] == "ok" and q["lighthouse"]["status"] == "ok" + args = helper.run_quality.call_args[0] + assert args[0] == str(tar_path) and args[1] == "public" + assert args[2]["post"] == {"url": "/post/hello/", "file": "post/hello/index.html"} + out = results_dir / "quality" / "hugo" + assert json.loads((out / "quality.json").read_text()) == q + assert (out / "raw.json").exists() and (out / "site" / "index.html").exists() + assert not tar_path.exists() + + +def test_run_pass_missing_archive(tmp_path): + test, results_dir = _generator(tmp_path), tmp_path / "results" + q = runner.run_pass(mock.Mock(), test, str(results_dir)) + assert q["status"] == "error" and "no exported output" in q["error"] + assert json.loads((results_dir / "quality" / "hugo" / "quality.json").read_text()) == q + + +def test_run_pass_collector_error(tmp_path): + test, results_dir = _generator(tmp_path), tmp_path / "results" + _archive(results_dir) + helper = mock.Mock() + helper.run_quality.side_effect = RuntimeError("collector wrote no raw.json (exit 1)") + q = runner.run_pass(helper, test, str(results_dir)) + assert q == {"status": "error", "error": "RuntimeError: collector wrote no raw.json (exit 1)"} + + +def _bm(tmp_path, quality=True, results=None, failure=None): + from toolset.benchmark.benchmarker import Benchmarker + tmp_path.mkdir(parents=True, exist_ok=True) + b = Benchmarker.__new__(Benchmarker) + b.config = types.SimpleNamespace(min_runs="1", run_test_timeout_seconds=10, resources={"cpus": 1}, + quality=quality, suite_info=None, number_of_files="50", + content_size="5") + test_type = mock.Mock() + test_type.get_script_name.return_value = "build.sh" + test_type.get_script_variables.return_value = {"min_runs": "1"} + b.config.types = {"datarate": test_type} + b.results = mock.Mock() + b.results.directory = str(tmp_path / "results") + b.results.get_raw_file.return_value = str(tmp_path / "raw.txt") + exports = [] + + def bench(ft, script, variables, raw_file, res, timeout, export=None): + exports.append(export) + return {"status": "ok", "exitCode": 0} + + b.docker_helper = mock.Mock() + b.docker_helper.benchmark.side_effect = bench + b.docker_helper._quality_image = "sha256:q" + b.results.parse_test.return_value = { + "results": [{"cv": None, "median": 1.0}] if results is None else results, + "status": "ok", "failureReason": failure, "unsupported": False} + b._Benchmarker__begin_logging = mock.Mock() + b._Benchmarker__end_logging = mock.Mock() + ft = _generator(tmp_path) + ft.runTests = {"datarate": mock.Mock(failed=False)} + return b, ft, exports + + +def test_quality_pass_runs_after_ok(tmp_path, monkeypatch): + b, ft, exports = _bm(tmp_path) + monkeypatch.setattr(runner, "run_pass", lambda helper, test, d: {"status": "ok", "test": test.name}) + b._Benchmarker__benchmark(ft, open(tmp_path / "log", "w")) + assert exports == [("public", runner.archive_path(b.results.directory, "hugo"))] + b.results.add_quality.assert_called_once_with("hugo", {"status": "ok", "test": "hugo"}, "sha256:q") + + +def test_quality_pass_missing_archive_keeps_result(tmp_path): + reported = [] + for quality in (False, True): + b, ft, _ = _bm(tmp_path / str(quality), quality=quality) + b._Benchmarker__benchmark(ft, open(tmp_path / ("log%s" % quality), "w")) + # everything but the generator object (its directory differs per run) + reported.append(b.results.report_benchmark_results.call_args[0][1:]) + if quality: + name, q, _image = b.results.add_quality.call_args[0] + assert name == "hugo" and q["status"] == "error" + assert reported[0] == reported[1] + + +def test_no_pass_without_results_or_with_failure(tmp_path, monkeypatch): + called = [] + monkeypatch.setattr(runner, "run_pass", lambda *a: called.append(a) or {"status": "ok"}) + for i, kw in enumerate(({"results": []}, {"failure": "nonconformant: index"})): + b, ft, _ = _bm(tmp_path / str(i), **kw) + b._Benchmarker__benchmark(ft, open(tmp_path / "log", "w")) + b.results.add_quality.assert_not_called() + assert called == [] + + +def test_skipped_cell_exports_nothing(tmp_path): + b, ft, exports = _bm(tmp_path) + b.config.suite_info = {"name": "P"} + b.config.content_size = "50" + b._Benchmarker__benchmark(ft, open(tmp_path / "log", "w")) + assert exports == [None] + b.results.add_quality.assert_not_called() + + +def test_results_add_quality(fake_benchmarker): + res = Results(fake_benchmarker) + res._environment = {"fingerprint": "f"} # what environment.capture would have cached + before = res._Results__to_jsonable() + assert "quality" not in before and "quality" not in before["environment"] + res.add_quality("hugo", {"status": "ok", "tools": {"lighthouse": "1.0.0"}}, "sha256:q") + after = res._Results__to_jsonable() + assert after["quality"] == {"hugo": {"status": "ok", "tools": {"lighthouse": "1.0.0"}}} + assert after["rawData"] == before["rawData"] and after["succeeded"] == before["succeeded"] + assert after["environment"] == {"fingerprint": "f", + "quality": {"lighthouse": "1.0.0", "image": "sha256:q"}} + + +def test_reparse_keeps_quality(fake_benchmarker, monkeypatch): + assert "quality" in STORED_RUN_KEYS + monkeypatch.setattr(Results, "_Results__count_commits", lambda self: None) + monkeypatch.setattr(Results, "_Results__count_sloc", lambda self: None) + res = Results(fake_benchmarker) + with open(res.file, "w") as f: + json.dump({"quality": {"hugo": {"status": "ok"}}, "rawData": {"datarate": {}}}, f) + res.reparse([]) + with open(res.file) as f: + assert json.load(f)["quality"] == {"hugo": {"status": "ok"}} diff --git a/toolset/benchmark/benchmarker.py b/toolset/benchmark/benchmarker.py index c230153..d4ce05e 100644 --- a/toolset/benchmark/benchmarker.py +++ b/toolset/benchmark/benchmarker.py @@ -16,6 +16,8 @@ from toolset.utils.audit import Audit from toolset.utils import resources from toolset.benchmark import noise +from toolset.quality import runner as quality_runner +from toolset.quality import site as quality_site class Benchmarker: @@ -242,6 +244,8 @@ def benchmark_type(test_type): with open(raw_file, 'w'): pass + export = self.__quality_export(framework_test) + self._quality_kwargs = {'export': export} if export else {} if not test.failed: script = self.config.types[test_type].get_script_name() script_variables = self.config.types[test_type].get_script_variables() @@ -251,7 +255,8 @@ def benchmark_type(test_type): try: outcome = self.docker_helper.benchmark( framework_test, script, script_variables, raw_file, - self.resolve_resources(), self.config.run_test_timeout_seconds) + self.resolve_resources(), self.config.run_test_timeout_seconds, + **self._quality_kwargs) finally: self.__end_logging() @@ -272,6 +277,9 @@ def benchmark_type(test_type): self.results.report_benchmark_results(framework_test, test_type, results['results'], results.get('unsupported', False), results.get('failureReason')) + if export and results['results'] and not results.get('failureReason') \ + and not results.get('unsupported'): + self.__quality_pass(framework_test, benchmark_log) log("Complete", file=benchmark_log) return bool(results['results']) or bool(results.get('unsupported', False)) @@ -281,6 +289,29 @@ def benchmark_type(test_type): ok = False return ok + def __quality_export(self, framework_test): + '''(output_folder, dest_tar) when the quality pass runs in this cell (spec 009 R-1).''' + if quality_runner.skipped(self.config): + log("quality: skipped (only cell nf50-cs5 of a suite)") + return None + if not quality_runner.eligible(self.config): + return None + try: + output_folder, _glob = quality_site.generator_config(framework_test.directory) + except Exception as e: + log("quality: cannot read output_folder: %s" % e) + return None + return (output_folder, + quality_runner.archive_path(self.results.directory, framework_test.name)) + + def __quality_pass(self, framework_test, benchmark_log): + '''Untimed; never changes the build result (spec 009 R-2).''' + log("QUALITY PASS %s (untimed)" % framework_test.name, file=benchmark_log, border='*') + quality = quality_runner.run_pass(self.docker_helper, framework_test, self.results.directory) + self.results.add_quality(framework_test.name, quality, + getattr(self.docker_helper, '_quality_image', None)) + log("quality: %s" % quality.get('status'), file=benchmark_log) + def __noise_control(self, framework_test, test_type, script, script_variables, raw_file, results, benchmark_log): ''' @@ -304,7 +335,8 @@ def __noise_control(self, framework_test, test_type, script, try: outcome = self.docker_helper.benchmark( framework_test, script, variables, raw_file, - self.resolve_resources(), self.config.run_test_timeout_seconds) + self.resolve_resources(), self.config.run_test_timeout_seconds, + **getattr(self, '_quality_kwargs', {})) finally: self.__end_logging() second = self.results.parse_test(framework_test, test_type) diff --git a/toolset/quality/runner.py b/toolset/quality/runner.py new file mode 100644 index 0000000..f7cedc6 --- /dev/null +++ b/toolset/quality/runner.py @@ -0,0 +1,55 @@ +''' +The untimed quality pass of one generator (spec 009 R-1..R-3, R-16). +''' +import json +import os + +from toolset.quality import reduce, site + +# R-1: with --suite, the pass runs only in this (numberOfFiles, contentSize) cell +QUALITY_CELL = (50, 5.0) + + +def _cell(config): + return int(config.number_of_files), float(config.content_size) + + +def eligible(config): + if not getattr(config, 'quality', False): + return False + if getattr(config, 'suite_info', None) is None: + return True + return _cell(config) == QUALITY_CELL + + +def skipped(config): + return bool(getattr(config, 'quality', False)) and not eligible(config) + + +def archive_path(results_dir, name): + return os.path.join(results_dir, 'quality', name, 'site.tar') + + +def run_pass(docker_helper, test, results_dir): + '''Runs the pass, writes quality.json and returns it. Never raises.''' + out_dir = os.path.join(results_dir, 'quality', test.name) + tar_path = archive_path(results_dir, test.name) + try: + output_folder, output_glob = site.generator_config(test.directory) + if not os.path.exists(tar_path): + raise RuntimeError('no exported output (%s)' % output_folder) + site_dir = site.extract(tar_path, os.path.join(out_dir, 'site')) + pages = site.resolve_pages(site_dir) + site_name = os.path.basename(output_folder.rstrip('/')) + raw = docker_helper.run_quality(tar_path, site_name, pages) + with open(os.path.join(out_dir, 'raw.json'), 'w') as f: + json.dump(raw, f, indent=2) + quality = reduce.build_quality(raw, site_dir, pages, output_glob) + except Exception as e: + quality = {'status': 'error', 'error': '%s: %s' % (type(e).__name__, e)} + os.makedirs(out_dir, exist_ok=True) + with open(os.path.join(out_dir, 'quality.json'), 'w') as f: + json.dump(quality, f, indent=2) + if os.path.exists(tar_path): + os.remove(tar_path) + return quality diff --git a/toolset/run-tests.py b/toolset/run-tests.py index c7d59f9..45dc4d1 100644 --- a/toolset/run-tests.py +++ b/toolset/run-tests.py @@ -221,6 +221,13 @@ def build_parser(): action='store_true', default=False, help='Build generator images without the Docker layer cache') + parser.add_argument( + '--quality', + action='store_true', + default=False, + help='After the timed runs, audit the output of the last build (Lighthouse, SEO ' + 'signals, HTML, accessibility, links, weight); untimed. With --suite, only in ' + 'the cell nf50-cs5 (spec 009)') # Resource limits applied to every generator container of the run parser.add_argument( '--cpus', diff --git a/toolset/utils/benchmark_config.py b/toolset/utils/benchmark_config.py index 171d2f0..b0e019d 100644 --- a/toolset/utils/benchmark_config.py +++ b/toolset/utils/benchmark_config.py @@ -43,6 +43,7 @@ def __init__(self, args): self.require_conformance = bool(getattr(args, 'require_conformance', False)) self.verbose_build = args.verbose self.no_cache = bool(getattr(args, 'no_cache', False)) + self.quality = bool(getattr(args, 'quality', False)) self.cpus = getattr(args, 'cpus', 4.0) self.memory = getattr(args, 'memory', 8 * 1024 ** 3) self.cpuset = getattr(args, 'cpuset', 'auto') diff --git a/toolset/utils/results.py b/toolset/utils/results.py index cd55480..6ea6f9d 100644 --- a/toolset/utils/results.py +++ b/toolset/utils/results.py @@ -28,7 +28,7 @@ # Run-level keys a --parse keeps from the stored results.json: they describe # how the run was made, which raw.txt cannot tell. STORED_RUN_KEYS = ('suite', 'profile', 'resources', 'protocol', 'environment', - 'generators', 'imageBuild') + 'generators', 'imageBuild', 'quality') # Per-result keys a --parse keeps from the stored result. STORED_RESULT_KEYS = ('noisy', 'attempts', 'resources') # Failure reasons decided by the container outcome, not by raw.txt. @@ -214,11 +214,21 @@ def __init__(self, benchmarker): self.unsupported = dict() self.unsupported['datarate'] = [] self.failureReasons = dict() + self.quality = {} + self.quality_environment = None ############################################################################# # PUBLIC FUNCTIONS ############################################################################# + def add_quality(self, name, quality, image_id=None): + '''results.json quality. and environment.quality (spec 009 R-6, R-17).''' + self.quality[name] = quality + tools = quality.get('tools') + if self.quality_environment is None and isinstance(tools, dict) \ + and not tools.get('error'): + self.quality_environment = dict(tools, image=image_id) + def parse(self, tests): ''' Ensures that the system has all necessary software to run @@ -502,9 +512,14 @@ def __to_jsonable(self): toRet['failureReasons'] = self.failureReasons toRet['resources'] = getattr(self.config, 'resources', None) toRet['protocol'] = self.__protocol() - toRet['environment'] = self.__environment() + env = self.__environment() + if env is not None and self.quality_environment: + env = dict(env, quality=self.quality_environment) + toRet['environment'] = env toRet['generators'] = self.__generators() toRet['imageBuild'] = self.__image_build() + if getattr(self, 'quality', None): + toRet['quality'] = self.quality # --parse: what the stored run recorded wins over this process's config toRet.update(getattr(self, '_stored_run', None) or {}) From d59f0b4bb6851d2cb620ee8efead2be80ca9d972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matheus=20Bregu=C3=AAz?= Date: Sat, 10 Oct 2026 14:51:40 -0300 Subject: [PATCH 09/15] feat(quality): Qualidade section in summary.md and quality-summary.csv (spec 009 R-18, R-19) Co-Authored-By: Claude Haiku 5.5 --- tests/test_quality_summary.py | 67 +++++++++++++++++++++ toolset/utils/results.py | 3 + toolset/utils/summary.py | 107 ++++++++++++++++++++++++++++++++++ 3 files changed, 177 insertions(+) create mode 100644 tests/test_quality_summary.py diff --git a/tests/test_quality_summary.py b/tests/test_quality_summary.py new file mode 100644 index 0000000..3c36584 --- /dev/null +++ b/tests/test_quality_summary.py @@ -0,0 +1,67 @@ +import csv +import io +import json +import pathlib + +from toolset.quality import reduce, site +from toolset.utils import summary +from toolset.utils.results import parse_build_output + +FIX = pathlib.Path(__file__).parent / "fixtures" / "quality" +RAW_OK = pathlib.Path(__file__).parent / "fixtures" / "raw_ok.txt" + + +def _quality(): + raw = json.loads((FIX / "raw.json").read_text()) + pages = site.resolve_pages(str(FIX / "site")) + return reduce.build_quality(raw, str(FIX / "site"), pages, "post/*/index.html") + + +def _data(with_quality): + [entry] = parse_build_output(RAW_OK.read_text(), "50", "5", "1") + data = {"name": "r", "frameworks": ["hugo", "gatsby"], + "rawData": {"datarate": {"hugo": [entry]}}, + "succeeded": {"datarate": ["hugo"]}, "failed": {"datarate": ["gatsby"]}, "excluded": []} + if with_quality: + data["quality"] = {"hugo": _quality(), "gatsby": {"status": "error", "error": "RuntimeError: boom"}} + return data + + +def test_quality_rows(): + rows = summary.quality_rows(_data(True)) + assert [r["framework"] for r in rows] == ["gatsby", "hugo"] + hugo = rows[1] + assert hugo["performance"] == 0.98 and hugo["lcpMs"] == 800 + assert hugo["postJsBytes"] == 0 and hugo["jsClass"] == "none" + assert hugo["htmlErrors"] == 1 and hugo["a11yViolations"] == 2 and hugo["brokenLinks"] == 2 + assert (hugo["seoPresent"], hugo["seoTotal"]) == (2, 12) + assert rows[0]["status"] == "error" and rows[0]["performance"] is None + + +def test_quality_markdown_section(): + lines = summary.quality_markdown(summary.quality_rows(_data(True))) + text = "\n".join(lines) + assert lines[0] == "## Qualidade" + assert "| hugo | 98 | 100 | 100 | 82 | 800 | 0.0 | none |" in text + assert "2/12" in text and "no meta description and no viewport" in text + assert "gatsby: RuntimeError: boom" in text + assert "Rank" not in text + + +def test_quality_csv_has_every_numeric_field(): + rows = summary.quality_rows(_data(True)) + parsed = list(csv.DictReader(io.StringIO(summary.quality_csv(rows)))) + assert parsed[1]["framework"] == "hugo" + header = list(parsed[0].keys()) + assert header[:len(summary.QUALITY_COLUMNS)] == summary.QUALITY_COLUMNS + assert "lighthouse.desktop.index.scores.seo.median" in header + assert "weight.brotliBytes" in header + + +def test_quality_never_changes_timing_summary(): + without, with_q = _data(False), _data(True) + assert summary.build_rows(with_q, {}) == summary.build_rows(without, {}) + md_without = summary.to_markdown(summary.build_rows(without, {}), without) + md_with = summary.to_markdown(summary.build_rows(with_q, {}), with_q) + assert md_with.startswith(md_without.rstrip("\n")) + assert "## Qualidade" in md_with and "## Qualidade" not in md_without diff --git a/toolset/utils/results.py b/toolset/utils/results.py index 6ea6f9d..d5302d8 100644 --- a/toolset/utils/results.py +++ b/toolset/utils/results.py @@ -262,6 +262,9 @@ def write_summary(self): f.write(summary.to_csv(rows)) with open(os.path.join(self.directory, 'summary.md'), 'w') as f: f.write(markdown) + if data.get('quality'): + with open(os.path.join(self.directory, 'quality-summary.csv'), 'w', newline='') as f: + f.write(summary.quality_csv(summary.quality_rows(data))) return markdown except Exception as e: log("Error writing summary: %s" % e) diff --git a/toolset/utils/summary.py b/toolset/utils/summary.py index 140fa6d..f83fd42 100644 --- a/toolset/utils/summary.py +++ b/toolset/utils/summary.py @@ -339,9 +339,116 @@ def to_markdown(rows, meta): lines += _ranking_section(rows, ranking.not_ranked_reason(meta)) lines += _caveats(rows, meta) lines += _failed_section(rows) + if meta.get('quality'): + lines += [''] + quality_markdown(quality_rows(meta)) return '\n'.join(lines) + '\n' +QUALITY_COLUMNS = ['framework', 'status', 'performance', 'accessibility', 'bestPractices', 'seo', + 'lcpMs', 'postJsBytes', 'jsClass', 'gzipBytes', 'htmlErrors', + 'a11yViolations', 'brokenLinks', 'seoPresent', 'seoTotal'] +QUALITY_NOTE = ('The SF 005 templates carry no meta description and no viewport, so every ' + 'generator loses the same Lighthouse SEO points; that is not a difference ' + 'between generators.') + + +def _get(data, *path): + for key in path: + if not isinstance(data, dict): + return None + data = data.get(key) + return data + + +def _sum_pages(q, section, key): + pages = _get(q, section, 'pages') or {} + values = [p.get(key) for p in pages.values() if isinstance(p, dict) and p.get(key) is not None] + return sum(values) if values else None + + +def _flatten(data, prefix=''): + for key, value in (data or {}).items(): + name = prefix + str(key) + if isinstance(value, dict): + yield from _flatten(value, name + '.') + elif isinstance(value, (int, float)) and not isinstance(value, bool): + yield name, value + + +def quality_rows(results): + '''One row per generator with a quality pass (spec 009 R-18), sorted by name.''' + rows = [] + for name in sorted(results.get('quality') or {}): + q = results['quality'][name] + post = _get(q, 'lighthouse', 'mobile', 'post') or {} + rows.append({ + 'framework': name, + 'status': q.get('status'), + 'error': q.get('error'), + 'performance': _get(post, 'scores', 'performance', 'median'), + 'accessibility': _get(post, 'scores', 'accessibility', 'median'), + 'bestPractices': _get(post, 'scores', 'best-practices', 'median'), + 'seo': _get(post, 'scores', 'seo', 'median'), + 'lcpMs': _get(post, 'metrics', 'lcp', 'median'), + 'postJsBytes': _get(q, 'js', 'pages', 'post'), + 'jsClass': _get(q, 'js', 'jsClass'), + 'gzipBytes': _get(q, 'weight', 'gzipBytes'), + 'htmlErrors': _sum_pages(q, 'html', 'errors'), + 'a11yViolations': _sum_pages(q, 'a11y', 'total'), + 'brokenLinks': _get(q, 'links', 'broken'), + 'seoPresent': _get(q, 'seo', 'present', 'post'), + 'seoTotal': _get(q, 'seo', 'total'), + 'numeric': dict(_flatten({k: v for k, v in q.items() if k != 'tools'})), + }) + return rows + + +def quality_csv(rows): + extra = sorted({k for r in rows for k in r['numeric']}) + out = io.StringIO() + writer = csv.writer(out, lineterminator='\n') + writer.writerow(QUALITY_COLUMNS + extra) + for r in rows: + writer.writerow([_blank(r[c]) for c in QUALITY_COLUMNS] + + [_blank(r['numeric'].get(k)) for k in extra]) + return out.getvalue() + + +def _score(value): + return '—' if value is None else '%d' % round(value * 100) + + +def _kb(value): + return '—' if value is None else '%.1f' % (value / 1024) + + +def _int(value): + return '—' if value is None else '%d' % round(value) + + +def quality_markdown(rows): + lines = ['## Qualidade', '', + 'Untimed pass over the output of the last build (spec 009). Scores and LCP: ' + 'Lighthouse mobile preset on the post page, median of 3 runs. No combined score ' + 'and no ranking.', '', + '| Framework | Performance | Accessibility | Best practices | SEO | LCP (ms) ' + '| Post JS (KB) | JS | Gzip (KB) | HTML errors | axe violations | Broken links ' + '| SEO signals |', + '|' + '---|' * 13] + for r in rows: + signals = '—' if r['seoPresent'] is None else '%s/%s' % (r['seoPresent'], r['seoTotal']) + cells = [r['framework'], _score(r['performance']), _score(r['accessibility']), + _score(r['bestPractices']), _score(r['seo']), _int(r['lcpMs']), + _kb(r['postJsBytes']), r['jsClass'] or '—', _kb(r['gzipBytes']), + _int(r['htmlErrors']), _int(r['a11yViolations']), _int(r['brokenLinks']), signals] + lines.append('| ' + ' | '.join(str(c) for c in cells) + ' |') + lines += ['', '- ' + QUALITY_NOTE] + for r in rows: + if r['status'] == 'error': + lines.append('- %s: %s' % (r['framework'], r['error'])) + return lines + + def _read_json(path): with open(path) as f: return json.load(f) From 01eef487d9bce6fa08a4f96db928513f434c737b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matheus=20Bregu=C3=AAz?= Date: Sat, 10 Oct 2026 14:52:44 -0300 Subject: [PATCH 10/15] ci(quality): quality image and hugo quality pass on amd64 and arm64; round input (spec 009 goal 4) Co-Authored-By: Claude Haiku 5.5 --- .github/workflows/benchmark-round.yml | 11 +++++-- .github/workflows/ci.yml | 41 +++++++++++++++++++++++++++ README.md | 16 +++++++++++ 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark-round.yml b/.github/workflows/benchmark-round.yml index 7adb589..c0ebe1b 100644 --- a/.github/workflows/benchmark-round.yml +++ b/.github/workflows/benchmark-round.yml @@ -8,6 +8,10 @@ on: type: choice default: "" options: ["", smoke, standard, stress, P, M, G, GG] + quality: + description: Run the untimed quality pass (spec 009); with a suite, only in cell nf50-cs5 + type: boolean + default: false number_of_files: description: Number of posts per build (-nf) type: string @@ -102,11 +106,14 @@ jobs: NF: ${{ matrix.number_of_files }} CS: ${{ matrix.content_size }} MR: ${{ matrix.min_runs }} + QUALITY: ${{ inputs.quality }} run: | + Q="" + if [ "$QUALITY" = "true" ]; then Q="--quality"; fi if [ -n "$SUITE" ]; then - ./ssgberk --test "$TEST" --suite "$SUITE" --cell "$CELL" + ./ssgberk --test "$TEST" --suite "$SUITE" --cell "$CELL" $Q else - ./ssgberk --test "$TEST" -nf "$NF" -cs "$CS" -mr "$MR" + ./ssgberk --test "$TEST" -nf "$NF" -cs "$CS" -mr "$MR" $Q fi - name: Record runner environment if: always() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c37f39..99eab51 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,3 +32,44 @@ jobs: - run: | R=$(ls -td results/*/ | head -1) jq -e '.rawData.datarate.hugo[0].mean > 0' "$R/results.json" + + quality-image: + needs: toolset + strategy: + matrix: + runner: [ubuntu-24.04, ubuntu-24.04-arm] + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + - run: docker build -t ssgberk/quality quality + - name: Collector on the fixture site + run: | + docker run --rm --network none \ + -v "$PWD/tests/fixtures/quality/site:/fixture:ro" -v "$PWD/quality/test:/test:ro" \ + ssgberk/quality sh -c 'node /quality/run.mjs --site /fixture --pages "$(cat /test/pages.json)" --out /tmp/out \ + && node /test/check-shape.mjs /tmp/out/raw.json /test/expected-shape.json' + + quality: + needs: quality-image + strategy: + matrix: + runner: [ubuntu-24.04, ubuntu-24.04-arm] + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - run: ./ssgberk --test hugo -nf 50 -cs 5 -mr 1 --quality + - run: | + R=$(ls -td results/*/ | head -1) + jq -e '.rawData.datarate.hugo[0].mean > 0' "$R/results.json" + jq -e '.quality.hugo.status == "ok"' "$R/results.json" + for s in lighthouse seo html a11y links weight js extra; do + jq -e --arg s "$s" '.quality.hugo[$s].status == "ok"' "$R/results.json" + done + jq -e '[.quality.hugo.lighthouse.mobile[], .quality.hugo.lighthouse.desktop[] + | .scores[] | .median | select(. < 0 or . > 1)] | length == 0' "$R/results.json" + jq -e '.environment.quality.lighthouse and .environment.quality.chromium' "$R/results.json" + test -s "$R/quality-summary.csv" + grep -q '^## Qualidade' "$R/summary.md" diff --git a/README.md b/README.md index 04b5cfc..c2b324d 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,22 @@ Both summary files are regenerated by `--parse`. After a `--suite` run, `results//` also holds `suite.json` (cells, generator order, outcome counts), `suite-summary.csv`/`suite-summary.md` (all cells in one table) and `scaling.csv` (scaling exponent per generator). Each cell has its own `results.json` and summaries under `/nf-cs/`. +## Quality pass + +`--quality` audits the output of the last timed build of each generator, without timing it (spec `docs/specs/009-output-quality`): + + $ ./ssgberk --test hugo -nf 50 -cs 5 --quality + +With `--suite`, the pass runs only in the cell `nf50-cs5` (suite P, cell 0). The toolset's tests need Python 3.12 (the toolset image and CI use it). It builds the `ssgberk/quality` image from `quality/` once. That image pins Lighthouse, Playwright's Chromium, axe-core, html-validate and lychee. The image runs with no network and with the site copied in, and it audits the index, the first post and `404.html`: +- Lighthouse, mobile and desktop presets, median of 3 runs; +- SEO signals (title, lang, description, viewport, canonical, Open Graph, Twitter card, JSON-LD, robots, hreflang, headings, `alt`, plus `robots.txt`, `sitemap.xml`, feed and favicon); +- HTML validity, axe-core violations and broken internal links; +- weight on disk, gzip and brotli; +- JS bytes, and whether the post works without JS; +- files outside the reference page set. + +Results go to `results//quality//quality.json`, to `quality` in `results.json`, to the "Qualidade" section of `summary.md` and to `quality-summary.csv`. They have no combined score and no ranking. A failing check is a result and never fails the run. + ## Content size `-cs` is the content size per post, in KB: From dda092c4b2af7dbde7b352841637f97edecf35f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matheus=20Bregu=C3=AAz?= Date: Sat, 10 Oct 2026 14:55:00 -0300 Subject: [PATCH 11/15] feat(quality): pinned collector image with Lighthouse, axe-core, html-validate, lychee (spec 009 R-4, R-5) Pinned: lighthouse 13.5.0, chrome-launcher 1.2.2, playwright 1.64.0 (Chromium 156.0.8078.4), axe-core 4.14.0, html-validate 11.16.2, lychee 0.24.2, Node 24.21.0. Co-Authored-By: Claude Opus 5.5 (1M context) --- quality/.dockerignore | 2 + quality/Dockerfile | 33 + quality/package-lock.json | 1496 ++++++++++++++++++++++++++++++ quality/package.json | 13 + quality/run.mjs | 284 ++++++ quality/test/check-shape.mjs | 42 + quality/test/expected-shape.json | 208 +++++ quality/test/pages.json | 1 + 8 files changed, 2079 insertions(+) create mode 100644 quality/.dockerignore create mode 100644 quality/Dockerfile create mode 100644 quality/package-lock.json create mode 100644 quality/package.json create mode 100644 quality/run.mjs create mode 100644 quality/test/check-shape.mjs create mode 100644 quality/test/expected-shape.json create mode 100644 quality/test/pages.json diff --git a/quality/.dockerignore b/quality/.dockerignore new file mode 100644 index 0000000..f05b1f2 --- /dev/null +++ b/quality/.dockerignore @@ -0,0 +1,2 @@ +node_modules +test diff --git a/quality/Dockerfile b/quality/Dockerfile new file mode 100644 index 0000000..be9d1f0 --- /dev/null +++ b/quality/Dockerfile @@ -0,0 +1,33 @@ +# SSGBerk quality collector (benchmark-tool spec 009): Lighthouse, axe-core, +# html-validate and lychee against Playwright's Chromium. Runs with no network. +FROM ubuntu:24.04 + +ARG DEBIAN_FRONTEND=noninteractive +ARG NODE_VERSION=24.21.0 +ARG LYCHEE_VERSION=0.24.2 + +RUN apt-get -yqq update \ + && apt-get -yqq install --no-install-recommends ca-certificates curl xz-utils \ + && rm -rf /var/lib/apt/lists/* + +RUN ARCH="$(dpkg --print-architecture)" \ + && case "$ARCH" in amd64) NARCH=x64 ;; arm64) NARCH=arm64 ;; *) echo "unsupported $ARCH"; exit 1 ;; esac \ + && curl -fsSL "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${NARCH}.tar.xz" \ + | tar -xJ -C /usr/local --strip-components=1 \ + && node --version + +RUN ARCH="$(dpkg --print-architecture)" \ + && case "$ARCH" in amd64) LARCH=x86_64 ;; arm64) LARCH=aarch64 ;; *) echo "unsupported $ARCH"; exit 1 ;; esac \ + && curl -fsSL "https://github.com/lycheeverse/lychee/releases/download/lychee-v${LYCHEE_VERSION}/lychee-${LARCH}-unknown-linux-gnu.tar.gz" \ + | tar -xz -C /usr/local/bin --strip-components=1 --wildcards '*/lychee' \ + && lychee --version + +ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright NPM_CONFIG_UPDATE_NOTIFIER=false +WORKDIR /quality +COPY package.json package-lock.json ./ +RUN npm ci --omit=dev \ + && npx playwright install --with-deps chromium \ + && rm -rf /root/.npm /var/lib/apt/lists/* +COPY run.mjs ./ +RUN mkdir -p /work/out +WORKDIR /work diff --git a/quality/package-lock.json b/quality/package-lock.json new file mode 100644 index 0000000..267bb0a --- /dev/null +++ b/quality/package-lock.json @@ -0,0 +1,1496 @@ +{ + "name": "ssgberk-quality", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "ssgberk-quality", + "dependencies": { + "axe-core": "4.14.0", + "chrome-launcher": "1.2.2", + "html-validate": "11.16.2", + "lighthouse": "13.5.0", + "playwright": "1.64.0" + } + }, + "node_modules/@formatjs/ecma402-abstract": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/@formatjs/ecma402-abstract/-/ecma402-abstract-2.3.6.tgz", + "integrity": "sha512-HJnTFeRM2kVFVr5gr5kH1XP6K0JcJtE7Lzvtr3FS/so5f1kpsqqqxy5JF+FRaO6H2qmcMfAUIox7AJteieRtVw==", + "license": "MIT", + "dependencies": { + "@formatjs/fast-memoize": "2.2.7", + "@formatjs/intl-localematcher": "0.6.2", + "decimal.js": "^10.4.3", + "tslib": "^2.8.0" + } + }, + "node_modules/@formatjs/fast-memoize": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/@formatjs/fast-memoize/-/fast-memoize-2.2.7.tgz", + "integrity": "sha512-Yabmi9nSvyOMrlSeGGWDiH7rf3a7sIwplbvo/dlz9WCIjzIQAfy1RMf4S0X3yG724n5Ghu2GmEl5NJIV6O9sZQ==", + "license": "MIT", + "dependencies": { + "tslib": "^2.8.0" + } + }, + "node_modules/@formatjs/icu-messageformat-parser": { + "version": "2.11.4", + "resolved": "https://registry.npmjs.org/@formatjs/icu-messageformat-parser/-/icu-messageformat-parser-2.11.4.tgz", + "integrity": "sha512-7kR78cRrPNB4fjGFZg3Rmj5aah8rQj9KPzuLsmcSn4ipLXQvC04keycTI1F7kJYDwIXtT2+7IDEto842CfZBtw==", + "license": "MIT", + "dependencies": { + "@formatjs/ecma402-abstract": "2.3.6", + "@formatjs/icu-skeleton-parser": "1.8.16", + "tslib": "^2.8.0" + } + }, + "node_modules/@formatjs/icu-skeleton-parser": { + "version": "1.8.16", + "resolved": "https://registry.npmjs.org/@formatjs/icu-skeleton-parser/-/icu-skeleton-parser-1.8.16.tgz", + "integrity": "sha512-H13E9Xl+PxBd8D5/6TVUluSpxGNvFSlN/b3coUp0e0JpuWXXnQDiavIpY3NnvSp4xhEMoXyyBvVfdFX8jglOHQ==", + "license": "MIT", + "dependencies": { + "@formatjs/ecma402-abstract": "2.3.6", + "tslib": "^2.8.0" + } + }, + "node_modules/@formatjs/intl-localematcher": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/@formatjs/intl-localematcher/-/intl-localematcher-0.6.2.tgz", + "integrity": "sha512-XOMO2Hupl0wdd172Y06h6kLpBz6Dv+J4okPLl4LPtzbr8f66WbIoy4ev98EBuZ6ZK4h5ydTN6XneT4QVpD7cdA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.8.0" + } + }, + "node_modules/@html-validate/stylish": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/@html-validate/stylish/-/stylish-6.1.0.tgz", + "integrity": "sha512-ceNvF6/5hgvw/OrxXtdl2KurRVLN/Q/OkD/qminRRY/IIERmVaqJOpDF1nhRZMrOG/vyKKW0+Nxw6lSoHUP4Ow==", + "license": "MIT", + "engines": { + "node": "^22.16.0 || >= 24.0.0" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.220.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.220.0.tgz", + "integrity": "sha512-CmVa4ImJ+ynfrPMNaAXHET6Bhb44SwzmfyVJFq9ni2jgXJR/l7C6gfVFddNmHP+ZOkP9cf4f9DBe68qVLTHc9w==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/core": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.12.0.tgz", + "integrity": "sha512-1HsSAuvT4/my0QrXWsyXFdWjKaPeCHsismKZgEEIb8NK13LP6DrMRlsUbvD/0F9mDWK3xDom9AMv3IscxgOnOQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/instrumentation": { + "version": "0.220.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.220.0.tgz", + "integrity": "sha512-xQx3E2WxP1mDvKzxLxX+CTCtNLa560YJZ3087qYHerl2YmiKpv7AH+dAy7vmx+eVrZ5BwhfWUAVoKOoxCNHcpw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.220.0", + "import-in-the-middle": "^3.0.0", + "require-in-the-middle": "^8.0.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/resources": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.12.0.tgz", + "integrity": "sha512-+rSRoOdln6NhXTVcBdHSlZxJDNzwxadeweK2ltZO3tVtfp6aklABc06xclDk6ovRR+EiGgIcSbh/6NgjFhnb3A==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.12.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace/-/sdk-trace-2.12.0.tgz", + "integrity": "sha512-ImVuRQ6faOsTi1Wna88s2avgrrf/iDgXPigIeynkUhjstMuoQUDOJXgkez2j1mlU4btjXcXBGa+q7qKFYR3GsA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.12.0", + "@opentelemetry/resources": "2.12.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.12.0.tgz", + "integrity": "sha512-JOERgZCxRM06FEJ5kEWxlSRChah6MmXDx3KwCruqEv/iEyLkTokCSemf2y1w8oIHtfrr8EVTpzebq/31vn2Cpg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.12.0", + "@opentelemetry/resources": "2.12.0", + "@opentelemetry/sdk-trace": "2.12.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.43.0.tgz", + "integrity": "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@paulirish/trace_engine": { + "version": "0.0.65", + "resolved": "https://registry.npmjs.org/@paulirish/trace_engine/-/trace_engine-0.0.65.tgz", + "integrity": "sha512-Qsm6F5C8xf6ZzQXbQc2+wcpe6sggfs/gvc/ytqSurdvYg3kyW0ECHCqE0CWBKZpqgjVfPNX9c7SCS3r2nEIRGg==", + "license": "BSD-3-Clause", + "dependencies": { + "legacy-javascript": "latest", + "third-party-web": "latest" + } + }, + "node_modules/@puppeteer/browsers": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-3.2.4.tgz", + "integrity": "sha512-okJ6yQMc0SBexOKxMi966omwAV1uAPwgF/YMZdb04gZQPXVjWHl9bmXjZYtIcsN1+PN3Bz4iFfGK3g+c6lZJ0Q==", + "license": "Apache-2.0", + "dependencies": { + "modern-tar": "^0.8.5", + "yargs": "^18.0.0" + }, + "bin": { + "browsers": "lib/main-cli.js" + }, + "engines": { + "node": ">=22.12.0" + }, + "peerDependencies": { + "proxy-agent": ">=8.0.1", + "yauzl": "^2.10.0 || ^3.4.0" + }, + "peerDependenciesMeta": { + "proxy-agent": { + "optional": true + }, + "yauzl": { + "optional": true + } + } + }, + "node_modules/@puppeteer/browsers/node_modules/ansi-regex": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.4.0.tgz", + "integrity": "sha512-KzTVk2tCWAHtYrvvvaP8bJKJq2pVinhLcGEQdtLIYPbmNGNyYe8QwNaTUYQp2J7/vIsUKt5QCqAfUkYyG9DkOw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@puppeteer/browsers/node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@puppeteer/browsers/node_modules/cliui": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz", + "integrity": "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==", + "license": "ISC", + "dependencies": { + "string-width": "^7.2.0", + "strip-ansi": "^7.1.0", + "wrap-ansi": "^9.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@puppeteer/browsers/node_modules/cliui/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@puppeteer/browsers/node_modules/emoji-regex": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", + "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "license": "MIT" + }, + "node_modules/@puppeteer/browsers/node_modules/string-width": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.3.0.tgz", + "integrity": "sha512-ZbmZM0JCihQN91dWnxoipT2KOEyHqEyfRXUyjuRhW8b/xnqPDoq4gWEVApTVa9db2wN8mmoikgFBbjh71+cGeQ==", + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.5.0", + "strip-ansi": "^7.1.2" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@puppeteer/browsers/node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/@puppeteer/browsers/node_modules/wrap-ansi": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", + "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "string-width": "^7.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/@puppeteer/browsers/node_modules/wrap-ansi/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@puppeteer/browsers/node_modules/yargs": { + "version": "18.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-18.2.0.tgz", + "integrity": "sha512-9OpKOLeaoNFecEp7P6iYbzze/5CqWoH7N3SMs/6y1XF4nMvFMspEGZzJ6uFi9MmQwXhyzqYoSEKO3tvA3Z/o2w==", + "license": "MIT", + "dependencies": { + "cliui": "^9.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "string-width": "^8.2.1", + "y18n": "^5.0.5", + "yargs-parser": "^22.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, + "node_modules/@puppeteer/browsers/node_modules/yargs-parser": { + "version": "22.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz", + "integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==", + "license": "ISC", + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, + "node_modules/@sentry/conventions": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/@sentry/conventions/-/conventions-0.16.0.tgz", + "integrity": "sha512-fO9PLmHdVURcSPUpWCItWAtgKiMwGdJHbovoSEyLplX5sxs2ugvI4CBPTrkkgqhObnZOD0CnWBKDzSVQYBKEyQ==", + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/@sentry/core": { + "version": "10.76.2", + "resolved": "https://registry.npmjs.org/@sentry/core/-/core-10.76.2.tgz", + "integrity": "sha512-vndrQalRiwYatTkjlLWvyf69KocXIb9TKJ4A9oiyWguL8PJYtqIRTlRe4OpkpjLrxS18fvRdV0jEm29c/9p3hQ==", + "license": "MIT", + "dependencies": { + "@sentry/conventions": "^0.16.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@sentry/node": { + "version": "10.76.2", + "resolved": "https://registry.npmjs.org/@sentry/node/-/node-10.76.2.tgz", + "integrity": "sha512-LSwGGE/GceTbkDHRoQQr3lq/dpxxA6RdooBbuGTJ/S6BdqKX2kw2FdV7Se9eJzZJJJ5AxCZum37+ejYS7N2pJg==", + "license": "MIT", + "dependencies": { + "@opentelemetry/api": "^1.9.1", + "@opentelemetry/instrumentation": "^0.220.0", + "@opentelemetry/sdk-trace-base": "^2.9.0", + "@sentry/conventions": "^0.16.0", + "@sentry/core": "10.76.2", + "@sentry/node-core": "10.76.2", + "@sentry/opentelemetry": "10.76.2", + "@sentry/server-utils": "10.76.2", + "import-in-the-middle": "^3.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@sentry/node-core": { + "version": "10.76.2", + "resolved": "https://registry.npmjs.org/@sentry/node-core/-/node-core-10.76.2.tgz", + "integrity": "sha512-TrgAc/SvU5kQfjw8pz6I8LLIUiblOSf9BPPNkTbS0Dzcqa4yQLVbmsbS3rvGT7ot64m9Vjzn327ZRy63MlI6/w==", + "license": "MIT", + "dependencies": { + "@sentry/conventions": "^0.16.0", + "@sentry/core": "10.76.2", + "@sentry/opentelemetry": "10.76.2", + "import-in-the-middle": "^3.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/core": "^1.30.1 || ^2.1.0", + "@opentelemetry/exporter-trace-otlp-http": ">=0.57.0 <1", + "@opentelemetry/instrumentation": ">=0.57.1 <1", + "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/core": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-http": { + "optional": true + }, + "@opentelemetry/instrumentation": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + } + } + }, + "node_modules/@sentry/opentelemetry": { + "version": "10.76.2", + "resolved": "https://registry.npmjs.org/@sentry/opentelemetry/-/opentelemetry-10.76.2.tgz", + "integrity": "sha512-deqWrKJGaku3nhiZchg34z8W9v6kUwtzoQ4cmEq6CeT8/4X93X4YfeJgoh1dhbswFRpwor4pQ6kj0KttagKtfA==", + "license": "MIT", + "dependencies": { + "@sentry/conventions": "^0.16.0", + "@sentry/core": "10.76.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/core": "^1.30.1 || ^2.1.0", + "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0" + } + }, + "node_modules/@sentry/server-utils": { + "version": "10.76.2", + "resolved": "https://registry.npmjs.org/@sentry/server-utils/-/server-utils-10.76.2.tgz", + "integrity": "sha512-wogj6Trj8fAu+aD7HbpvHEBlyANMOvvO9MP5NFJaiQbprLKugtCdE6rFTxDhofXvRo4nswL9m4Cvysg3k5KcvQ==", + "license": "MIT", + "dependencies": { + "@sentry/conventions": "^0.16.0", + "@sentry/core": "10.76.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@sidvind/better-ajv-errors": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@sidvind/better-ajv-errors/-/better-ajv-errors-7.0.0.tgz", + "integrity": "sha512-imVsp5D3KxJ8+uUmu2dsLKnFg3qdX952v/L1gZoCa0NO2ivhQWHMpYM2KmpFrRXHWFjlqkNZ/935nxiTqXEi1A==", + "license": "Apache-2.0", + "engines": { + "node": "^22.12 || >= 24.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + } + }, + "node_modules/@types/node": { + "version": "26.6.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.5.tgz", + "integrity": "sha512-uMzsv9stDdofIZBx4UGh4Z9ZgXnGC9seRDscilBcTHAeOFk67MZg/k9bWVfO+bk4qZmG+l52hRnLwUiH2LnJTg==", + "license": "MIT", + "dependencies": { + "undici-types": "~8.9.0" + } + }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ansi-colors": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", + "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/atomically": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/atomically/-/atomically-2.1.1.tgz", + "integrity": "sha512-P4w9o2dqARji6P7MHprklbfiArZAWvo07yW7qs3pdljb3BWr12FIB7W+p0zJiuiVsUpRO0iZn1kFFcpPegg0tQ==", + "license": "MIT", + "dependencies": { + "stubborn-fs": "^2.0.0", + "when-exit": "^2.1.4" + } + }, + "node_modules/axe-core": { + "version": "4.14.0", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.14.0.tgz", + "integrity": "sha512-9WTZxEjsZ7b13TH8JPmbV2z8CHbl80/2hm3XPEG4JgNdQLK81IBRXmSxHfMAOkSqQeRxT/0dwNDz2GOm3zzpcQ==", + "license": "MPL-2.0", + "engines": { + "node": ">=4" + } + }, + "node_modules/chrome-launcher": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/chrome-launcher/-/chrome-launcher-1.2.2.tgz", + "integrity": "sha512-sI90u5qf3854bTrWsQMMdsQfSjjMbeSn7QtxVb9zrikf+QgMfHQ71KepHW7ZfixQKrDlAFKb+qnVlSR+xepBzw==", + "license": "Apache-2.0", + "dependencies": { + "@types/node": "*", + "escape-string-regexp": "^4.0.0", + "is-wsl": "^2.2.0", + "lighthouse-logger": "^2.0.1" + }, + "bin": { + "print-chrome-path": "bin/print-chrome-path.cjs" + }, + "engines": { + "node": ">=12.13.0" + } + }, + "node_modules/chromium-bidi": { + "version": "157.0.8090-0", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-157.0.8090-0.tgz", + "integrity": "sha512-KJw7IQgqXwxpJgJlEIPg5mr2ZoqA+jkTRtc5EABI7sUJ3bu3cpI7NuEBoj4Gn1AsF78CaT9So5WQWspK3/wYag==", + "license": "Apache-2.0", + "dependencies": { + "mitt": "^3.0.1", + "zod": "^4.6.5" + }, + "engines": { + "node": ">=20.19.0 <22.0.0 || >=22.12.0" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, + "node_modules/cjs-module-lexer": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-2.3.0.tgz", + "integrity": "sha512-lmNyBzi6iYiqyrExTctfmjd8tInx+dHu9PBomTfH5RXS53c27JMFJgzNv5TyAnm3PgGf+MLy0+Ws41/HlVxdKQ==", + "license": "MIT" + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/configstore": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/configstore/-/configstore-7.1.0.tgz", + "integrity": "sha512-N4oog6YJWbR9kGyXvS7jEykLDXIE2C0ILYqNBZBp9iwiJpoCBWYsuAdW6PPFn6w06jjnC+3JstVvWHO4cZqvRg==", + "license": "BSD-2-Clause", + "dependencies": { + "atomically": "^2.0.3", + "dot-prop": "^9.0.0", + "graceful-fs": "^4.2.11", + "xdg-basedir": "^5.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/csp_evaluator": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/csp_evaluator/-/csp_evaluator-1.1.8.tgz", + "integrity": "sha512-EwOnfYuNbTytvbMKsLixTrRgnjOa0WZCxGy8A9nnSYAicrdwn+T/epU/yjgymmOxlgKnvH+8wXt+7p/8ak5Feg==", + "license": "Apache-2.0" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decimal.js": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz", + "integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==", + "license": "MIT" + }, + "node_modules/define-lazy-prop": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz", + "integrity": "sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/devtools-protocol": { + "version": "0.0.1687809", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1687809.tgz", + "integrity": "sha512-t7kSb+UKYdyxcmqy98U2P86ne6rCve3DIJ4+lp54i/hhClgFuG35JrLj23rO14Jw3sWDsp7i+22QhNU0+/qk8g==", + "license": "BSD-3-Clause" + }, + "node_modules/dot-prop": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-9.0.0.tgz", + "integrity": "sha512-1gxPBJpI/pcjQhKgIU91II6Wkay+dLcN3M6rf2uwP8hRur3HtQXjVrdAK3sjC0piaEuxzMwjXChcETiJl47lAQ==", + "license": "MIT", + "dependencies": { + "type-fest": "^4.18.2" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/enquirer": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.4.1.tgz", + "integrity": "sha512-rRqJg/6gd538VHvR3PSrdRBb/1Vy2YfzHqzvbhGIQpDRKIa4FgV/54b5Q1xYSxOOwKvjXweS26E0Q+nAMwp2pQ==", + "license": "MIT", + "dependencies": { + "ansi-colors": "^4.1.1", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/es-module-lexer": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-3.0.3.tgz", + "integrity": "sha512-yWGe1O/j+almnV4hOYrmZo8u58JdnmXEMneRQAYkWtqSiT86yoMIEKF4rpqEDj52tbYl3xceBD3DQIBwcsC9Pw==", + "license": "MIT" + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", + "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-east-asian-width": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.7.0.tgz", + "integrity": "sha512-XjH1AECxf0giL2V1aU8vKyRR2ppRUb5c0EvT7zuJTokQ74bNo52zOtghqdWIqrhUD79fo3x0WfKZdOqxF6LG1Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/html-validate": { + "version": "11.16.2", + "resolved": "https://registry.npmjs.org/html-validate/-/html-validate-11.16.2.tgz", + "integrity": "sha512-oci8F4t1AQdAwkOK895hOOrMTSY0zZ6fk4p2LCvGKQDjL/CYDTGyIoVTzZWII0DfwA7gpq7+qJxAKubUofgo1Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/html-validate" + } + ], + "license": "MIT", + "dependencies": { + "@html-validate/stylish": "^6.0.0", + "@sidvind/better-ajv-errors": "7.0.0", + "ajv": "^8.0.0", + "kleur": "^4.1.0", + "prompts": "^2.0.0", + "semver": "^7.0.0" + }, + "bin": { + "html-validate": "bin/html-validate.mjs" + }, + "engines": { + "node": "^22.22.0 || >= 24.8.0" + }, + "peerDependencies": { + "@jest/globals": "^29.0.3 || ^30.0.0", + "@vitest/expect": "^3.2.0 || ^4.0.1 || ^5.0.0", + "jest": "^29.0.3 || ^30.0.0", + "jest-snapshot": "^29.0.3 || ^30.0.0", + "vitest": "^3.2.0 || ^4.0.1 || ^5.0.0" + }, + "peerDependenciesMeta": { + "@jest/globals": { + "optional": true + }, + "@vitest/expect": { + "optional": true + }, + "jest": { + "optional": true + }, + "jest-snapshot": { + "optional": true + }, + "vitest": { + "optional": true + } + } + }, + "node_modules/http-link-header": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/http-link-header/-/http-link-header-1.1.4.tgz", + "integrity": "sha512-xT3GPW6/ZbGuw4UvwHqErSCEjNUlwbQJuZn9/q5U4WEKfp2kENVCAlousG1zLxHeaQ/ffOHUNpWamvkbBW0eNw==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/image-ssim": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/image-ssim/-/image-ssim-0.2.0.tgz", + "integrity": "sha512-W7+sO6/yhxy83L0G7xR8YAc5Z5QFtYEXXRV6EaE8tuYBZJnA3gVgp3q7X7muhLZVodeb9UfvjSbwt9VJwjIYAg==", + "license": "MIT" + }, + "node_modules/import-in-the-middle": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-3.5.2.tgz", + "integrity": "sha512-WseIA/4o56+GYO3RmlOPvejYi771OFeJEeK/ejROsr9bdjW+E5NXTOD1Di8TL40Qgg3JJX9aa97Hf8iiSVIKZw==", + "license": "Apache-2.0", + "dependencies": { + "cjs-module-lexer": "^2.2.0", + "es-module-lexer": "^3.0.2", + "module-details-from-path": "^1.0.4" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/intl-messageformat": { + "version": "10.7.18", + "resolved": "https://registry.npmjs.org/intl-messageformat/-/intl-messageformat-10.7.18.tgz", + "integrity": "sha512-m3Ofv/X/tV8Y3tHXLohcuVuhWKo7BBq62cqY15etqmLxg2DZ34AGGgQDeR+SCta2+zICb1NX83af0GJmbQ1++g==", + "license": "BSD-3-Clause", + "dependencies": { + "@formatjs/ecma402-abstract": "2.3.6", + "@formatjs/fast-memoize": "2.2.7", + "@formatjs/icu-messageformat-parser": "2.11.4", + "tslib": "^2.8.0" + } + }, + "node_modules/is-docker": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", + "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-wsl": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "license": "MIT", + "dependencies": { + "is-docker": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/jpeg-js": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz", + "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==", + "license": "BSD-3-Clause" + }, + "node_modules/js-library-detector": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/js-library-detector/-/js-library-detector-6.7.0.tgz", + "integrity": "sha512-c80Qupofp43y4cJ7+8TTDN/AsDwLi5oOm/plBrWI+iQt485vKXCco+yVmOwEgdo9VOdsYTuV0UlTeetVPTriXA==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/legacy-javascript": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/legacy-javascript/-/legacy-javascript-0.0.3.tgz", + "integrity": "sha512-1r6MT4dVaJyXt4G07Cm2cQ0AUaLJdefiEp4SmIAylA++gnEQ82YvmzE58RkM5EP0fk5hROLCslUPGnwIYdZQVA==", + "license": "Apache-2.0" + }, + "node_modules/lighthouse": { + "version": "13.5.0", + "resolved": "https://registry.npmjs.org/lighthouse/-/lighthouse-13.5.0.tgz", + "integrity": "sha512-w/yYIwg7mP9pCwkcbAXRztnvKHua2dJnZaNptPLa1cEw+akr6lbsIDiUOLsQwvaiAUr0xvxdmwvAzl3pj3Np0A==", + "license": "Apache-2.0", + "dependencies": { + "@paulirish/trace_engine": "0.0.65", + "@sentry/node": "^10.0.0", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", + "axe-core": "^4.13.0", + "chrome-launcher": "^1.2.1", + "configstore": "^7.0.0", + "csp_evaluator": "1.1.8", + "devtools-protocol": "0.0.1687809", + "enquirer": "^2.3.6", + "http-link-header": "^1.1.1", + "intl-messageformat": "^10.5.3", + "jpeg-js": "^0.4.4", + "js-library-detector": "^6.7.0", + "lighthouse-logger": "^2.0.2", + "lighthouse-stack-packs": "1.12.3", + "lodash-es": "^4.17.21", + "lookup-closest-locale": "6.2.0", + "open": "^8.4.0", + "puppeteer-core": "^25.9.0", + "robots-parser": "^3.0.1", + "speedline-core": "^1.4.3", + "third-party-web": "^0.30.0", + "tldts-icann": "^7.4.13", + "web-features": "^3.39.0", + "ws": "^7.0.0", + "yargs": "^17.3.1", + "yargs-parser": "^21.0.0" + }, + "bin": { + "chrome-debug": "core/scripts/manual-chrome-launcher.js", + "lighthouse": "cli/index.js", + "smokehouse": "cli/test/smokehouse/frontends/smokehouse-bin.js" + }, + "engines": { + "node": ">=22.19" + } + }, + "node_modules/lighthouse-logger": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/lighthouse-logger/-/lighthouse-logger-2.0.2.tgz", + "integrity": "sha512-vWl2+u5jgOQuZR55Z1WM0XDdrJT6mzMP8zHUct7xTlWhuQs+eV0g+QL0RQdFjT54zVmbhLCP8vIVpy1wGn/gCg==", + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.4.1", + "marky": "^1.2.2" + } + }, + "node_modules/lighthouse-stack-packs": { + "version": "1.12.3", + "resolved": "https://registry.npmjs.org/lighthouse-stack-packs/-/lighthouse-stack-packs-1.12.3.tgz", + "integrity": "sha512-d8IsOpE83kbANgnM+Tp8+x6HcMpX9o2ITBiUERssgzAIFdZCQzs/f4k6D0DLQTE59enml9mbAOU52Wu35exWtg==", + "license": "Apache-2.0" + }, + "node_modules/lodash-es": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.18.1.tgz", + "integrity": "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==", + "license": "MIT" + }, + "node_modules/lookup-closest-locale": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/lookup-closest-locale/-/lookup-closest-locale-6.2.0.tgz", + "integrity": "sha512-/c2kL+Vnp1jnV6K6RpDTHK3dgg0Tu2VVp+elEiJpjfS1UyY7AjOYHohRug6wT0OpoX2qFgNORndE9RqesfVxWQ==", + "license": "MIT" + }, + "node_modules/marky": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/marky/-/marky-1.3.0.tgz", + "integrity": "sha512-ocnPZQLNpvbedwTy9kNrQEsknEfgvcLMvOtz3sFeWApDq1MXH1TqkCIx58xlpESsfwQOnuBO9beyQuNGzVvuhQ==", + "license": "Apache-2.0" + }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "license": "MIT" + }, + "node_modules/modern-tar": { + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/modern-tar/-/modern-tar-0.8.5.tgz", + "integrity": "sha512-snEhs+6G5Tjd4I7tLCDOaoln2RgE0bD19RzEKgvgK2hZ5VKy3MpLhLTZ2fWpXSTg4K2cyPwp+VHATFJhxfnOeA==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/module-details-from-path": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz", + "integrity": "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==", + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/open": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/open/-/open-8.4.2.tgz", + "integrity": "sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==", + "license": "MIT", + "dependencies": { + "define-lazy-prop": "^2.0.0", + "is-docker": "^2.1.1", + "is-wsl": "^2.2.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/playwright": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.64.0.tgz", + "integrity": "sha512-kVzTnYFYEyQ9RGzBjNCd2+yEqWmVbxl3AHPynv3qQ6JTjv4a+kwRcnqQwIpuP3NnrnIrE/Q6wWAIAvMfIE2/Yg==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.64.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.64.0.tgz", + "integrity": "sha512-T9r+MZkTECl2+oUcZ26YgZzTrYidTsIhYYkJix+6iDOymOb1FB2RLLSOxRGnZGv5DStt+aduIVbt+E8WygXYpg==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/prompts": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", + "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", + "license": "MIT", + "dependencies": { + "kleur": "^3.0.3", + "sisteransi": "^1.0.5" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/prompts/node_modules/kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/puppeteer-core": { + "version": "25.13.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-25.13.0.tgz", + "integrity": "sha512-cyBS0OEx3e9NS53B5F6+UazXtR7f9Tw0cR1/FbPW59/hV6JPZbkIiMnxK2543IkFp3rsGlLn1iPDLF77TJsJsA==", + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "3.2.4", + "chromium-bidi": "157.0.8090-0", + "devtools-protocol": "0.0.1696802", + "typed-query-selector": "^2.12.3", + "webdriver-bidi-protocol": "1.0.1", + "ws": "^8.21.3" + }, + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/puppeteer-core/node_modules/devtools-protocol": { + "version": "0.0.1696802", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1696802.tgz", + "integrity": "sha512-FO5/SENfvc2K8wRnWQflKcwuoTH4dBkXCWTQ9ebIWtuX8JM5+ZvCotaEmkfh21pO7TvIKGTM3hG4xFiq9WTwmg==", + "license": "BSD-3-Clause" + }, + "node_modules/puppeteer-core/node_modules/ws": { + "version": "8.22.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.22.0.tgz", + "integrity": "sha512-Ydggc987+RO0AnWtZ/7Wq9FtNvcrL1b/RO0ud9mWjUPgDrsAAwQSF51sm2hm1XofbU/4jkpGEsLFsZZxU+1DOg==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-in-the-middle": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-8.0.1.tgz", + "integrity": "sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.3.5", + "module-details-from-path": "^1.0.3" + }, + "engines": { + "node": ">=9.3.0 || >=8.10.0 <9.0.0" + } + }, + "node_modules/robots-parser": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/robots-parser/-/robots-parser-3.0.1.tgz", + "integrity": "sha512-s+pyvQeIKIZ0dx5iJiQk1tPLJAWln39+MI5jtM8wnyws+G5azk+dMnMX0qfbqNetKKNgcWWOdi0sfm+FbQbgdQ==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sisteransi": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", + "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", + "license": "MIT" + }, + "node_modules/speedline-core": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/speedline-core/-/speedline-core-1.4.3.tgz", + "integrity": "sha512-DI7/OuAUD+GMpR6dmu8lliO2Wg5zfeh+/xsdyJZCzd8o5JgFUjCeLsBDuZjIQJdwXS3J0L/uZYrELKYqx+PXog==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "image-ssim": "^0.2.0", + "jpeg-js": "^0.4.1" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/stubborn-fs": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/stubborn-fs/-/stubborn-fs-2.0.0.tgz", + "integrity": "sha512-Y0AvSwDw8y+nlSNFXMm2g6L51rBGdAQT20J3YSOqxC53Lo3bjWRtr2BKcfYoAf352WYpsZSTURrA0tqhfgudPA==", + "license": "MIT", + "dependencies": { + "stubborn-utils": "^1.0.1" + } + }, + "node_modules/stubborn-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/stubborn-utils/-/stubborn-utils-1.0.2.tgz", + "integrity": "sha512-zOh9jPYI+xrNOyisSelgym4tolKTJCQd5GBhK0+0xJvcYDcwlOoxF/rnFKQ2KRZknXSG9jWAp66fwP6AxN9STg==", + "license": "MIT" + }, + "node_modules/third-party-web": { + "version": "0.30.0", + "resolved": "https://registry.npmjs.org/third-party-web/-/third-party-web-0.30.0.tgz", + "integrity": "sha512-p+PfyL5U0ediGzvwPzMSdD9MnC8rHW6nWJJVLIQvqN7RsSqfaZ+7n4c9GILqH36EYR0eXB9CzAeCRnKFr/9uJg==", + "license": "MIT" + }, + "node_modules/tldts-core": { + "version": "7.4.18", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.4.18.tgz", + "integrity": "sha512-KI2PKvCmvQI/xD6ypPqkU0i7oxNXR/O58RLRaHR4svsgVUQ2JE/XC8BtMbG6PSehD27I8sZJRcC1ASnCy6lp3g==", + "license": "MIT" + }, + "node_modules/tldts-icann": { + "version": "7.4.18", + "resolved": "https://registry.npmjs.org/tldts-icann/-/tldts-icann-7.4.18.tgz", + "integrity": "sha512-+29GkEj4qOTb8oQRzg7Lxe1TQdkIDFvyqVTcu775faf9iLczNTix8L/wDFiQi4/Ncib7ZjyWPcVf0iRQCrmUjg==", + "license": "MIT", + "dependencies": { + "tldts-core": "^7.4.18" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/type-fest": { + "version": "4.41.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", + "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/typed-query-selector": { + "version": "2.12.3", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.3.tgz", + "integrity": "sha512-/dtuMZyutHLlrxWhhC1uib00ELUr2dZ8C9B8YzZYsJmeE3V56OxRblrSM2UaF142pcMtak7ehZ6+poWGeehNIg==", + "license": "MIT" + }, + "node_modules/undici-types": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", + "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", + "license": "MIT" + }, + "node_modules/web-features": { + "version": "3.42.0", + "resolved": "https://registry.npmjs.org/web-features/-/web-features-3.42.0.tgz", + "integrity": "sha512-Vq97bzyedg6EuPFnz1GIJqsI9me8CbxzwYrcBI/cNwjj66v77IsGmIZi9SxPHTu2t25F7ta4kZJPu4qvuYlhfQ==", + "license": "Apache-2.0" + }, + "node_modules/webdriver-bidi-protocol": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-1.0.1.tgz", + "integrity": "sha512-WXWs/aXF2a6Al1Z2/V/7Gp2oPZOAAmHjJOTmv4KCXRJ122IMj3tSSMc6a4q3K4QOmhQbse/Ahr+oTVJOU2dUiw==", + "license": "Apache-2.0" + }, + "node_modules/when-exit": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/when-exit/-/when-exit-2.1.5.tgz", + "integrity": "sha512-VGkKJ564kzt6Ms1dbgPP/yuIoQCrsFAnRbptpC5wOEsDaNsbCB2bnfnaA8i/vRs5tjUSEOtIuvl9/MyVsvQZCg==", + "license": "MIT" + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/ws": { + "version": "7.5.13", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.13.tgz", + "integrity": "sha512-rsKI6xDBFVf4r/x8XyChGK04QR/XHroxs/jUcoWvtEZM8TPU/X/uIY9B1CsSzYws9ZJb/6bbBu7dPhFW00CAoA==", + "license": "MIT", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xdg-basedir": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-5.1.0.tgz", + "integrity": "sha512-GCPAHLvrIH13+c0SuacwvRYj2SxJXQ4kaVTT5xgL3kPrz56XxkF21IGhjSE1+W0aw7gpBWRGXLCPnPby6lSpmQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz", + "integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/quality/package.json b/quality/package.json new file mode 100644 index 0000000..bc1c618 --- /dev/null +++ b/quality/package.json @@ -0,0 +1,13 @@ +{ + "name": "ssgberk-quality", + "private": true, + "type": "module", + "description": "SSGBerk output quality collector (benchmark-tool spec 009)", + "dependencies": { + "axe-core": "4.14.0", + "chrome-launcher": "1.2.2", + "html-validate": "11.16.2", + "lighthouse": "13.5.0", + "playwright": "1.64.0" + } +} diff --git a/quality/run.mjs b/quality/run.mjs new file mode 100644 index 0000000..8870679 --- /dev/null +++ b/quality/run.mjs @@ -0,0 +1,284 @@ +// SSGBerk quality collector (benchmark-tool spec 009). Writes raw data only: +// toolset/quality/reduce.py turns raw.json into quality.json. +import { execFile } from 'node:child_process'; +import fs from 'node:fs/promises'; +import http from 'node:http'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs, promisify } from 'node:util'; +import zlib from 'node:zlib'; + +import * as chromeLauncher from 'chrome-launcher'; +import { HtmlValidate } from 'html-validate'; +import lighthouse from 'lighthouse'; +import desktopConfig from 'lighthouse/core/config/desktop-config.js'; +import { chromium } from 'playwright'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const RUNS = 3; +const PRESETS = { mobile: undefined, desktop: desktopConfig }; +const CATEGORIES = ['performance', 'accessibility', 'best-practices', 'seo']; +const CHROME_FLAGS = ['--headless=new', '--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage']; +const NOT_SCORED = new Set(['informative', 'manual', 'notApplicable', 'error']); +const TYPES = { + '.html': 'text/html; charset=utf-8', '.css': 'text/css; charset=utf-8', + '.js': 'text/javascript; charset=utf-8', '.mjs': 'text/javascript; charset=utf-8', + '.json': 'application/json', '.xml': 'application/xml', '.txt': 'text/plain; charset=utf-8', + '.svg': 'image/svg+xml', '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', + '.webp': 'image/webp', '.gif': 'image/gif', '.ico': 'image/x-icon', '.woff': 'font/woff', + '.woff2': 'font/woff2', '.wasm': 'application/wasm', +}; +const COMPRESSIBLE = /^(text\/|application\/(json|xml|wasm)|image\/svg\+xml)/; +const execFileP = promisify(execFile); + +async function guard(fn) { + try { + return await fn(); + } catch (e) { + return { error: String(e && e.message ? e.message : e) }; + } +} + +async function isFile(p) { + try { + return (await fs.stat(p)).isFile(); + } catch { + return false; + } +} + +async function resolveFile(root, pathname) { + const rel = decodeURIComponent(pathname).replace(/^\/+/, ''); + const base = path.resolve(root, rel); + if (base !== root && !base.startsWith(root + path.sep)) return null; + const candidates = rel === '' || rel.endsWith('/') + ? [path.join(base, 'index.html')] + : [base, path.join(base, 'index.html'), `${base}.html`]; + for (const c of candidates) if (await isFile(c)) return c; + return null; +} + +// Fixed static server: gzip, no-cache, directory index, 404.html with status 404. +function serve(root) { + const server = http.createServer(async (req, res) => { + const { pathname } = new URL(req.url, 'http://127.0.0.1'); + let file = await resolveFile(root, pathname); + let status = 200; + if (!file) { + status = 404; + file = (await isFile(path.join(root, '404.html'))) ? path.join(root, '404.html') : null; + } + let body = file ? await fs.readFile(file) : Buffer.from('Not found'); + const type = file ? TYPES[path.extname(file).toLowerCase()] || 'application/octet-stream' + : 'text/plain; charset=utf-8'; + const headers = { 'content-type': type, 'cache-control': 'no-cache' }; + if (COMPRESSIBLE.test(type) && /\bgzip\b/.test(req.headers['accept-encoding'] || '')) { + body = zlib.gzipSync(body); + headers['content-encoding'] = 'gzip'; + headers.vary = 'accept-encoding'; + } + headers['content-length'] = body.length; + res.writeHead(status, headers); + res.end(req.method === 'HEAD' ? undefined : body); + }); + return new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve({ + server, origin: `http://127.0.0.1:${server.address().port}`, + }))); +} + +function summarize(lhr) { + const audits = lhr.audits; + const value = (id) => audits[id]?.numericValue ?? null; + const requests = audits['network-requests']?.details?.items ?? []; + const bytes = (type) => requests.filter((r) => r.resourceType === type) + .reduce((sum, r) => sum + (r.transferSize || 0), 0); + return { + scores: Object.fromEntries(CATEGORIES.map((c) => [c, lhr.categories[c]?.score ?? null])), + metrics: { + fcp: value('first-contentful-paint'), + lcp: value('largest-contentful-paint'), + tbt: value('total-blocking-time'), + cls: value('cumulative-layout-shift'), + si: value('speed-index'), + }, + resources: { + totalBytes: value('total-byte-weight'), + jsBytes: bytes('Script'), + cssBytes: bytes('Stylesheet'), + requests: requests.length, + domSize: value('dom-size') ?? value('dom-size-insight'), + }, + failedAudits: Object.values(audits) + .filter((a) => !NOT_SCORED.has(a.scoreDisplayMode) && a.score !== null && a.score < 1) + .map((a) => a.id).sort(), + }; +} + +async function runLighthouse(origin, pages) { + const chrome = await chromeLauncher.launch({ + chromePath: chromium.executablePath(), chromeFlags: CHROME_FLAGS, + }); + try { + const out = {}; + for (const [preset, config] of Object.entries(PRESETS)) { + out[preset] = {}; + for (const [key, page] of Object.entries(pages)) { + const runs = []; + for (let i = 0; i < RUNS; i += 1) { + runs.push(await guard(async () => { + const result = await lighthouse(origin + page.url, { + port: chrome.port, output: 'json', logLevel: 'error', onlyCategories: CATEGORIES, + }, config); + if (!result?.lhr) throw new Error('lighthouse returned no report'); + if (result.lhr.runtimeError) throw new Error(result.lhr.runtimeError.message); + return summarize(result.lhr); + })); + } + out[preset][key] = runs; + } + } + return out; + } finally { + await chrome.kill(); + } +} + +async function runBrowserChecks(origin, pages) { + const axeSource = await fs.readFile(path.join(HERE, 'node_modules', 'axe-core', 'axe.min.js'), 'utf8'); + const browser = await chromium.launch({ args: ['--no-sandbox', '--disable-dev-shm-usage'] }); + try { + const axe = {}; + for (const [key, page] of Object.entries(pages)) { + axe[key] = await guard(async () => { + const tab = await browser.newPage(); + try { + await tab.goto(origin + page.url, { waitUntil: 'load' }); + await tab.addScriptTag({ content: axeSource }); + const result = await tab.evaluate(() => window.axe.run(document, { resultTypes: ['violations'] })); + return { + violations: result.violations.map((v) => ({ id: v.id, impact: v.impact, nodes: v.nodes.length })), + }; + } finally { + await tab.close(); + } + }); + } + const post = await guard(async () => { + const tab = await browser.newPage(); + try { + await tab.goto(origin + pages.post.url, { waitUntil: 'networkidle' }); + const text = async (selector) => ((await tab.locator(selector).first() + .textContent({ timeout: 5000 })) || '').trim(); + return { title: await text('h1.post-title'), firstParagraph: await text('.post-body p') }; + } finally { + await tab.close(); + } + }); + return { axe, rendered: { post }, chromium: browser.version() }; + } finally { + await browser.close(); + } +} + +async function runHtmlValidate(site, pages) { + const validator = new HtmlValidate({ extends: ['html-validate:recommended'] }); + const out = {}; + for (const [key, page] of Object.entries(pages)) { + out[key] = await guard(async () => { + const report = await validator.validateFile(path.join(site, page.file)); + return { + messages: report.results.flatMap((r) => r.messages) + .map((m) => ({ ruleId: m.ruleId, severity: m.severity })), + }; + }); + } + return out; +} + +async function runLychee(site, outDir) { + const report = path.join(outDir, 'lychee.json'); + try { + await execFileP('lychee', ['--offline', '--no-progress', '--include-fragments', '--index-files', 'index.html', + '--format', 'json', '--output', report, '--root-dir', site, site], { maxBuffer: 64 * 1024 * 1024 }); + } catch (e) { + // exit code 2 means broken links; the report is still written + if (!(await isFile(report))) throw e; + } + return { report: JSON.parse(await fs.readFile(report, 'utf8')) }; +} + +async function listFiles(dir) { + const out = []; + for (const entry of await fs.readdir(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) out.push(...await listFiles(full)); + else if (entry.isFile()) out.push(full); + } + return out; +} + +async function sizes(site) { + const files = []; + for (const full of (await listFiles(site)).sort()) { + const data = await fs.readFile(full); + files.push({ + path: path.relative(site, full).split(path.sep).join('/'), + bytes: data.length, + gzip: zlib.gzipSync(data, { level: 9 }).length, + br: zlib.brotliCompressSync(data, { + params: { [zlib.constants.BROTLI_PARAM_QUALITY]: 11 }, + }).length, + }); + } + return files; +} + +async function packageVersion(name) { + const file = path.join(HERE, 'node_modules', name, 'package.json'); + return JSON.parse(await fs.readFile(file, 'utf8')).version; +} + +async function tools(chromiumVersion) { + const lychee = await guard(async () => (await execFileP('lychee', ['--version'])).stdout.trim() + .split(/\s+/).pop()); + return { + node: process.version, + lighthouse: await packageVersion('lighthouse'), + playwright: await packageVersion('playwright'), + chromium: chromiumVersion ?? null, + axeCore: await packageVersion('axe-core'), + htmlValidate: await packageVersion('html-validate'), + chromeLauncher: await packageVersion('chrome-launcher'), + lychee: typeof lychee === 'string' ? lychee : null, + }; +} + +async function main() { + const { values } = parseArgs({ + options: { site: { type: 'string' }, pages: { type: 'string' }, out: { type: 'string' } }, + }); + const site = path.resolve(values.site); + const pages = JSON.parse(values.pages); + const outDir = path.resolve(values.out); + await fs.mkdir(outDir, { recursive: true }); + const raw = { pages }; + const { server, origin } = await serve(site); + try { + const browser = await guard(() => runBrowserChecks(origin, pages)); + raw.axe = browser.error ? browser : browser.axe; + raw.rendered = browser.error ? browser : browser.rendered; + raw.lighthouse = await guard(() => runLighthouse(origin, pages)); + raw.html = await guard(() => runHtmlValidate(site, pages)); + raw.links = await guard(() => runLychee(site, outDir)); + raw.files = await guard(() => sizes(site)); + raw.tools = await guard(() => tools(browser.chromium)); + } finally { + server.close(); + } + await fs.writeFile(path.join(outDir, 'raw.json'), JSON.stringify(raw, null, 2)); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/quality/test/check-shape.mjs b/quality/test/check-shape.mjs new file mode 100644 index 0000000..3cba4d7 --- /dev/null +++ b/quality/test/check-shape.mjs @@ -0,0 +1,42 @@ +// Checks a collector raw.json against expected-shape.json (keys and types, not values). +import fs from 'node:fs'; + +const [rawPath, shapePath] = process.argv.slice(2); +const raw = JSON.parse(fs.readFileSync(rawPath, 'utf8')); +const shape = JSON.parse(fs.readFileSync(shapePath, 'utf8')); +const errors = []; + +function typeOf(v) { + if (v === null) return 'null'; + if (Array.isArray(v)) return 'array'; + return typeof v; +} + +function check(value, expected, where) { + if (typeof expected === 'string') { + if (!expected.split('|').includes(typeOf(value))) errors.push(`${where}: ${typeOf(value)} is not ${expected}`); + } else if (Array.isArray(expected)) { + if (!Array.isArray(value) || value.length === 0) errors.push(`${where}: expected a non-empty array`); + else value.forEach((v, i) => check(v, expected[0], `${where}[${i}]`)); + } else { + if (typeOf(value) !== 'object') { + errors.push(`${where}: expected an object, got ${JSON.stringify(value).slice(0, 200)}`); + return; + } + for (const [k, e] of Object.entries(expected)) check(value[k], e, `${where}.${k}`); + } +} + +check(raw, shape, 'raw'); +for (const preset of ['mobile', 'desktop']) { + for (const page of ['index', 'post', '404']) { + if ((raw.lighthouse?.[preset]?.[page] || []).length !== 3) errors.push(`lighthouse.${preset}.${page}: expected 3 runs`); + } +} +if (raw.rendered?.post?.title !== 'Hello') errors.push('rendered.post.title is not "Hello"'); +if (raw.lighthouse?.mobile?.post?.[0]?.resources?.jsBytes !== 0) errors.push('fixture post page should request 0 JS bytes'); +if (errors.length) { + console.error(errors.join('\n')); + process.exit(1); +} +console.log('raw.json shape ok'); diff --git a/quality/test/expected-shape.json b/quality/test/expected-shape.json new file mode 100644 index 0000000..5ac2df6 --- /dev/null +++ b/quality/test/expected-shape.json @@ -0,0 +1,208 @@ +{ + "tools": { + "node": "string", + "lighthouse": "string", + "playwright": "string", + "chromium": "string", + "axeCore": "string", + "htmlValidate": "string", + "chromeLauncher": "string", + "lychee": "string" + }, + "pages": "object", + "lighthouse": { + "mobile": { + "index": [ + { + "scores": { + "performance": "number", + "accessibility": "number", + "best-practices": "number", + "seo": "number" + }, + "metrics": { + "fcp": "number", + "lcp": "number", + "tbt": "number", + "cls": "number", + "si": "number" + }, + "resources": { + "totalBytes": "number", + "jsBytes": "number", + "cssBytes": "number", + "requests": "number", + "domSize": "number|null" + }, + "failedAudits": "array" + } + ], + "post": [ + { + "scores": { + "performance": "number", + "accessibility": "number", + "best-practices": "number", + "seo": "number" + }, + "metrics": { + "fcp": "number", + "lcp": "number", + "tbt": "number", + "cls": "number", + "si": "number" + }, + "resources": { + "totalBytes": "number", + "jsBytes": "number", + "cssBytes": "number", + "requests": "number", + "domSize": "number|null" + }, + "failedAudits": "array" + } + ], + "404": [ + { + "scores": { + "performance": "number", + "accessibility": "number", + "best-practices": "number", + "seo": "number" + }, + "metrics": { + "fcp": "number", + "lcp": "number", + "tbt": "number", + "cls": "number", + "si": "number" + }, + "resources": { + "totalBytes": "number", + "jsBytes": "number", + "cssBytes": "number", + "requests": "number", + "domSize": "number|null" + }, + "failedAudits": "array" + } + ] + }, + "desktop": { + "index": [ + { + "scores": { + "performance": "number", + "accessibility": "number", + "best-practices": "number", + "seo": "number" + }, + "metrics": { + "fcp": "number", + "lcp": "number", + "tbt": "number", + "cls": "number", + "si": "number" + }, + "resources": { + "totalBytes": "number", + "jsBytes": "number", + "cssBytes": "number", + "requests": "number", + "domSize": "number|null" + }, + "failedAudits": "array" + } + ], + "post": [ + { + "scores": { + "performance": "number", + "accessibility": "number", + "best-practices": "number", + "seo": "number" + }, + "metrics": { + "fcp": "number", + "lcp": "number", + "tbt": "number", + "cls": "number", + "si": "number" + }, + "resources": { + "totalBytes": "number", + "jsBytes": "number", + "cssBytes": "number", + "requests": "number", + "domSize": "number|null" + }, + "failedAudits": "array" + } + ], + "404": [ + { + "scores": { + "performance": "number", + "accessibility": "number", + "best-practices": "number", + "seo": "number" + }, + "metrics": { + "fcp": "number", + "lcp": "number", + "tbt": "number", + "cls": "number", + "si": "number" + }, + "resources": { + "totalBytes": "number", + "jsBytes": "number", + "cssBytes": "number", + "requests": "number", + "domSize": "number|null" + }, + "failedAudits": "array" + } + ] + } + }, + "axe": { + "index": { + "violations": "array" + }, + "post": { + "violations": "array" + }, + "404": { + "violations": "array" + } + }, + "rendered": { + "post": { + "title": "string", + "firstParagraph": "string" + } + }, + "html": { + "index": { + "messages": "array" + }, + "post": { + "messages": "array" + }, + "404": { + "messages": "array" + } + }, + "links": { + "report": "object" + }, + "files": [ + { + "path": "string", + "bytes": "number", + "gzip": "number", + "br": "number" + } + ] +} diff --git a/quality/test/pages.json b/quality/test/pages.json new file mode 100644 index 0000000..af21b59 --- /dev/null +++ b/quality/test/pages.json @@ -0,0 +1 @@ +{"index": {"url": "/", "file": "index.html"}, "post": {"url": "/post/hello/", "file": "post/hello/index.html"}, "404": {"url": "/404.html", "file": "404.html"}} From ac26714411b6c548d4747c9fc9f9e92333b5d891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matheus=20Bregu=C3=AAz?= Date: Sat, 10 Oct 2026 14:59:39 -0300 Subject: [PATCH 12/15] fix(quality): collector static server never crashes on a bad request (spec R-16) A malformed percent-escape or a read error now answers 404/500 instead of an unhandled rejection that killed the process before raw.json was written. Adds quality/test/bad-url.mjs as a regression check. Co-Authored-By: Claude Opus 5.5 (1M context) --- quality/run.mjs | 18 ++++++++++++++++-- quality/test/bad-url.mjs | 10 ++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 quality/test/bad-url.mjs diff --git a/quality/run.mjs b/quality/run.mjs index 8870679..a9e7fe2 100644 --- a/quality/run.mjs +++ b/quality/run.mjs @@ -48,7 +48,12 @@ async function isFile(p) { } async function resolveFile(root, pathname) { - const rel = decodeURIComponent(pathname).replace(/^\/+/, ''); + let rel; + try { + rel = decodeURIComponent(pathname).replace(/^\/+/, ''); + } catch { + return null; + } const base = path.resolve(root, rel); if (base !== root && !base.startsWith(root + path.sep)) return null; const candidates = rel === '' || rel.endsWith('/') @@ -60,7 +65,7 @@ async function resolveFile(root, pathname) { // Fixed static server: gzip, no-cache, directory index, 404.html with status 404. function serve(root) { - const server = http.createServer(async (req, res) => { + const handle = async (req, res) => { const { pathname } = new URL(req.url, 'http://127.0.0.1'); let file = await resolveFile(root, pathname); let status = 200; @@ -80,6 +85,15 @@ function serve(root) { headers['content-length'] = body.length; res.writeHead(status, headers); res.end(req.method === 'HEAD' ? undefined : body); + }; + // A failing request must never take the collector down (spec R-16). + const server = http.createServer(async (req, res) => { + try { + await handle(req, res); + } catch (e) { + if (!res.headersSent) res.writeHead(500, { 'content-type': 'text/plain; charset=utf-8' }); + res.end('Internal error'); + } }); return new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve({ server, origin: `http://127.0.0.1:${server.address().port}`, diff --git a/quality/test/bad-url.mjs b/quality/test/bad-url.mjs new file mode 100644 index 0000000..a1f599b --- /dev/null +++ b/quality/test/bad-url.mjs @@ -0,0 +1,10 @@ +// Regression check: a malformed percent-escape in a page URL must not crash the collector. +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; + +const pages = { post: { url: '/%E0%A4%A/', file: 'post/hello/index.html' } }; +const r = spawnSync('node', ['/quality/run.mjs', '--site', '/fixture', '--pages', JSON.stringify(pages), '--out', '/tmp/bad'], + { encoding: 'utf8' }); +if (r.status !== 0) { console.error(`collector exited ${r.status}\n${r.stderr}`); process.exit(1); } +if (!fs.existsSync('/tmp/bad/raw.json')) { console.error('raw.json not written'); process.exit(1); } +console.log('bad url survived, raw.json written'); From 5a712bef7432ecc72f38ae819d6c77c9de1099e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matheus=20Bregu=C3=AAz?= Date: Sat, 10 Oct 2026 15:08:58 -0300 Subject: [PATCH 13/15] fix(quality): isolate collector out dir, make the pass unable to affect the build, CI checks Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/ci.yml | 5 ++- docs/specs/009-output-quality/spec.md | 2 +- quality/Dockerfile | 2 +- tests/test_quality_docker.py | 19 ++++++++++- tests/test_quality_runner.py | 49 +++++++++++++++++++++++++++ toolset/benchmark/benchmarker.py | 35 ++++++++++++++----- toolset/quality/runner.py | 17 +++++++--- toolset/utils/docker_helper.py | 7 ++-- 8 files changed, 117 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99eab51..bb8ed45 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,8 @@ jobs: docker run --rm --network none \ -v "$PWD/tests/fixtures/quality/site:/fixture:ro" -v "$PWD/quality/test:/test:ro" \ ssgberk/quality sh -c 'node /quality/run.mjs --site /fixture --pages "$(cat /test/pages.json)" --out /tmp/out \ - && node /test/check-shape.mjs /tmp/out/raw.json /test/expected-shape.json' + && node /test/check-shape.mjs /tmp/out/raw.json /test/expected-shape.json \ + && node /test/bad-url.mjs' quality: needs: quality-image @@ -68,6 +69,8 @@ jobs: for s in lighthouse seo html a11y links weight js extra; do jq -e --arg s "$s" '.quality.hugo[$s].status == "ok"' "$R/results.json" done + jq -e '[.quality.hugo.lighthouse.mobile[], .quality.hugo.lighthouse.desktop[]] + | length == 6 and all(.status == "ok")' "$R/results.json" jq -e '[.quality.hugo.lighthouse.mobile[], .quality.hugo.lighthouse.desktop[] | .scores[] | .median | select(. < 0 or . > 1)] | length == 0' "$R/results.json" jq -e '.environment.quality.lighthouse and .environment.quality.chromium' "$R/results.json" diff --git a/docs/specs/009-output-quality/spec.md b/docs/specs/009-output-quality/spec.md index 3a2a5aa..305eadd 100644 --- a/docs/specs/009-output-quality/spec.md +++ b/docs/specs/009-output-quality/spec.md @@ -1,7 +1,7 @@ # 009 Output Quality - **Date:** 2026-10-10 -- **Status:** proposed, awaiting review +- **Status:** implemented - **Repo:** `ssgberk/benchmark-tool` (BT). No change to `ssgberk/ssg-frameworks` (SF). - **Issue:** #22 - **Related:** BT `008-benchmark-methodology` (suites, `results.json`, environment fingerprint), BT `006-results-summary` (`summary.md`), SF `005-reference-site-design` (what is built, R-14 page set), SF `006-layout-conformance` (post page resolved from the index). diff --git a/quality/Dockerfile b/quality/Dockerfile index be9d1f0..749b6a8 100644 --- a/quality/Dockerfile +++ b/quality/Dockerfile @@ -29,5 +29,5 @@ RUN npm ci --omit=dev \ && npx playwright install --with-deps chromium \ && rm -rf /root/.npm /var/lib/apt/lists/* COPY run.mjs ./ -RUN mkdir -p /work/out +RUN mkdir -p /work /quality-out WORKDIR /work diff --git a/tests/test_quality_docker.py b/tests/test_quality_docker.py index 945e9df..bb89c6b 100644 --- a/tests/test_quality_docker.py +++ b/tests/test_quality_docker.py @@ -96,10 +96,27 @@ def test_run_quality_isolated_and_cleaned_up(tmp_path): assert kwargs["command"][:4] == ["node", "/quality/run.mjs", "--site", "/work/public"] assert json.loads(kwargs["command"][kwargs["command"].index("--pages") + 1]) == pages c.put_archive.assert_called_once_with("/work", b"site-tar") - c.get_archive.assert_called_once_with("/work/out/raw.json") + c.get_archive.assert_called_once_with("/quality-out/raw.json") c.remove.assert_called_once_with(force=True) +def test_run_quality_out_dir_never_inside_site(tmp_path): + helper = _helper() + helper._quality_image = "sha256:q" + c = mock.Mock() + c.wait.return_value = {"StatusCode": 0} + c.get_archive.return_value = (iter([_raw_tar({})]), {}) + helper.server.containers.create.return_value = c + (tmp_path / "site.tar").write_bytes(b"x") + helper.run_quality(str(tmp_path / "site.tar"), "out", {}) + command = helper.server.containers.create.call_args[1]["command"] + site = command[command.index("--site") + 1] + out = command[command.index("--out") + 1] + assert site == "/work/out" + assert out != site and not out.startswith(site + "/") + assert out == docker_helper.QUALITY_OUT and not out.startswith("/work") + + def test_run_quality_without_raw_json_raises_and_cleans_up(tmp_path): helper = _helper() helper._quality_image = "sha256:q" diff --git a/tests/test_quality_runner.py b/tests/test_quality_runner.py index 5cd3794..fa7444a 100644 --- a/tests/test_quality_runner.py +++ b/tests/test_quality_runner.py @@ -168,3 +168,52 @@ def test_reparse_keeps_quality(fake_benchmarker, monkeypatch): res.reparse([]) with open(res.file) as f: assert json.load(f)["quality"] == {"hugo": {"status": "ok"}} + + +def test_run_pass_write_failure_still_returns(tmp_path, monkeypatch): + test, results_dir = _generator(tmp_path), tmp_path / "results" + tar_path = _archive(results_dir) + helper = mock.Mock() + helper.run_quality.return_value = json.loads((FIX / "raw.json").read_text()) + real_open = open + + def fake_open(path, *a, **kw): + if str(path).endswith("quality.json"): + raise OSError("disk full") + return real_open(path, *a, **kw) + + monkeypatch.setattr("builtins.open", fake_open) + q = runner.run_pass(helper, test, str(results_dir)) + assert isinstance(q, dict) and q["status"] == "ok" + assert not tar_path.exists() + + +def test_quality_failure_never_changes_build_result(tmp_path, monkeypatch): + def boom(*a): + raise RuntimeError("kaboom") + + outcomes = [] + for quality in (False, True): + b, ft, _ = _bm(tmp_path / str(quality), quality=quality) + if quality: + monkeypatch.setattr(runner, "run_pass", boom) + ret = b._Benchmarker__benchmark(ft, open(tmp_path / ("log%s" % quality), "w")) + outcomes.append((ret, b.results.report_benchmark_results.call_args[0][1:])) + if quality: + name, q, image = b.results.add_quality.call_args[0] + assert name == "hugo" and q == {"status": "error", "error": "RuntimeError: kaboom"} + assert image is None + assert outcomes[0] == outcomes[1] + + +def test_unused_export_tar_removed(tmp_path, monkeypatch): + called = [] + monkeypatch.setattr(runner, "run_pass", lambda *a: called.append(a) or {"status": "ok"}) + for i, kw in enumerate(({"results": []}, {"failure": "nonconformant: index"})): + b, ft, _ = _bm(tmp_path / str(i), **kw) + tar_path = pathlib.Path(runner.archive_path(b.results.directory, "hugo")) + tar_path.parent.mkdir(parents=True) + tar_path.write_bytes(b"x") + b._Benchmarker__benchmark(ft, open(tmp_path / "log", "w")) + assert not tar_path.exists() + assert called == [] diff --git a/toolset/benchmark/benchmarker.py b/toolset/benchmark/benchmarker.py index d4ce05e..8ffdc38 100644 --- a/toolset/benchmark/benchmarker.py +++ b/toolset/benchmark/benchmarker.py @@ -277,9 +277,12 @@ def benchmark_type(test_type): self.results.report_benchmark_results(framework_test, test_type, results['results'], results.get('unsupported', False), results.get('failureReason')) - if export and results['results'] and not results.get('failureReason') \ - and not results.get('unsupported'): - self.__quality_pass(framework_test, benchmark_log) + if export: + if results['results'] and not results.get('failureReason') \ + and not results.get('unsupported'): + self.__quality_pass(framework_test, benchmark_log) + else: + self.__discard_export(export[1]) log("Complete", file=benchmark_log) return bool(results['results']) or bool(results.get('unsupported', False)) @@ -306,11 +309,27 @@ def __quality_export(self, framework_test): def __quality_pass(self, framework_test, benchmark_log): '''Untimed; never changes the build result (spec 009 R-2).''' - log("QUALITY PASS %s (untimed)" % framework_test.name, file=benchmark_log, border='*') - quality = quality_runner.run_pass(self.docker_helper, framework_test, self.results.directory) - self.results.add_quality(framework_test.name, quality, - getattr(self.docker_helper, '_quality_image', None)) - log("quality: %s" % quality.get('status'), file=benchmark_log) + try: + log("QUALITY PASS %s (untimed)" % framework_test.name, file=benchmark_log, border='*') + quality = quality_runner.run_pass(self.docker_helper, framework_test, self.results.directory) + self.results.add_quality(framework_test.name, quality, + getattr(self.docker_helper, '_quality_image', None)) + log("quality: %s" % quality.get('status'), file=benchmark_log) + except Exception as e: + error = "%s: %s" % (type(e).__name__, e) + try: + log("quality: pass failed: %s" % error) + self.results.add_quality(framework_test.name, {'status': 'error', 'error': error}, None) + except Exception: + pass + + def __discard_export(self, tar_path): + '''The pass will not run: drop the exported site so it does not linger (spec 009 R-2).''' + try: + if os.path.exists(tar_path): + os.remove(tar_path) + except Exception as e: + log("quality: could not remove %s: %s" % (tar_path, e)) def __noise_control(self, framework_test, test_type, script, script_variables, raw_file, results, benchmark_log): diff --git a/toolset/quality/runner.py b/toolset/quality/runner.py index f7cedc6..9e7120e 100644 --- a/toolset/quality/runner.py +++ b/toolset/quality/runner.py @@ -5,6 +5,7 @@ import os from toolset.quality import reduce, site +from toolset.utils.output_helper import log # R-1: with --suite, the pass runs only in this (numberOfFiles, contentSize) cell QUALITY_CELL = (50, 5.0) @@ -47,9 +48,15 @@ def run_pass(docker_helper, test, results_dir): quality = reduce.build_quality(raw, site_dir, pages, output_glob) except Exception as e: quality = {'status': 'error', 'error': '%s: %s' % (type(e).__name__, e)} - os.makedirs(out_dir, exist_ok=True) - with open(os.path.join(out_dir, 'quality.json'), 'w') as f: - json.dump(quality, f, indent=2) - if os.path.exists(tar_path): - os.remove(tar_path) + try: + os.makedirs(out_dir, exist_ok=True) + with open(os.path.join(out_dir, 'quality.json'), 'w') as f: + json.dump(quality, f, indent=2) + except Exception as e: + log("quality: could not write quality.json: %s: %s" % (type(e).__name__, e)) + try: + if os.path.exists(tar_path): + os.remove(tar_path) + except Exception as e: + log("quality: could not remove %s: %s: %s" % (tar_path, type(e).__name__, e)) return quality diff --git a/toolset/utils/docker_helper.py b/toolset/utils/docker_helper.py index 0dd7814..d6ec412 100644 --- a/toolset/utils/docker_helper.py +++ b/toolset/utils/docker_helper.py @@ -25,6 +25,9 @@ QUALITY_IMAGE = 'ssgberk/quality' QUALITY_TIMEOUT_SECONDS = 1800 QUALITY_MEMORY = '4g' +# collector output dir: outside /work so it can never land inside the site (a site +# folder named "out" is /work/out) +QUALITY_OUT = '/quality-out' class DockerHelper: @@ -466,7 +469,7 @@ def run_quality(self, tar_path, site_name, pages, ''' image = self.build_quality_image() command = ['node', '/quality/run.mjs', '--site', '/work/' + site_name, - '--pages', json.dumps(pages), '--out', '/work/out'] + '--pages', json.dumps(pages), '--out', QUALITY_OUT] container = self.server.containers.create( image, command=command, labels=self._run_labels(), network_mode='none', mem_limit=QUALITY_MEMORY, working_dir='/work') @@ -476,7 +479,7 @@ def run_quality(self, tar_path, site_name, pages, container.start() result = container.wait(timeout=timeout_seconds) try: - bits, _ = container.get_archive('/work/out/raw.json') + bits, _ = container.get_archive(QUALITY_OUT + '/raw.json') except docker.errors.NotFound: logs = container.logs().decode('utf-8', 'replace') raise RuntimeError('collector wrote no raw.json (exit %s): %s' % ( From 58aeec58b222a87df2acfcb884d64fc135dbfd92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matheus=20Bregu=C3=AAz?= Date: Sat, 10 Oct 2026 16:47:32 -0300 Subject: [PATCH 14/15] ci(quality): run the CI quality pass on mdbook, which uses the SF 005 reference site Co-Authored-By: Claude Haiku 5.5 --- .github/workflows/ci.yml | 12 ++++++------ README.md | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb8ed45..e72d87d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,17 +61,17 @@ jobs: - uses: actions/checkout@v4 with: submodules: true - - run: ./ssgberk --test hugo -nf 50 -cs 5 -mr 1 --quality + - run: ./ssgberk --test mdbook -nf 50 -cs 5 -mr 1 --quality - run: | R=$(ls -td results/*/ | head -1) - jq -e '.rawData.datarate.hugo[0].mean > 0' "$R/results.json" - jq -e '.quality.hugo.status == "ok"' "$R/results.json" + jq -e '.rawData.datarate.mdbook[0].mean > 0' "$R/results.json" + jq -e '.quality.mdbook.status == "ok"' "$R/results.json" for s in lighthouse seo html a11y links weight js extra; do - jq -e --arg s "$s" '.quality.hugo[$s].status == "ok"' "$R/results.json" + jq -e --arg s "$s" '.quality.mdbook[$s].status == "ok"' "$R/results.json" done - jq -e '[.quality.hugo.lighthouse.mobile[], .quality.hugo.lighthouse.desktop[]] + jq -e '[.quality.mdbook.lighthouse.mobile[], .quality.mdbook.lighthouse.desktop[]] | length == 6 and all(.status == "ok")' "$R/results.json" - jq -e '[.quality.hugo.lighthouse.mobile[], .quality.hugo.lighthouse.desktop[] + jq -e '[.quality.mdbook.lighthouse.mobile[], .quality.mdbook.lighthouse.desktop[] | .scores[] | .median | select(. < 0 or . > 1)] | length == 0' "$R/results.json" jq -e '.environment.quality.lighthouse and .environment.quality.chromium' "$R/results.json" test -s "$R/quality-summary.csv" diff --git a/README.md b/README.md index c2b324d..91425df 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ After a `--suite` run, `results//` also holds `suite.json` (cells, ge `--quality` audits the output of the last timed build of each generator, without timing it (spec `docs/specs/009-output-quality`): - $ ./ssgberk --test hugo -nf 50 -cs 5 --quality + $ ./ssgberk --test mdbook -nf 50 -cs 5 --quality With `--suite`, the pass runs only in the cell `nf50-cs5` (suite P, cell 0). The toolset's tests need Python 3.12 (the toolset image and CI use it). It builds the `ssgberk/quality` image from `quality/` once. That image pins Lighthouse, Playwright's Chromium, axe-core, html-validate and lychee. The image runs with no network and with the site copied in, and it audits the index, the first post and `404.html`: - Lighthouse, mobile and desktop presets, median of 3 runs; @@ -113,7 +113,7 @@ With `--suite`, the pass runs only in the cell `nf50-cs5` (suite P, cell 0). The - JS bytes, and whether the post works without JS; - files outside the reference page set. -Results go to `results//quality//quality.json`, to `quality` in `results.json`, to the "Qualidade" section of `summary.md` and to `quality-summary.csv`. They have no combined score and no ranking. A failing check is a result and never fails the run. +Results go to `results//quality//quality.json`, to `quality` in `results.json`, to the "Qualidade" section of `summary.md` and to `quality-summary.csv`. They have no combined score and no ranking. A failing check is a result and never fails the run. Generators that do not use the SF 005 reference site yet (the post link is not inside `.post-item` on the index) record `quality..status = "error"`. ## Content size From 4c6e185b5b4a72c9779a722800b1133ff23bebdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matheus=20Bregu=C3=AAz?= Date: Sat, 10 Oct 2026 16:55:44 -0300 Subject: [PATCH 15/15] fix(quality): fall back to output_glob for the audited post page (spec 009 R-7) Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 2 +- docs/specs/009-output-quality/plan.md | 1 + docs/specs/009-output-quality/spec.md | 2 +- tests/test_quality_reduce.py | 1 + tests/test_quality_runner.py | 21 +++++++++- tests/test_quality_site.py | 51 ++++++++++++++++++++++-- toolset/quality/reduce.py | 1 + toolset/quality/runner.py | 2 +- toolset/quality/site.py | 57 +++++++++++++++++++-------- 9 files changed, 113 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 91425df..d42a3fa 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ With `--suite`, the pass runs only in the cell `nf50-cs5` (suite P, cell 0). The - JS bytes, and whether the post works without JS; - files outside the reference page set. -Results go to `results//quality//quality.json`, to `quality` in `results.json`, to the "Qualidade" section of `summary.md` and to `quality-summary.csv`. They have no combined score and no ranking. A failing check is a result and never fails the run. Generators that do not use the SF 005 reference site yet (the post link is not inside `.post-item` on the index) record `quality..status = "error"`. +Results go to `results//quality//quality.json`, to `quality` in `results.json`, to the "Qualidade" section of `summary.md` and to `quality-summary.csv`. They have no combined score and no ranking. A failing check is a result and never fails the run. For generators that do not use the SF 005 index layout yet, the audited post is the first file matching `output_glob` (`postSource: "glob"` in `quality.json`). ## Content size diff --git a/docs/specs/009-output-quality/plan.md b/docs/specs/009-output-quality/plan.md index 3ac33bc..6ac67f9 100644 --- a/docs/specs/009-output-quality/plan.md +++ b/docs/specs/009-output-quality/plan.md @@ -114,6 +114,7 @@ Metric units are Lighthouse's `numericValue`: milliseconds for FCP, LCP, TBT and { "status": "ok", "pages": {"index": "/", "post": "/post/hello/", "404": "/404.html"}, + "postSource": "index", "tools": {...copied from raw.json...}, "lighthouse": {"status": "ok", "mobile": {"post": {"status": "ok", "runs": 3, diff --git a/docs/specs/009-output-quality/spec.md b/docs/specs/009-output-quality/spec.md index 305eadd..c64f96c 100644 --- a/docs/specs/009-output-quality/spec.md +++ b/docs/specs/009-output-quality/spec.md @@ -27,7 +27,7 @@ This spec adds an **untimed quality pass** over the output of one build per gene ### URLs -- **R-7** Three URLs are audited: the index `/`, one post page and `/404.html`. The post page is the first `a[href]` inside `.post-item`, the same link SF 006 resolves. +- **R-7** Up to three URLs are audited: the index `/`, one post page and `/404.html` when that file exists. The post page is the first `a[href]` inside `.post-item`, the same link SF 006 resolves. When the index has no such link (generators that do not use the SF 005 layout yet), it is the first file, in sorted order, that matches `output_glob`. `quality.json` records which rule chose it in `postSource` (`index` or `glob`). ### Checks diff --git a/tests/test_quality_reduce.py b/tests/test_quality_reduce.py index 2b2cc17..e692959 100644 --- a/tests/test_quality_reduce.py +++ b/tests/test_quality_reduce.py @@ -91,6 +91,7 @@ def test_build_quality_fixture_all_ok(): q = reduce.build_quality(_raw(), str(SITE), pages, GLOB) assert q["status"] == "ok" assert q["pages"] == {"index": "/", "post": "/post/hello/", "404": "/404.html"} + assert q["postSource"] == "index" for section in reduce.SECTIONS: assert q[section]["status"] == "ok", (section, q[section]) assert q["js"]["worksWithoutJs"] is True and q["js"]["jsClass"] == "none" diff --git a/tests/test_quality_runner.py b/tests/test_quality_runner.py index fa7444a..ae6f5d3 100644 --- a/tests/test_quality_runner.py +++ b/tests/test_quality_runner.py @@ -49,13 +49,32 @@ def test_run_pass_ok(tmp_path): assert q["status"] == "ok" and q["lighthouse"]["status"] == "ok" args = helper.run_quality.call_args[0] assert args[0] == str(tar_path) and args[1] == "public" - assert args[2]["post"] == {"url": "/post/hello/", "file": "post/hello/index.html"} + assert args[2]["post"] == {"url": "/post/hello/", "file": "post/hello/index.html", "source": "index"} out = results_dir / "quality" / "hugo" assert json.loads((out / "quality.json").read_text()) == q assert (out / "raw.json").exists() and (out / "site" / "index.html").exists() assert not tar_path.exists() +def test_run_pass_uses_output_glob_when_index_has_no_post_link(tmp_path): + test, results_dir = _generator(tmp_path), tmp_path / "results" + src = tmp_path / "plain" + (src / "post" / "a").mkdir(parents=True) + (src / "index.html").write_text("

    no layout

    ") + (src / "post" / "a" / "index.html").write_text("

    a

    ") + tar_path = pathlib.Path(runner.archive_path(str(results_dir), "hugo")) + tar_path.parent.mkdir(parents=True) + with tarfile.open(tar_path, "w") as tar: + tar.add(src, arcname="public") + helper = mock.Mock() + helper.run_quality.return_value = json.loads((FIX / "raw.json").read_text()) + q = runner.run_pass(helper, test, str(results_dir)) + pages = helper.run_quality.call_args[0][2] + assert pages["post"] == {"url": "/post/a/", "file": "post/a/index.html", "source": "glob"} + assert "404" not in pages + assert q["postSource"] == "glob" and q["pages"] == {"index": "/", "post": "/post/a/"} + + def test_run_pass_missing_archive(tmp_path): test, results_dir = _generator(tmp_path), tmp_path / "results" q = runner.run_pass(mock.Mock(), test, str(results_dir)) diff --git a/tests/test_quality_site.py b/tests/test_quality_site.py index 038b1cc..1ec23ac 100644 --- a/tests/test_quality_site.py +++ b/tests/test_quality_site.py @@ -51,7 +51,7 @@ def test_extract_rejects_path_traversal(tmp_path): def test_resolve_pages_fixture(): assert site.resolve_pages(str(SITE)) == { "index": {"url": "/", "file": "index.html"}, - "post": {"url": "/post/hello/", "file": "post/hello/index.html"}, + "post": {"url": "/post/hello/", "file": "post/hello/index.html", "source": "index"}, "404": {"url": "/404.html", "file": "404.html"}, } @@ -71,7 +71,7 @@ def test_resolve_pages_variants(tmp_path, href, url, file): (tmp_path / "index.html").write_text( '' % href) pages = site.resolve_pages(str(tmp_path)) - assert pages["post"] == {"url": url, "file": file} + assert pages["post"] == {"url": url, "file": file, "source": "index"} def test_resolve_pages_without_post_link(tmp_path): @@ -80,12 +80,55 @@ def test_resolve_pages_without_post_link(tmp_path): site.resolve_pages(str(tmp_path)) -def test_resolve_pages_link_to_missing_file(tmp_path): +def test_resolve_pages_link_to_missing_file_without_glob(tmp_path): (tmp_path / "index.html").write_text('
  • g
    • ') - with pytest.raises(ValueError, match="no file"): + with pytest.raises(ValueError, match="post-item"): site.resolve_pages(str(tmp_path)) +def _site(tmp_path, index, files): + (tmp_path / "index.html").write_text(index) + for f in files: + (tmp_path / f).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / f).write_text("x") + + +def test_resolve_pages_glob_fallback_dir_style(tmp_path): + _site(tmp_path, 'x', ["post/b/index.html", "post/a/index.html", "404.html"]) + pages = site.resolve_pages(str(tmp_path), "post/*/index.html") + assert pages["post"] == {"url": "/post/a/", "file": "post/a/index.html", "source": "glob"} + assert pages["404"] == {"url": "/404.html", "file": "404.html"} + + +def test_resolve_pages_glob_fallback_html_style(tmp_path): + _site(tmp_path, "

    none

    ", ["posts/b.html", "posts/a.html"]) + pages = site.resolve_pages(str(tmp_path), "posts/*.html") + assert pages["post"] == {"url": "/posts/a.html", "file": "posts/a.html", "source": "glob"} + + +def test_resolve_pages_missing_link_target_falls_back_to_glob(tmp_path): + _site(tmp_path, '
  • g
    • ', ["post/a/index.html"]) + pages = site.resolve_pages(str(tmp_path), "post/*/index.html") + assert pages["post"]["source"] == "glob" and pages["post"]["url"] == "/post/a/" + + +def test_resolve_pages_no_404_file(tmp_path): + _site(tmp_path, '
  • a
    • ', ["post/a/index.html"]) + pages = site.resolve_pages(str(tmp_path), "post/*/index.html") + assert set(pages) == {"index", "post"} + + +def test_resolve_pages_nothing_found(tmp_path): + _site(tmp_path, "

    none

    ", ["other/file.txt"]) + with pytest.raises(ValueError, match=r"post-item.*post/\*/index\.html"): + site.resolve_pages(str(tmp_path), "post/*/index.html") + + +def test_resolve_pages_requires_index(tmp_path): + with pytest.raises(ValueError, match="index.html"): + site.resolve_pages(str(tmp_path), "*.html") + + def test_list_files_is_sorted_posix(): files = site.list_files(str(SITE)) assert files == sorted(files) and "post/hello/index.html" in files diff --git a/toolset/quality/reduce.py b/toolset/quality/reduce.py index 8b68e5e..354c4da 100644 --- a/toolset/quality/reduce.py +++ b/toolset/quality/reduce.py @@ -155,6 +155,7 @@ def _section(fn): def build_quality(raw, site_dir, pages, output_glob): '''quality.json (R-16): one section per check, each ok or error.''' q = {'status': 'ok', 'pages': {k: v['url'] for k, v in pages.items()}, + 'postSource': pages['post'].get('source'), 'tools': raw.get('tools')} q['lighthouse'] = _section(lambda: reduce_lighthouse(_raw(raw, 'lighthouse'))) q['seo'] = _section(lambda: seo.seo_section(site_dir, pages)) diff --git a/toolset/quality/runner.py b/toolset/quality/runner.py index 9e7120e..30109cc 100644 --- a/toolset/quality/runner.py +++ b/toolset/quality/runner.py @@ -40,7 +40,7 @@ def run_pass(docker_helper, test, results_dir): if not os.path.exists(tar_path): raise RuntimeError('no exported output (%s)' % output_folder) site_dir = site.extract(tar_path, os.path.join(out_dir, 'site')) - pages = site.resolve_pages(site_dir) + pages = site.resolve_pages(site_dir, output_glob) site_name = os.path.basename(output_folder.rstrip('/')) raw = docker_helper.run_quality(tar_path, site_name, pages) with open(os.path.join(out_dir, 'raw.json'), 'w') as f: diff --git a/toolset/quality/site.py b/toolset/quality/site.py index a82a4ae..754c50a 100644 --- a/toolset/quality/site.py +++ b/toolset/quality/site.py @@ -74,24 +74,47 @@ def _read(path): return f.read() -def resolve_pages(site_dir): - '''The three audited pages (R-7): index, first post link, 404.''' +def _glob_post(site_dir, output_glob): + '''(url, file) of the first sorted file matching output_glob, or None.''' + for f in list_files(site_dir): + if fnmatch.fnmatch(f, output_glob): + url = '/' + f + if posixpath.basename(f) == 'index.html': + url = url[:-len('index.html')] + return url, f + return None + + +def resolve_pages(site_dir, output_glob=None): + ''' + The audited pages (R-7): index, the post (first .post-item link, else the + first file matching output_glob) and 404 when 404.html exists. + ''' + index_path = os.path.join(site_dir, 'index.html') + if not os.path.isfile(index_path): + raise ValueError('no index.html at the site root') parser = _PostLink() - parser.feed(_read(os.path.join(site_dir, 'index.html'))) - if not parser.href: - raise ValueError('no a[href] inside .post-item on the index') - path = urlsplit(parser.href).path - url = posixpath.normpath(posixpath.join('/', path)) - if path.endswith('/') and url != '/': - url += '/' - post_file = url_to_file(site_dir, url) - if post_file is None: - raise ValueError('post link %s matches no file' % parser.href) - return { - 'index': {'url': '/', 'file': 'index.html'}, - 'post': {'url': url, 'file': post_file}, - '404': {'url': '/404.html', 'file': '404.html'}, - } + parser.feed(_read(index_path)) + post = None + if parser.href: + path = urlsplit(parser.href).path + url = posixpath.normpath(posixpath.join('/', path)) + if path.endswith('/') and url != '/': + url += '/' + post_file = url_to_file(site_dir, url) + if post_file is not None: + post = {'url': url, 'file': post_file, 'source': 'index'} + if post is None and output_glob: + found = _glob_post(site_dir, output_glob) + if found: + post = {'url': found[0], 'file': found[1], 'source': 'glob'} + if post is None: + raise ValueError('no post page: no a[href] to an existing file inside ' + '.post-item on the index and no file matches %s' % output_glob) + pages = {'index': {'url': '/', 'file': 'index.html'}, 'post': post} + if os.path.isfile(os.path.join(site_dir, '404.html')): + pages['404'] = {'url': '/404.html', 'file': '404.html'} + return pages def list_files(site_dir):