diff --git a/.github/workflows/jekyll-gh-pages.yml b/.github/workflows/jekyll-gh-pages.yml index 67f93c6..66e31d9 100644 --- a/.github/workflows/jekyll-gh-pages.yml +++ b/.github/workflows/jekyll-gh-pages.yml @@ -3,7 +3,14 @@ name: Deploy fastled.io on: push: branches: ["gh-pages"] + schedule: + - cron: '47 10 * * *' workflow_dispatch: + inputs: + docs_only: + description: Deploy only if the upstream documentation publication changed + type: boolean + default: false permissions: contents: read @@ -13,9 +20,48 @@ concurrency: cancel-in-progress: false jobs: - build: + check: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + publication: ${{ steps.docs.outputs.publication }} + changed: ${{ steps.docs.outputs.changed }} + deploy: ${{ steps.docs.outputs.deploy }} + release_id: ${{ steps.docs.outputs.release_id }} + tag: ${{ steps.docs.outputs.tag }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - name: Check the published documentation generation + id: docs + env: + DOCS_ONLY: ${{ inputs.docs_only || false }} + run: python3 scripts/import_docs.py check + + record-check: + needs: check runs-on: ubuntu-24.04 timeout-minutes: 5 + permissions: + contents: write + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - name: Record a successful documentation check at most monthly + env: + GH_TOKEN: ${{ github.token }} + RELEASE_ID: ${{ needs.check.outputs.release_id }} + RELEASE_TAG: ${{ needs.check.outputs.tag }} + RELEASE_CHANGED: ${{ needs.check.outputs.changed }} + run: python3 scripts/record_check.py + + build: + needs: check + if: needs.check.outputs.deploy == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 20 permissions: contents: read pages: read @@ -29,7 +75,11 @@ jobs: - name: Stage static site run: | mkdir _site - git archive --format=tar HEAD -- . ':!.github' ':!README.md' | tar -xf - -C _site + git archive --format=tar HEAD -- . ':!.github' ':!README.md' ':!scripts' ':!.gitignore' | tar -xf - -C _site + - name: Import the verified documentation publication + env: + PUBLICATION_JSON: ${{ needs.check.outputs.publication }} + run: python3 scripts/import_docs.py import --site _site - name: Upload artifact uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4 with: diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6f3eff8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +_site/ +__pycache__/ diff --git a/404.html b/404.html index 966336f..825f589 100644 --- a/404.html +++ b/404.html @@ -4,12 +4,6 @@
Open the FastLED API reference.
- diff --git a/scripts/import_docs.py b/scripts/import_docs.py new file mode 100644 index 0000000..4e57543 --- /dev/null +++ b/scripts/import_docs.py @@ -0,0 +1,190 @@ +"""Import a verified FastLED/dox publication into the website deployment.""" + +import argparse +import hashlib +import json +import os +import re +import tarfile +import tempfile +import urllib.error +import urllib.request +from pathlib import Path, PurePosixPath +from urllib.parse import urlencode + +UPSTREAM = "https://fastled.github.io/dox/publication.json" +CURRENT = "https://fastled.io/docs/publication.json" +MAX_ARCHIVE = 2 * 1024**3 +MAX_EXTRACTED = 10 * 1024**3 + + +def validate_manifest(manifest: dict) -> None: + try: + generation = manifest["generation"] + release = manifest["release"] + archive = manifest["archive"] + valid = ( + manifest["schema_version"] == 1 + and isinstance(generation, str) + and re.fullmatch(r"[0-9]+-[0-9]+", generation) + and release["generation"] == generation + and type(release["id"]) is int + and release["id"] > 0 + and re.fullmatch(r"v?[0-9]+\.[0-9]+\.[0-9]+", release["tag_name"]) + and re.fullmatch(r"[0-9a-f]{40}", release["source_sha"]) + and isinstance(release["doxygen_version"], str) + and release["doxygen_version"] + and archive["url"] + == f"https://github.com/FastLED/dox/releases/download/docs-{generation}/docs.tar.gz" + and re.fullmatch(r"[0-9a-f]{64}", archive["sha256"]) + and type(archive["size"]) is int + and 0 < archive["size"] <= MAX_ARCHIVE + ) + except (KeyError, TypeError): + valid = False + if not valid: + raise ValueError("Invalid documentation publication manifest") + + +def fetch_manifest(url: str, *, missing_ok: bool = False) -> dict | None: + request = urllib.request.Request(url, headers={"Cache-Control": "no-cache"}) + try: + with urllib.request.urlopen(request, timeout=60) as response: + data = response.read(65537) + except urllib.error.HTTPError as error: + if error.code == 404 and missing_ok: + return None + raise + if len(data) > 65536: + raise ValueError("Documentation manifest exceeds 64 KiB") + manifest = json.loads(data) + validate_manifest(manifest) + return manifest + + +def check_publication(upstream: str, current: str) -> tuple[dict, bool]: + manifest = fetch_manifest(upstream) + deployed = fetch_manifest(current, missing_ok=True) + return manifest, manifest != deployed + + +def download_archive(manifest: dict, destination: Path) -> None: + validate_manifest(manifest) + archive = manifest["archive"] + size = 0 + digest = hashlib.sha256() + with urllib.request.urlopen(archive["url"], timeout=60) as response: + with destination.open("wb") as output: + while chunk := response.read(1024 * 1024): + size += len(chunk) + if size > archive["size"]: + raise ValueError("Documentation download exceeds manifest size") + digest.update(chunk) + output.write(chunk) + if size != archive["size"] or digest.hexdigest() != archive["sha256"]: + raise ValueError("Documentation download checksum or size mismatch") + + +def stage_archive(archive: Path, site: Path, manifest: dict) -> None: + validate_manifest(manifest) + if archive.stat().st_size != manifest["archive"]["size"]: + raise ValueError("Documentation archive size mismatch") + with archive.open("rb") as source: + if ( + hashlib.file_digest(source, "sha256").hexdigest() + != manifest["archive"]["sha256"] + ): + raise ValueError("Documentation archive checksum mismatch") + site.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix="dox-import-", dir=site.parent + ) as temporary: + staging = Path(temporary) / "docs" + staging.mkdir() + with tarfile.open(archive, "r:gz") as tar: + members = tar.getmembers() + total = 0 + for member in members: + path = PurePosixPath(member.name) + if ( + path.is_absolute() + or ".." in path.parts + or "\\" in member.name + or not (member.isfile() or member.isdir()) + or (member.isfile() and not path.parts) + ): + raise ValueError( + f"Unsafe documentation archive entry: {member.name}" + ) + if member.isfile(): + total += member.size + if total > MAX_EXTRACTED: + raise ValueError( + "Documentation archive expands beyond Pages size limit" + ) + tar.extractall(staging, members=members, filter="data") + actual_release = json.loads((staging / "release.json").read_text()) + if actual_release != manifest["release"]: + raise ValueError("Archive release metadata does not match publication") + for required in ("index.html", "annotated.html", "search/search.js"): + path = staging / required + if not path.is_file() or path.stat().st_size == 0: + raise ValueError(f"Missing required documentation output: {required}") + (staging / "publication.json").write_text(json.dumps(manifest, indent=2) + "\n") + destination = site / "docs" + backup = Path(temporary) / "previous" + if destination.exists(): + destination.rename(backup) + try: + staging.rename(destination) + except OSError: + if backup.exists(): + backup.rename(destination) + raise + print( + f"Staged FastLED {actual_release['tag_name']} publication {manifest['generation']}" + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("check", "import")) + parser.add_argument("--site", type=Path, default=Path("_site")) + args = parser.parse_args() + if args.command == "check": + query = urlencode( + { + "run": os.environ["GITHUB_RUN_ID"], + "attempt": os.environ["GITHUB_RUN_ATTEMPT"], + } + ) + manifest, changed = check_publication( + UPSTREAM + "?" + query, CURRENT + "?" + query + ) + docs_only = os.environ["GITHUB_EVENT_NAME"] == "schedule" or ( + os.environ.get("DOCS_ONLY") == "true" + ) + outputs = { + "publication": json.dumps(manifest, separators=(",", ":")), + "changed": str(changed).lower(), + "deploy": str(changed or not docs_only).lower(), + "release_id": str(manifest["release"]["id"]), + "tag": manifest["release"]["tag_name"], + } + with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: + for key, value in outputs.items(): + output.write(f"{key}={value}\n") + print( + f"Upstream publication {manifest['generation']}; update required: {outputs['deploy']}" + ) + else: + manifest = json.loads(os.environ["PUBLICATION_JSON"]) + validate_manifest(manifest) + with tempfile.TemporaryDirectory(prefix="dox-download-") as temporary: + archive = Path(temporary) / "docs.tar.gz" + download_archive(manifest, archive) + stage_archive(archive, args.site, manifest) + + +if __name__ == "__main__": + main() diff --git a/scripts/record_check.py b/scripts/record_check.py new file mode 100644 index 0000000..98e8d3b --- /dev/null +++ b/scripts/record_check.py @@ -0,0 +1,109 @@ +"""Record a successful release check monthly without rebuilding documentation.""" + +import json +import os +import urllib.error +import urllib.request +from datetime import datetime, timedelta, timezone + +BRANCH = "automation/release-checks" + + +def record_check(api, repository, release_id, tag, changed, run_url, now): + # Authenticate against the repository before treating a missing ref as new. + api("GET", "") + ref = api("GET", f"git/ref/heads/{BRANCH}") + previous = ref["object"]["sha"] if ref else None + if previous: + commit = api("GET", f"git/commits/{previous}") + committed = datetime.fromisoformat( + commit["committer"]["date"].replace("Z", "+00:00") + ) + if now - committed < timedelta(days=30): + print("Release check audit is current; no repository write needed.") + return False + + snapshot = { + "checked_at": now.isoformat().replace("+00:00", "Z"), + "release_id": int(release_id), + "tag": tag, + "changed": changed, + "run_url": run_url, + } + # A tree with no base_tree contains only this small audit, never site HTML. + tree = api( + "POST", + "git/trees", + { + "tree": [ + { + "path": "release-check.json", + "mode": "100644", + "type": "blob", + "content": json.dumps(snapshot, indent=2) + "\n", + } + ] + }, + ) + commit = api( + "POST", + "git/commits", + { + "message": "chore: record successful release check", + "tree": tree["sha"], + "parents": [previous] if previous else [], + }, + ) + if previous: + api("PATCH", f"git/refs/heads/{BRANCH}", {"sha": commit["sha"], "force": False}) + else: + api("POST", "git/refs", {"ref": f"refs/heads/{BRANCH}", "sha": commit["sha"]}) + print(f"Recorded successful release check on {repository}:{BRANCH}.") + return True + + +class GitHubAPI: + def __init__(self, repository, token): + self.base = f"{os.environ.get('GITHUB_API_URL', 'https://api.github.com')}/repos/{repository}" + self.token = token + + def __call__(self, method, path, body=None): + request = urllib.request.Request( + self.base + ("/" + path if path else ""), + data=json.dumps(body).encode() if body is not None else None, + method=method, + headers={ + "Authorization": f"Bearer {self.token}", + "Accept": "application/vnd.github+json", + "Content-Type": "application/json", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + except urllib.error.HTTPError as error: + if ( + error.code == 404 + and method == "GET" + and path == f"git/ref/heads/{BRANCH}" + ): + return None + raise + + +def main(): + repository = os.environ["GITHUB_REPOSITORY"] + record_check( + GitHubAPI(repository, os.environ["GH_TOKEN"]), + repository, + os.environ["RELEASE_ID"], + os.environ["RELEASE_TAG"], + os.environ["RELEASE_CHANGED"] == "true", + f"{os.environ['GITHUB_SERVER_URL']}/{repository}/actions/runs/{os.environ['GITHUB_RUN_ID']}", + datetime.now(timezone.utc), + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/test_import_docs.py b/scripts/test_import_docs.py new file mode 100644 index 0000000..91bea3c --- /dev/null +++ b/scripts/test_import_docs.py @@ -0,0 +1,146 @@ +"""Exercise real documentation archives and HTTP publication checks.""" + +import hashlib +import io +import json +import tarfile +import tempfile +import threading +import unittest +from functools import partial +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +from import_docs import check_publication, stage_archive, validate_manifest + + +class DocumentationImportTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.site = self.root / "site" + (self.site / "docs").mkdir(parents=True) + (self.site / "docs/index.html").write_text("old redirect") + self.release = { + "id": 416153555, + "tag_name": "3.10.8", + "source_sha": "d7402f1294e4548d9f8317f76afad37afaddb237", + "doxygen_version": "1.13.2", + "generation": "123-1", + } + self.manifest = { + "schema_version": 1, + "generation": "123-1", + "release": self.release, + "archive": { + "url": "https://github.com/FastLED/dox/releases/download/docs-123-1/docs.tar.gz", + "sha256": "", + "size": 0, + }, + } + self.archive = self.root / "docs.tar.gz" + self.make_archive() + + def make_archive(self, extra=None): + files = { + "./index.html": b"