diff --git a/schema/card.schema.json b/schema/card.schema.json index 87cbae7..1ed7a31 100644 --- a/schema/card.schema.json +++ b/schema/card.schema.json @@ -128,7 +128,8 @@ "capacity-cost", "observability", "dependency-failure", - "backup-restore" + "backup-restore", + "model-pipeline" ] }, "scaleAxis": { diff --git a/schema/vocabulary.md b/schema/vocabulary.md index 7727f2f..321ba05 100644 --- a/schema/vocabulary.md +++ b/schema/vocabulary.md @@ -49,3 +49,4 @@ The design question a record answers. | `observability` | Detecting and diagnosing problems | | `dependency-failure` | Behaviour when something the system relies on degrades | | `backup-restore` | Backups, restores and recovering deleted or corrupted data. Rebooting a site and failing over are `replication-failover` | +| `model-pipeline` | How a model is built and run: model architecture, training and feature pipelines, and the inference path. A cost or sizing question about them is still `capacity-cost` | diff --git a/scripts/fetch-browser.py b/scripts/fetch-browser.py index 559e84f..d840370 100644 --- a/scripts/fetch-browser.py +++ b/scripts/fetch-browser.py @@ -3,7 +3,7 @@ One page, for a source a person has chosen to write a card from. Never used to crawl a site. Usage: uvx --from camoufox python -I scripts/fetch-browser.py (out must be outside the repository) """ -import pathlib, re, sys +import pathlib, re, sys, time from urllib.parse import urlparse from camoufox.sync_api import Camoufox @@ -15,6 +15,12 @@ with Camoufox(headless=True, i_know_what_im_doing=True) as browser: page = browser.new_page() response = page.goto(url, timeout=60000, wait_until="domcontentloaded") + # Some hosts answer 429 to a second page fetched straight after the first; wait and ask again. + for wait in (20, 60): + if response is None or response.status != 429: + break + time.sleep(wait) + response = page.goto(url, timeout=60000, wait_until="domcontentloaded") if response is not None and response.status >= 400: sys.exit(f"HTTP {response.status} for {url}; nothing written") # A retired article often redirects to a home page; the article's own name must survive the redirect.