From cd3c88cbb4c5a19dfc8df4d1909f14467efd0245 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Wed, 14 Oct 2026 14:26:06 +0000 Subject: [PATCH 1/5] feat(crawl): read the discovery strategies a crawl ran with, and save a page as a file A crawl reports `discovery`, the strategies it finds URLs with on each page it walks, and a create can ask for them. `links` follows the links in the page's HTML, and is what a crawl runs with when the create leaves it out. The field reads `["links"]` when the server does not send it, so the model parses an older answer too. `content(..., download=True)` asks for the page as a file to save rather than one to show. The text is the same; the server sends a file name built from the content id. `CrawlErrorCode` gains `api_key_cap_reached`, which a crawl ends with when the key that created it reaches one of its credit caps. It parsed as an unknown value before. --- acceptance/crawl/surface/test.py | 71 +++++++++++++++++++++++++++++++- src/zenrows/crawl/__init__.py | 2 + src/zenrows/crawl/client.py | 20 ++++++++- src/zenrows/crawl/models.py | 20 ++++++++- 4 files changed, 109 insertions(+), 4 deletions(-) diff --git a/acceptance/crawl/surface/test.py b/acceptance/crawl/surface/test.py index 1f1ba40..709f353 100644 --- a/acceptance/crawl/surface/test.py +++ b/acceptance/crawl/surface/test.py @@ -248,7 +248,8 @@ def do_GET(self): threading.Thread(target=srv.serve_forever, daemon=True).start() from zenrows import ZenRowsCrawlClient # noqa: E402 (imported once installed, as a user would) -from zenrows.crawl import CrawlAPIError # noqa: E402 +from zenrows.batch._open_enum import is_unknown # noqa: E402 +from zenrows.crawl import CrawlAPIError, CrawlErrorCode # noqa: E402 cap = ZenRowsCrawlClient( "capture-key", base_url=f"http://127.0.0.1:{srv.server_address[1]}/v1", retries=0 @@ -839,6 +840,72 @@ def c21(): live=True, ) + +def c22(): + """discovery: sent as given, echoed back, and ["links"] when left out.""" + given = create(discovery=["links"], **BASE) + left_out = create(**{**BASE, "max_items": 1}) + for c in (given, left_out): + client.stop(c.crawl_id) + values = [[d.value for d in c.discovery] for c in (given, left_out)] + return ( + pass_if(values == [["links"], ["links"]]), + json.dumps({"discovery=['links']": values[0], "left out": values[1]}), + "", + ) + + +def c23(): + """content(download=True): the same page, asked for as a file to save.""" + b = state.get("B") + if not b: + return SKIP, "no html crawl to read a page from", "" + fetched = next( + (r for r in client.get(b.crawl_id, limit=100).results if r.content_url), + None, + ) + if not fetched: + return SKIP, "no result had been fetched", "" + plain = client.content(b.crawl_id, fetched) + as_file = client.content(b.crawl_id, fetched, download=True) + return ( + pass_if(plain == as_file and bool(plain)), + f"same {len(plain)} chars with and without download=True", + trim(plain, 400), + ) + + +def c24(): + """api_key_cap_reached parses as the named member, not as UNKNOWN.""" + code = CrawlErrorCode("api_key_cap_reached") + return ( + pass_if(code is CrawlErrorCode.API_KEY_CAP_REACHED and not is_unknown(code)), + f"{code!r} (name {code.name})", + "", + ) + + +run_check( + "22", + "Discovery", + 'create(discovery=["links"]) is echoed back, and a create without it reads ["links"]', + c22, + live=True, +) +run_check( + "23", + "Content as a file", + "content(download=True) returns the same page as content() does", + c23, + live=True, +) +run_check( + "24", + "Run error codes", + "api_key_cap_reached is a named CrawlErrorCode member, not an unknown value", + c24, +) + with open(f"{RESULTS}/naming.json", "w") as f: json.dump( { @@ -856,11 +923,13 @@ def c21(): "include_patterns", "exclude_patterns", "output_format", + "discovery", "idempotency_key", "cursor", "limit", "timeout", "content_id", + "download", ] }, }, diff --git a/src/zenrows/crawl/__init__.py b/src/zenrows/crawl/__init__.py index 2f7eb56..d0e22e8 100644 --- a/src/zenrows/crawl/__init__.py +++ b/src/zenrows/crawl/__init__.py @@ -28,6 +28,7 @@ CrawlStatus, CrawlStop, CrawlWithResults, + DiscoveryStrategy, DownloadLine, OutputFormat, RunError, @@ -47,6 +48,7 @@ "CrawlStatus", "CrawlStop", "CrawlWithResults", + "DiscoveryStrategy", "DownloadLine", "OutputFormat", "RunError", diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index abac80b..7b911f9 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -172,6 +172,7 @@ def create( include_patterns: list[str] | None = None, exclude_patterns: list[str] | None = None, output_format: Literal["html", "markdown"] | None = None, + discovery: list[Literal["links"]] | None = None, idempotency_key: str | None = None, ) -> Crawl: """`POST /crawls` — start a crawl and return it, `running`. @@ -190,6 +191,9 @@ def create( - `output_format="html"` or `"markdown"`: also fetch every kept URL's page, as HTML or as Markdown, readable with `content` / `download`. Omit for URLs only. + - `discovery`: how the crawl finds the URLs it follows on each + page it walks. `["links"]` follows the links in the page's + HTML, and is what a crawl runs with when you leave it out. - `idempotency_key`: a retry with the same key and body answers with the crawl the first request created. @@ -206,6 +210,7 @@ def create( "include_patterns": include_patterns, "exclude_patterns": exclude_patterns, "output_format": output_format, + "discovery": discovery, } body.update({k: v for k, v in optional.items() if v is not None}) headers = {"Content-Type": "application/json"} @@ -300,7 +305,13 @@ def stop(self, crawl_id: str) -> CrawlStop: # ===== contents ===== - def content(self, crawl_id: str, content_id: str | CrawlResult) -> str: + def content( + self, + crawl_id: str, + content_id: str | CrawlResult, + *, + download: bool = False, + ) -> str: """`GET /crawls/{crawl_id}/contents/{content_id}` — one kept URL's page, as text in the crawl's `output_format` (HTML or Markdown). @@ -310,6 +321,10 @@ def content(self, crawl_id: str, content_id: str | CrawlResult) -> str: segment of the `content_url`). Only crawls created with an `output_format` have contents; anything else is 404 `content_not_found`. + + `download=True` asks for the page as a file to save: the server + sends `Content-Disposition: attachment` with a file name built + from the content id. The text returned is the same either way. """ if isinstance(content_id, CrawlResult): if not content_id.content_url: @@ -319,7 +334,8 @@ def content(self, crawl_id: str, content_id: str | CrawlResult) -> str: ) content_id = content_id.content_url segment = content_id.rstrip("/").rsplit("/", 1)[-1] - return self._t.request("GET", f"/crawls/{crawl_id}/contents/{segment}").text + params = {"download": "true"} if download else None + return self._t.request("GET", f"/crawls/{crawl_id}/contents/{segment}", params=params).text def download(self, crawl_id: str) -> CrawlDownload: """`GET /crawls/{crawl_id}/download` — open the NDJSON file and diff --git a/src/zenrows/crawl/models.py b/src/zenrows/crawl/models.py index 85050d3..35f8fda 100644 --- a/src/zenrows/crawl/models.py +++ b/src/zenrows/crawl/models.py @@ -12,7 +12,7 @@ from enum import Enum from typing import Any -from pydantic import BaseModel +from pydantic import BaseModel, Field from zenrows.batch._open_enum import open_enum_missing @@ -61,10 +61,23 @@ class OutputFormat(Enum): _missing_ = classmethod(open_enum_missing) +class DiscoveryStrategy(Enum): + """How the crawl finds the URLs it follows on each page it walks. + + `links` follows the links in the page's HTML. The server may run a + crawl with a strategy this version does not list, and it parses as + an `UNKNOWN` member keeping the raw value.""" + + LINKS = "links" + + _missing_ = classmethod(open_enum_missing) + + class CrawlErrorCode(Enum): """Why a crawl failed (`Crawl.error.code`).""" INSUFFICIENT_CREDITS = "insufficient_credits" + API_KEY_CAP_REACHED = "api_key_cap_reached" SEED_UNREACHABLE = "seed_unreachable" DOMAIN_NOT_ALLOWED = "domain_not_allowed" NO_ITEMS_FOUND = "no_items_found" @@ -104,6 +117,10 @@ class Crawl(BaseModel): created without one: it has no depth limit.""" max_items: int max_pages: int + discovery: list[DiscoveryStrategy] = Field(default_factory=lambda: [DiscoveryStrategy.LINKS]) + """The strategies the crawl finds URLs with. `["links"]` when the + create left it out, which is also what this reads when the server + does not send the field at all.""" coverage: Coverage created_at: datetime stop_reason: StopReason | None = None @@ -179,6 +196,7 @@ class DownloadLine(BaseModel): "CrawlStatus", "CrawlStop", "CrawlWithResults", + "DiscoveryStrategy", "DownloadLine", "OutputFormat", "RunError", From 9ebf1f55ad513eafbb71f1e64c2352bba175a507 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Wed, 14 Oct 2026 15:19:56 +0000 Subject: [PATCH 2/5] test(crawl): read a page of the acceptance check's own crawl when the stopped one has none The check that reads a page as a file borrowed the crawl another check stops mid-flight. That stop can land before any page is fetched, and the check then skipped instead of covering anything. It now falls back to a one-item crawl of its own and waits for it, as the check beside it already did. --- acceptance/crawl/surface/test.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/acceptance/crawl/surface/test.py b/acceptance/crawl/surface/test.py index 709f353..92557ca 100644 --- a/acceptance/crawl/surface/test.py +++ b/acceptance/crawl/surface/test.py @@ -857,17 +857,27 @@ def c22(): def c23(): """content(download=True): the same page, asked for as a file to save.""" + + def first_fetched(crawl_id): + return next( + (r for r in client.get(crawl_id, limit=100).results if r.content_url), + None, + ) + + # The stop in check 20 can land before any of B's pages is fetched. Then read + # a one-item html crawl that runs to the end instead, as check 9 does. b = state.get("B") - if not b: - return SKIP, "no html crawl to read a page from", "" - fetched = next( - (r for r in client.get(b.crawl_id, limit=100).results if r.content_url), - None, - ) + src = b.crawl_id if b else None + fetched = first_fetched(src) if src else None + if not fetched: + own = create(**{**BASE, "max_items": 1, "output_format": "html"}) + client.wait(own.crawl_id) + src = own.crawl_id + fetched = first_fetched(src) if not fetched: - return SKIP, "no result had been fetched", "" - plain = client.content(b.crawl_id, fetched) - as_file = client.content(b.crawl_id, fetched, download=True) + return FAIL, f"no result with a content_url on {src}", "" + plain = client.content(src, fetched) + as_file = client.content(src, fetched, download=True) return ( pass_if(plain == as_file and bool(plain)), f"same {len(plain)} chars with and without download=True", From d7f65587d9663fea6b2e398a042f6d4847a97f57 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 08:27:55 +0000 Subject: [PATCH 3/5] chore(crawl): record the contract this surface follows The lock names the API description this surface was synced to, by its blob id, so a later drift check compares against it instead of reporting the surface as never synced. Two operations in that description are not served to callers and no surface exposes them; the lock lists them as pending, so a change to either does not read as drift here. --- acceptance/crawl/contract.lock | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 acceptance/crawl/contract.lock diff --git a/acceptance/crawl/contract.lock b/acceptance/crawl/contract.lock new file mode 100644 index 0000000..435df9b --- /dev/null +++ b/acceptance/crawl/contract.lock @@ -0,0 +1,3 @@ +1adcbc2fd87e3f508317f3ce24c3d48573aec657 +pending GET /healthz +pending POST /internal/v1/crawls/{crawl_id}/stop From 876134d05b474499221bbe0b047c0f19d1b973a9 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 14:13:35 +0000 Subject: [PATCH 4/5] test(crawl): run the README quickstart where CRAWL_API_BASE points The quickstart leaves the base URL out, so a run against another base URL still sent that one crawl to production. The harness now adds the run's base URL to it, and its start URL pattern matches the README again. --- acceptance/crawl/surface/test.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/acceptance/crawl/surface/test.py b/acceptance/crawl/surface/test.py index 92557ca..2203ce2 100644 --- a/acceptance/crawl/surface/test.py +++ b/acceptance/crawl/surface/test.py @@ -134,7 +134,7 @@ def crawls_since(t): ) subs = [ - ('"https://example.com/products/"', json.dumps(CRAWL_URL), "start URL → CRAWL_URL"), + ('"https://www.scrapingcourse.com/ecommerce/"', json.dumps(CRAWL_URL), "start URL → CRAWL_URL"), ("max_items=50,", "max_items=5,", "max_items 50 → 5"), ("max_pages=100,", "max_pages=10,", "max_pages 100 → 10 (at most 10)"), ( @@ -143,6 +143,17 @@ def crawls_since(t): "include → CRAWL_INCLUDE", ), ] +if API_BASE != "https://api.zenrows.com/v1": + # The quickstart leaves the base URL out, so it would reach production + # whatever CRAWL_API_BASE says. Point it where the rest of the run goes. + subs.append( + ( + 'ZenRowsCrawlClient(api_key=os.environ["ZENROWS_API_KEY"])', + 'ZenRowsCrawlClient(api_key=os.environ["ZENROWS_API_KEY"], ' + f"base_url={json.dumps(API_BASE)})", + "base URL → CRAWL_API_BASE", + ) + ) quick, applied = block or "", [] for a, b, why in subs: if a in quick: From 66beb04799ab2350f3f57c2e160d28a119bac5cc Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 15:25:00 +0000 Subject: [PATCH 5/5] test(crawl): record the gated output and discovery values in the lock The values that the json output and pagination discovery use stay off for every user. The lock now records them as held, so the drift check reads their flag on each pass. --- acceptance/crawl/contract.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/acceptance/crawl/contract.lock b/acceptance/crawl/contract.lock index 435df9b..0d3bf74 100644 --- a/acceptance/crawl/contract.lock +++ b/acceptance/crawl/contract.lock @@ -1,3 +1,5 @@ 1adcbc2fd87e3f508317f3ce24c3d48573aec657 pending GET /healthz pending POST /internal/v1/crawls/{crawl_id}/stop +held output_format: json +held discovery: pagination