Skip to content
10 changes: 7 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -532,7 +532,10 @@ for result in client.results(crawl.crawl_id): # follows next_cursor
`wait` returns the crawl when it ends, or when `timeout` (seconds,
default 600) runs out. It does not raise: on timeout the crawl is still
`running` (call `wait` again, or `stop`), and a `failed` crawl comes
back with `crawl.error` set. `results` does not poll: it returns at the first empty page, so on a
back with `crawl.error` set. A crawl that ran out of credits, or whose
API key hit a credit cap, ends `stopped` with `crawl.error.code`
`insufficient_credits` or `api_key_cap_reached` and no `stop_reason`; the
pages it fetched before that stay readable. `results` does not poll: it returns at the first empty page, so on a
running crawl it yields only the URLs kept so far. Call `wait` first to
read every result. `results(crawl_id, limit=...)` sets the page size of
each request. `get(crawl_id, cursor=..., limit=...)` returns one raw page.
Expand Down Expand Up @@ -613,8 +616,9 @@ except CrawlAPIError as exc:
| 503 | `crawl_busy` | `stop` could not be saved yet; the crawl is still running. `stop` retries it up to `retries` times, waiting `exc.retry_after` seconds, then raises it. Repeating the stop is safe. |

`code` is None when the response has no problem body with a code (for
example a 502 from a proxy). A crawl that ended `failed` is not an error
response: read `crawl.error.code` and `crawl.error.detail` on it. Response enums (`CrawlStatus`, `StopReason`,
example a 502 from a proxy). A crawl that ended `failed`, or `stopped` on
credits, is not an error response: read `crawl.error.code` and
`crawl.error.detail` on it. Response enums (`CrawlStatus`, `StopReason`,
`ContentStatus`, ...) are open: a value added later parses as an
`UNKNOWN` member that keeps the raw value.

Expand Down
6 changes: 5 additions & 1 deletion examples/11_crawl.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@
- `create(url, depth=..., include_patterns=..., output_format="html")`
starts a crawl and returns at once, `running`.
- `wait(crawl_id)` polls until it ends, or 600 s pass, and returns
the `Crawl` (still `running` on timeout; `error` set when `failed`).
the `Crawl` (still `running` on timeout; `error` set when `failed`,
or when it `stopped` because credits ran out or the key hit a cap).
- `results(crawl_id)` pages through the kept URLs.
- `content(crawl_id, result)` reads one kept URL's HTML.
- `download(crawl_id)` reads every result as NDJSON, with the crawl's
Expand Down Expand Up @@ -58,6 +59,9 @@ def main() -> None:
)
if crawl.status is CrawlStatus.FAILED and crawl.error:
raise SystemExit(f"crawl failed: {crawl.error.code.value}: {crawl.error.detail}")
if crawl.status is CrawlStatus.STOPPED and crawl.error:
# Out of credits or at the key's cap: what it fetched stays readable.
print(f"stopped: {crawl.error.code.value}: {crawl.error.detail}")
if crawl.status is CrawlStatus.RUNNING:
print("still running: the results below are what it has kept so far")

Expand Down
2 changes: 2 additions & 0 deletions src/zenrows/crawl/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,8 @@ def wait(self, crawl_id: str, *, timeout: float = 600.0) -> Crawl:
`failed`) or `timeout` seconds pass, and return it.

A `failed` crawl is returned, not raised: read `error` on it.
So is a crawl that `stopped` because credits ran out or the API
key hit a credit cap; `error` says which.
On timeout the crawl is returned still `running`; call `wait`
again to keep waiting, or `stop` to end it.
"""
Expand Down
16 changes: 12 additions & 4 deletions src/zenrows/crawl/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,9 @@ class CrawlStatus(Enum):
class StopReason(Enum):
"""What ended the crawl before nothing was left to open: a limit
(`max_items` / `max_pages`, with `completed`) or the caller (`user`,
with `stopped`)."""
with `stopped`). Absent on a crawl that stopped because the account
ran out of credits or the API key hit a credit cap: that one carries
`error` instead."""

MAX_ITEMS = "max_items"
MAX_PAGES = "max_pages"
Expand Down Expand Up @@ -74,7 +76,11 @@ class DiscoveryStrategy(Enum):


class CrawlErrorCode(Enum):
"""Why a crawl failed (`Crawl.error.code`)."""
"""Why a crawl failed, or why it stopped on credits (`Crawl.error.code`).

`insufficient_credits` and `api_key_cap_reached` come with
`status` `stopped`: the pages fetched before that stay readable.
The other codes come with `failed`."""

INSUFFICIENT_CREDITS = "insufficient_credits"
API_KEY_CAP_REACHED = "api_key_cap_reached"
Expand All @@ -87,8 +93,10 @@ class CrawlErrorCode(Enum):


class RunError(BaseModel):
"""Why a crawl failed. Present only when `status` is `failed`; part
of the crawl, not an error response."""
"""Why a crawl failed, or why it stopped because the account ran out
of credits or the API key hit a credit cap. Present when `status` is
`failed`, and when it is `stopped` with `code` `insufficient_credits`
or `api_key_cap_reached`; part of the crawl, not an error response."""

code: CrawlErrorCode
detail: str
Expand Down
35 changes: 35 additions & 0 deletions tests/test_crawl_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,41 @@ def test_get_parses_failed_crawl_error(client: ZenRowsCrawlClient):
assert page.error is not None and page.error.code is CrawlErrorCode.SEED_UNREACHABLE


# What the server says when credits end a crawl, as error.detail.
CREDIT_STOP_DETAIL = {
"insufficient_credits": "The crawl stopped because your account has no credits left.",
"api_key_cap_reached": "The crawl stopped because your API key reached one of its credit caps.",
}


@pytest.mark.parametrize(
("code", "member"),
[
("insufficient_credits", CrawlErrorCode.INSUFFICIENT_CREDITS),
("api_key_cap_reached", CrawlErrorCode.API_KEY_CAP_REACHED),
],
)
@respx.mock
def test_wait_returns_crawl_stopped_on_credits(
client: ZenRowsCrawlClient, no_sleep, code: str, member: CrawlErrorCode
):
respx.get(f"{BASE_URL}/crawls/c_1").mock(
return_value=Response(
200,
json=crawl_body(
status="stopped",
error={"code": code, "detail": CREDIT_STOP_DETAIL[code]},
results=[],
next_cursor=None,
),
)
)
crawl = client.wait("c_1")
assert crawl.status is CrawlStatus.STOPPED
assert crawl.stop_reason is None
assert crawl.error is not None and crawl.error.code is member


@respx.mock
def test_get_omits_unset_query_params(client: ZenRowsCrawlClient):
route = respx.get(f"{BASE_URL}/crawls/c_1").mock(
Expand Down
Loading