diff --git a/README.md b/README.md index 68a15b1..edc99a7 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Start using the API by [creating your API Key](https://www.zenrows.com/register? The SDK uses [requests](https://docs.python-requests.org/) for HTTP requests. The client's response will be a requests `Response`. -It also uses [Retry](https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html) to automatically retry failed requests (status codes 422, 429, 500, 502, 503, and 504, except Extract's `422 no_data_extracted`). Retries are not active by default; you need to specify the number of retries, as shown below. It already includes an exponential back-off retry delay between failed requests. +The client retries a request up to 2 times by default (`retries=2`), and only when another try can help: a `429`, `502`, `503` or `504` response, none of which is charged, or a failure to connect. It never retries any other status (including any `422`) or a read timeout, because the server may still be working on that request. Between tries it waits the server's `Retry-After` header, or else a random time between 0.25 seconds and `2 ** attempt` seconds (full jitter); either wait is capped at `retry_max_wait` (default 30 seconds). It starts no retry whose wait would end more than `retry_budget` seconds (default 90) after the first try. Pass `retries=0` to turn retries off. `client.fetch()` is the primary method for the main page-scraping product; `client.get()` still works and is kept as a deprecated alias. diff --git a/pyproject.toml b/pyproject.toml index 63bc1bb..49e9490 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,9 @@ requires-python = ">=3.10" dependencies = [ # Legacy scraping client (zenrows.ZenRowsClient). "requests>=2.31", + # The client's retry policy subclasses urllib3's Retry; `allowed_methods` + # needs 1.26. + "urllib3>=1.26", # Batch (zenrows.batch) — pydantic v2 models + httpx transport. "httpx>=0.27", "pydantic>=2.9", diff --git a/src/zenrows/client.py b/src/zenrows/client.py index 2a38502..fc74b31 100644 --- a/src/zenrows/client.py +++ b/src/zenrows/client.py @@ -12,7 +12,9 @@ """ import asyncio +import contextvars import os +import random import time from concurrent.futures import ThreadPoolExecutor from functools import partial @@ -21,6 +23,7 @@ import requests import urllib3 from requests.adapters import HTTPAdapter +from urllib3.exceptions import InvalidHeader, MaxRetryError, ResponseError from urllib3.util.retry import Retry from zenrows.__version__ import __version__ @@ -28,11 +31,129 @@ DEFAULT_SCRAPER_URL = "https://api.zenrows.com/v1/" DEFAULT_USER_AGENT = f"zenrows/{__version__} python" -# Status codes the urllib3 retry layer treats as transient. 422 is retried -# too, but in `_worker`, because whether it is worth retrying depends on the -# response body (see `is_no_data_extracted`). -_RETRY_STATUSES = [429, 500, 502, 503, 504] -_RETRY_BACKOFF_FACTOR = 0.5 +# Status codes worth another try. None of them is charged: 429 is a rate +# limit, 503 is the gateway asking to retry later, 502 and 504 never reached +# the target. Every other 4xx/5xx is the final answer. +_RETRY_STATUSES = (429, 502, 503, 504) +DEFAULT_RETRIES = 2 +# Longest wait between two tries, in seconds: the cap on `Retry-After` and on +# the jittered backoff. Same cap as the Crawl client's `stop()` retry. +DEFAULT_RETRY_MAX_WAIT = 30.0 +# Longest a call may keep retrying, in seconds, counted from its first try: +# no retry starts if the wait before it would end past this. +DEFAULT_RETRY_BUDGET = 90.0 +# Shortest jittered wait, so a retry never fires back to back. +_RETRY_MIN_WAIT = 0.25 + +# Clock for the retry budget; tests replace it. +_monotonic = time.monotonic + +# When the current call's first try started. Set per call by +# `_RetryBudgetAdapter.send`; a context variable, so concurrent calls on +# the client's thread pool each see their own. +_call_started: contextvars.ContextVar[float | None] = contextvars.ContextVar( + "zenrows_call_started", default=None +) + + +class _JitteredRetry(Retry): + """urllib3 `Retry` with the Zenrows client retry policy. + + Between tries it waits the server's `Retry-After` (delta-seconds or + HTTP-date), capped at `retry_max_wait`; without one it waits a random time + in `[0.25, min(retry_max_wait, 2 ** attempt)]` seconds (full jitter, + `attempt` counting from 0). It never starts a retry whose wait would end + more than `retry_budget` seconds after the call's first try. + + Only the hooks urllib3 1.26 and 2.x share are overridden, so it works on + both (no `backoff_jitter` / `retry_after_max`). + """ + + # urllib3 also retries 413 when it carries `Retry-After`; we don't. + RETRY_AFTER_STATUS_CODES = frozenset({429, 503}) + + def __init__( + self, + *args: Any, + retry_max_wait: float = DEFAULT_RETRY_MAX_WAIT, + retry_budget: float = DEFAULT_RETRY_BUDGET, + **kwargs: Any, + ): + super().__init__(*args, **kwargs) + self.retry_max_wait = retry_max_wait + self.retry_budget = retry_budget + self._started: float | None = None + # The wait before the next try, drawn once in `increment` so the + # budget check and the sleep agree on it. + self._delay: float | None = None + + def new(self, **kw: Any) -> "_JitteredRetry": + retry = super().new(**kw) + retry.retry_max_wait = self.retry_max_wait + retry.retry_budget = self.retry_budget + started = self._started + if started is None: + started = _call_started.get() + if started is None: + started = _monotonic() + retry._started = started + return retry + + def increment( + self, + method: str | None = None, + url: str | None = None, + response: Any = None, + error: Exception | None = None, + _pool: Any = None, + _stacktrace: Any = None, + ) -> "_JitteredRetry": + retry = super().increment(method, url, response, error, _pool, _stacktrace) + retry._delay = retry._next_delay(response) + started = retry._started if retry._started is not None else _monotonic() + if _monotonic() - started + retry._delay > retry.retry_budget: + reason = error or ResponseError(f"retry budget of {retry.retry_budget}s spent") + raise MaxRetryError(_pool, url, reason) from reason + return retry + + def get_backoff_time(self) -> float: + if self._delay is not None: + return self._delay + return self._jitter() + + def get_retry_after(self, response: Any) -> float | None: + try: + retry_after = super().get_retry_after(response) + except InvalidHeader: + return None # unparseable: fall back to the jittered backoff + if retry_after is None: + return None + return min(retry_after, self.retry_max_wait) + + def _next_delay(self, response: Any) -> float: + # Mirrors `Retry.sleep`: a positive `Retry-After` wins over the backoff. + if response is not None and self.respect_retry_after_header: + retry_after = self.get_retry_after(response) + if retry_after: + return retry_after + return self._jitter() + + def _jitter(self) -> float: + attempt = min(max(len(self.history) - 1, 0), 30) + ceiling = min(self.retry_max_wait, 2.0**attempt) + return random.uniform(min(_RETRY_MIN_WAIT, ceiling), ceiling) + + +class _RetryBudgetAdapter(HTTPAdapter): + """Marks when each call's first try starts, for the retry budget.""" + + def send(self, request: Any, *args: Any, **kwargs: Any) -> requests.Response: + token = _call_started.set(_monotonic()) + try: + return super().send(request, *args, **kwargs) + finally: + _call_started.reset(token) + # Extract's answer when it knows the kind of page but none of the fields came # back filled. Retrying returns the same answer, so it is never retried. @@ -105,6 +226,13 @@ class ZenRowsClient: `base_url` defaults to the public production endpoint; override via constructor arg or `ZENROWS_SCRAPER_BASE_URL` env var. + + `retries` (default 2) retries a 429, 502, 503 or 504, and a failure to + connect, never any other status or a read timeout. Between tries it waits + the server's `Retry-After`, or else a random 0.25s to `2 ** attempt` + seconds; either is capped at `retry_max_wait` (default 30). No retry starts + if its wait would end more than `retry_budget` seconds (default 90) after + the first try. Pass `retries=0` to turn retries off. """ # Kept as a class attribute for compatibility — pre-1.4 callers @@ -114,10 +242,12 @@ class ZenRowsClient: def __init__( self, apikey: str, - retries: int = 0, + retries: int = DEFAULT_RETRIES, concurrency: int = 5, *, base_url: str | None = None, + retry_max_wait: float = DEFAULT_RETRY_MAX_WAIT, + retry_budget: float = DEFAULT_RETRY_BUDGET, ): if not apikey: raise ValueError("ZenRowsClient: apikey is required.") @@ -127,14 +257,25 @@ def __init__( self.retries = retries self.executor = ThreadPoolExecutor(max_workers=concurrency) self.requests_session = requests.Session() + self.retry_max_wait = retry_max_wait + self.retry_budget = retry_budget if retries > 0: - max_retries = Retry( + # Retries a 429/502/503/504 and a failure to connect, nothing + # else: a read error or timeout may come after the server + # accepted the request, which keeps running and may be charged. + max_retries = _JitteredRetry( total=retries, - backoff_factor=_RETRY_BACKOFF_FACTOR, + connect=retries, + read=False, # re-raise as is, so it stays a ReadTimeout + status=retries, + other=0, + allowed_methods=None, status_forcelist=_RETRY_STATUSES, raise_on_status=False, + retry_max_wait=retry_max_wait, + retry_budget=retry_budget, ) - adapter = HTTPAdapter(max_retries=max_retries) + adapter = _RetryBudgetAdapter(max_retries=max_retries) self.requests_session.mount("https://", adapter) self.requests_session.mount("http://", adapter) @@ -354,7 +495,7 @@ def _worker( final_headers["Connection"] = None final_headers.update(headers) - response = self.requests_session.request( + return self.requests_session.request( method, self.api_url, params=final_params, @@ -362,25 +503,3 @@ def _worker( data=data, **kwargs, ) - # Retry a 422 here rather than in urllib3, which can't read the body: - # Extract's `no_data_extracted` is a final answer, any other 422 is a - # failure to get the target's content and is worth another try. - attempt = 0 - while ( - response.status_code == 422 - and attempt < self.retries - and not is_no_data_extracted(response) - ): - attempt += 1 - # Same schedule as urllib3's Retry: no wait before the first retry. - if attempt > 1: - time.sleep(_RETRY_BACKOFF_FACTOR * 2 ** (attempt - 1)) - response = self.requests_session.request( - method, - self.api_url, - params=final_params, - headers=final_headers, - data=data, - **kwargs, - ) - return response diff --git a/tests/test_retries.py b/tests/test_retries.py index 0fb37d5..7d39ae9 100644 --- a/tests/test_retries.py +++ b/tests/test_retries.py @@ -1,114 +1,442 @@ +"""The ZenRowsClient retry policy, end to end against a local HTTP server. + +Retried: 429, 502, 503, 504 and a failure to connect. Never retried: any +other status (422 included) or a read timeout. Waits honour `Retry-After` +(capped at `retry_max_wait`) or else use full jitter, and no retry starts +past `retry_budget`. Sleeps are patched out; the budget runs on a fake clock. +""" + import asyncio -from unittest import TestCase +import socket +import threading +from collections.abc import Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from unittest.mock import patch +from urllib.parse import parse_qs, urlparse -from requests import Response, Session +import pytest +import requests +from requests.adapters import HTTPAdapter from zenrows import ZenRowsClient, is_no_data_extracted -from zenrows.client import _RETRY_STATUSES +from zenrows.client import _JitteredRetry -apikey = "APIKEY" -url = "http://example.com" -api_url_base = "https://api.zenrows.com/v1/" +APIKEY = "APIKEY" +TARGET = "http://example.com" +NO_DATA = b'{"code": "no_data_extracted", "title": "No data extracted"}' +NO_CONTENT = b'{"code": "RESP001", "title": "Could not get content"}' +DOMAIN_NOT_PREPARED = b'{"code": "REQS007", "title": "Domain not prepared"}' -class TestZenRowsClientRetries(TestCase): - @patch("zenrows.client.Retry") - @patch.object(Session, "mount") - def test_custom_session_not_initiated(self, mock_mount, mock_retry): - ZenRowsClient(apikey, retries=0) +# A scripted response: (status, headers, body). HANG answers only after the +# client's read timeout has passed. +HANG = (0, {}, b"") - mock_retry.assert_not_called() - self.assertEqual(mock_mount.call_count, 2) # called internally - @patch("zenrows.client.Retry") - def test_retry_parameters(self, mock_retry): - ZenRowsClient(apikey, retries=2) +class _Server: + """Answers each request with the next scripted response (the last one + repeats) and records the query of every request it got.""" - mock_retry.assert_called_once_with( - total=2, - backoff_factor=0.5, - status_forcelist=list(_RETRY_STATUSES), - raise_on_status=False, - ) + def __init__(self) -> None: + self.script: list[tuple[int, dict[str, str], bytes]] = [(200, {}, b"ok")] + self.queries: list[dict[str, list[str]]] = [] + self.on_request = lambda: None + server = self - @patch.object(Session, "mount") - def test_adapter_is_mount(self, mock_mount): - ZenRowsClient(apikey, retries=2) + class Handler(BaseHTTPRequestHandler): + def _answer(self) -> None: + length = int(self.headers.get("Content-Length") or 0) + if length: + self.rfile.read(length) + server.queries.append(parse_qs(urlparse(self.path).query)) + server.on_request() + status, headers, body = ( + server.script.pop(0) if len(server.script) > 1 else server.script[0] + ) + if status == 0: + threading.Event().wait(0.5) # not time.sleep: tests patch it + status = 200 + try: + self.send_response(status) + for name, value in headers.items(): + self.send_header(name, value) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + except OSError: + pass # the client gave up first - self.assertEqual(mock_mount.call_count, 4) + do_GET = do_POST = _answer + def log_message(self, *args: object) -> None: + pass -def _fake_response(status_code: int, body: bytes = b"") -> Response: - response = Response() - response.status_code = status_code - response._content = body - return response + self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.url = f"http://127.0.0.1:{self.httpd.server_address[1]}/" + threading.Thread( + target=self.httpd.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True + ).start() + @property + def calls(self) -> int: + return len(self.queries) -NO_DATA = b'{"code": "no_data_extracted", "title": "No data extracted"}' -NO_CONTENT = b'{"code": "RESP001", "title": "Could not get content"}' + def close(self) -> None: + self.httpd.shutdown() + self.httpd.server_close() + + +@pytest.fixture +def server() -> Iterator[_Server]: + srv = _Server() + yield srv + srv.close() + + +@pytest.fixture +def sleeps() -> Iterator[list[float]]: + """Every wait between tries, in seconds; none of them actually waits.""" + waited: list[float] = [] + with patch("urllib3.util.retry.time.sleep", side_effect=waited.append): + yield waited + + +def client(server: _Server, **kwargs) -> ZenRowsClient: + return ZenRowsClient(APIKEY, base_url=server.url, **kwargs) + + +# ---- configuration ---- + + +def test_default_is_two_retries() -> None: + zr = ZenRowsClient(APIKEY) + + retry = zr.requests_session.get_adapter("https://api.zenrows.com/v1/").max_retries + assert zr.retries == 2 + assert isinstance(retry, _JitteredRetry) + assert (retry.total, retry.status, retry.connect, retry.other) == (2, 2, 2, 0) + assert retry.read is False + assert retry.allowed_methods is None + assert set(retry.status_forcelist) == {429, 502, 503, 504} + assert (retry.retry_max_wait, retry.retry_budget) == (30.0, 90.0) + + +def test_retries_zero_mounts_no_retry(server: _Server, sleeps: list[float]) -> None: + zr = client(server, retries=0) + server.script = [(503, {}, b"")] + + response = zr.fetch(TARGET) + + assert type(zr.requests_session.get_adapter(server.url)) is HTTPAdapter + assert response.status_code == 503 + assert server.calls == 1 + assert sleeps == [] + + +def test_default_retries_twice_then_returns_the_last_response( + server: _Server, sleeps: list[float] +) -> None: + server.script = [(503, {}, b"")] + + response = client(server).fetch(TARGET) + + assert response.status_code == 503 + assert server.calls == 3 + assert len(sleeps) == 2 + + +# ---- what is retried ---- + + +@pytest.mark.parametrize("status", [429, 502, 503, 504]) +def test_retryable_status_is_retried_until_success( + server: _Server, sleeps: list[float], status: int +) -> None: + server.script = [(status, {}, b""), (200, {}, b"ok")] + + response = client(server).fetch(TARGET) + + assert response.status_code == 200 + assert response.text == "ok" + assert server.calls == 2 + + +def test_post_is_retried(server: _Server, sleeps: list[float]) -> None: + server.script = [(503, {}, b""), (200, {}, b"ok")] + + response = client(server).post(TARGET, data="payload") + + assert response.status_code == 200 + assert server.calls == 2 + + +@pytest.mark.parametrize( + ("status", "body"), + [(400, b""), (401, b""), (404, b""), (422, NO_CONTENT), (422, NO_DATA), (500, b"")], +) +def test_other_statuses_are_not_retried( + server: _Server, sleeps: list[float], status: int, body: bytes +) -> None: + server.script = [(status, {}, body)] + + response = client(server, retries=3).fetch(TARGET) + + assert response.status_code == status + assert server.calls == 1 + assert sleeps == [] + + +def test_413_with_retry_after_is_not_retried(server: _Server, sleeps: list[float]) -> None: + server.script = [(413, {"Retry-After": "1"}, b"")] + + response = client(server).fetch(TARGET) + + assert response.status_code == 413 + assert server.calls == 1 + + +def test_read_timeout_is_not_retried(server: _Server, sleeps: list[float]) -> None: + server.script = [HANG] + + with pytest.raises(requests.exceptions.ReadTimeout): + client(server, retries=3).fetch(TARGET, timeout=0.1) + + assert server.calls == 1 + assert sleeps == [] + + +def test_connection_refused_is_retried(sleeps: list[float]) -> None: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] # closed once the block ends: nothing listens + + zr = ZenRowsClient(APIKEY, base_url=f"http://127.0.0.1:{port}/") + with pytest.raises(requests.exceptions.ConnectionError): + zr.fetch(TARGET) + + assert len(sleeps) == 2 + + +# ---- how long it waits ---- + + +def test_jitter_draws_between_quarter_second_and_doubling_ceiling( + server: _Server, sleeps: list[float] +) -> None: + server.script = [(503, {}, b"")] * 4 + [(200, {}, b"ok")] + with patch("zenrows.client.random.uniform", side_effect=lambda lo, hi: hi) as uniform: + response = client(server, retries=4).fetch(TARGET) -@patch("zenrows.client.time.sleep") -class TestUnprocessableRetries(TestCase): - """A 422 is retried in the client, not in urllib3, because only the body - says whether another try can help.""" + assert response.status_code == 200 + assert [c.args for c in uniform.call_args_list] == [ + (0.25, 1.0), + (0.25, 2.0), + (0.25, 4.0), + (0.25, 8.0), + ] + assert sleeps == [1.0, 2.0, 4.0, 8.0] - @patch.object(Session, "request") - def test_no_data_extracted_is_not_retried(self, mock_request, mock_sleep): - mock_request.return_value = _fake_response(422, NO_DATA) - response = ZenRowsClient(apikey, retries=3).extract(url) +def test_jitter_stays_in_bounds(server: _Server, sleeps: list[float]) -> None: + for _ in range(20): + sleeps.clear() + server.script = [(503, {}, b"")] * 3 + [(200, {}, b"ok")] + client(server, retries=3).fetch(TARGET) + assert len(sleeps) == 3 + for attempt, waited in enumerate(sleeps): + assert 0.25 <= waited <= 2.0**attempt - self.assertEqual(response.status_code, 422) - self.assertEqual(mock_request.call_count, 1) - mock_sleep.assert_not_called() - @patch.object(Session, "request") - def test_other_422_is_retried_up_to_retries(self, mock_request, mock_sleep): - mock_request.return_value = _fake_response(422, NO_CONTENT) +def test_retry_max_wait_caps_the_jitter() -> None: + retry = _JitteredRetry(total=10, retry_max_wait=3.0) + for _ in range(6): + retry = retry.new(history=(*retry.history, None)) - response = ZenRowsClient(apikey, retries=3).fetch(url) + with patch("zenrows.client.random.uniform", side_effect=lambda lo, hi: (lo, hi)): + assert retry.get_backoff_time() == (0.25, 3.0) - self.assertEqual(response.status_code, 422) - self.assertEqual(mock_request.call_count, 4) # 1 + 3 retries - self.assertEqual([c.args[0] for c in mock_sleep.call_args_list], [1.0, 2.0]) - @patch.object(Session, "request") - def test_other_422_stops_retrying_on_success(self, mock_request, mock_sleep): - mock_request.side_effect = [_fake_response(422, NO_CONTENT), _fake_response(200, b"ok")] +def test_retry_after_is_honoured(server: _Server, sleeps: list[float]) -> None: + server.script = [(503, {"Retry-After": "7"}, b""), (200, {}, b"ok")] - response = ZenRowsClient(apikey, retries=3).fetch(url) + response = client(server).fetch(TARGET) - self.assertEqual(response.status_code, 200) - self.assertEqual(mock_request.call_count, 2) + assert response.status_code == 200 + assert sleeps == [7.0] - @patch.object(Session, "request") - def test_422_not_retried_without_retries(self, mock_request, mock_sleep): - mock_request.return_value = _fake_response(422, NO_CONTENT) - ZenRowsClient(apikey).fetch(url) +def test_retry_after_is_capped_at_30_seconds(server: _Server, sleeps: list[float]) -> None: + server.script = [(429, {"Retry-After": "120"}, b""), (200, {}, b"ok")] - self.assertEqual(mock_request.call_count, 1) + client(server).fetch(TARGET) - @patch.object(Session, "request") - def test_no_data_extracted_is_not_retried_async(self, mock_request, mock_sleep): - mock_request.return_value = _fake_response(422, NO_DATA) + assert sleeps == [30.0] - response = asyncio.run(ZenRowsClient(apikey, retries=3).extract_async(url)) - self.assertEqual(response.status_code, 422) - self.assertEqual(mock_request.call_count, 1) +def test_retry_after_is_capped_at_retry_max_wait(server: _Server, sleeps: list[float]) -> None: + server.script = [(503, {"Retry-After": "20"}, b""), (200, {}, b"ok")] + + client(server, retry_max_wait=5.0).fetch(TARGET) + + assert sleeps == [5.0] + + +def test_unparseable_retry_after_falls_back_to_jitter(server: _Server, sleeps: list[float]) -> None: + server.script = [(503, {"Retry-After": "soon"}, b""), (200, {}, b"ok")] + + response = client(server).fetch(TARGET) + + assert response.status_code == 200 + assert len(sleeps) == 1 + assert 0.25 <= sleeps[0] <= 1.0 + + +# ---- the retry budget ---- + + +class _Clock: + def __init__(self) -> None: + self.now = 0.0 + + def __call__(self) -> float: + return self.now + + +@pytest.fixture +def clock() -> Iterator[tuple[_Clock, list[float]]]: + """A fake clock for the retry budget, moved on by every wait, and the + list of those waits.""" + fake = _Clock() + waited: list[float] = [] + + def wait(seconds: float) -> None: + waited.append(seconds) + fake.now += seconds + + with ( + patch("zenrows.client._monotonic", fake), + patch("urllib3.util.retry.time.sleep", side_effect=wait), + ): + yield fake, waited + + +def test_no_retry_starts_past_the_budget( + server: _Server, clock: tuple[_Clock, list[float]] +) -> None: + _, sleeps = clock + server.script = [(503, {"Retry-After": "20"}, b"")] + + response = client(server, retries=10).fetch(TARGET) + + # Waits end at 20, 40, 60, 80 s; a fifth would end at 100 s > 90 s. + assert response.status_code == 503 + assert sleeps == [20.0, 20.0, 20.0, 20.0] + assert server.calls == 5 + + +def test_budget_counts_the_first_try(server: _Server, clock: tuple[_Clock, list[float]]) -> None: + fake, sleeps = clock + server.script = [(503, {"Retry-After": "10"}, b"")] + + def slow_first_try() -> None: + if server.calls == 1: + fake.now += 85.0 + + server.on_request = slow_first_try + + response = client(server).fetch(TARGET) + + # 85 s spent + a 10 s wait would end past 90 s. + assert response.status_code == 503 + assert server.calls == 1 + assert sleeps == [] + + +def test_budget_is_per_call(server: _Server, clock: tuple[_Clock, list[float]]) -> None: + _, sleeps = clock + zr = client(server, retries=1) + for _ in range(4): + server.script = [(503, {"Retry-After": "30"}, b""), (200, {}, b"ok")] + assert zr.fetch(TARGET).status_code == 200 + + assert sleeps == [30.0] * 4 + + +def test_custom_budget(server: _Server, clock: tuple[_Clock, list[float]]) -> None: + _, sleeps = clock + server.script = [(503, {"Retry-After": "10"}, b"")] + + response = client(server, retry_budget=5.0).fetch(TARGET) + + assert response.status_code == 503 + assert server.calls == 1 + assert sleeps == [] + + +# ---- Extract ---- + + +def test_no_data_extracted_is_not_retried(server: _Server, sleeps: list[float]) -> None: + server.script = [(422, {}, NO_DATA)] + + response = client(server, retries=3).extract(TARGET) + + assert is_no_data_extracted(response) + assert server.calls == 1 + + +def test_autoparse_fallback_is_unchanged(server: _Server, sleeps: list[float]) -> None: + server.script = [(403, {}, DOMAIN_NOT_PREPARED), (200, {}, b"{}")] + + response = client(server).extract(TARGET) + + assert response.status_code == 200 + assert server.calls == 2 + assert server.queries[0]["extract"] == ["auto"] + assert server.queries[1]["autoparse"] == ["True"] + assert "extract" not in server.queries[1] + assert sleeps == [] + + +def test_autoparse_fallback_is_unchanged_async(server: _Server, sleeps: list[float]) -> None: + server.script = [(403, {}, DOMAIN_NOT_PREPARED), (200, {}, b"{}")] + + response = asyncio.run(client(server).extract_async(TARGET)) + + assert response.status_code == 200 + assert server.calls == 2 + assert server.queries[1]["autoparse"] == ["True"] + + +def test_extract_retries_a_503_before_the_fallback(server: _Server, sleeps: list[float]) -> None: + server.script = [(503, {}, b""), (403, {}, DOMAIN_NOT_PREPARED), (200, {}, b"{}")] + + response = client(server).extract(TARGET) + + assert response.status_code == 200 + assert server.calls == 3 + assert len(sleeps) == 1 + assert server.queries[2]["autoparse"] == ["True"] + + +# ---- is_no_data_extracted ---- + + +def _fake_response(status_code: int, body: bytes = b"") -> requests.Response: + response = requests.Response() + response.status_code = status_code + response._content = body + return response + +def test_is_no_data_extracted_matches_any_case() -> None: + assert is_no_data_extracted(_fake_response(422, NO_DATA)) + assert is_no_data_extracted(_fake_response(422, b'{"code": "NO_DATA_EXTRACTED"}')) -class TestIsNoDataExtracted(TestCase): - def test_matches_422_no_data_extracted_any_case(self): - self.assertTrue(is_no_data_extracted(_fake_response(422, NO_DATA))) - self.assertTrue(is_no_data_extracted(_fake_response(422, b'{"code": "NO_DATA_EXTRACTED"}'))) - def test_rejects_other_codes_statuses_and_bodies(self): - self.assertFalse(is_no_data_extracted(_fake_response(422, NO_CONTENT))) - self.assertFalse(is_no_data_extracted(_fake_response(400, NO_DATA))) - self.assertFalse(is_no_data_extracted(_fake_response(422, b"not json"))) - self.assertFalse(is_no_data_extracted(_fake_response(422, b"[]"))) +def test_is_no_data_extracted_rejects_other_codes_statuses_and_bodies() -> None: + assert not is_no_data_extracted(_fake_response(422, NO_CONTENT)) + assert not is_no_data_extracted(_fake_response(400, NO_DATA)) + assert not is_no_data_extracted(_fake_response(422, b"not json")) + assert not is_no_data_extracted(_fake_response(422, b"[]")) diff --git a/uv.lock b/uv.lock index ecde950..8a93685 100644 --- a/uv.lock +++ b/uv.lock @@ -978,6 +978,7 @@ dependencies = [ { name = "requests" }, { name = "tqdm" }, { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "urllib3" }, ] [package.dev-dependencies] @@ -999,6 +1000,7 @@ requires-dist = [ { name = "requests", specifier = ">=2.31" }, { name = "tqdm", specifier = ">=4.66" }, { name = "typing-extensions", marker = "python_full_version < '3.11'", specifier = ">=4.0" }, + { name = "urllib3", specifier = ">=1.26" }, ] [package.metadata.requires-dev]