From 44d6d98a6d11c8347e7808ba395c95262efe5a46 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Thu, 8 Oct 2026 16:53:04 +0200 Subject: [PATCH 01/19] feat(crawl): add `zenrows crawl` for the Crawl API Crawl takes one start URL, follows the links on each page up to `depth` on the start URL's domain, and returns the URLs it keeps, optionally with each page's HTML. The new `crawl` command drives https://api.zenrows.com/v1/crawls with the X-API-Key header: - start --depth N [--max-items N] [--max-pages N] [--include P]... [--exclude P]... [--html] [--follow|--wait] [--timeout ms] [--idempotency-key K] - status, wait, results (all pages, one page via --cursor/--limit, or the NDJSON export via --download), content, list, stop src/core/crawl-api.ts is the client (base override ZENROWS_CRAWL_API_BASE, otherwise the configured apiBase). Error responses map to new codes: CRAWL_NOT_ENABLED (403 REQS008), CRAWL_NOT_FOUND (404), CRAWL_QUOTA_EXCEEDED (429 too_many_crawls), CRAWL_INVALID (400/409/422), CRAWL_FAILED (a failed crawl or another error) and CRAWL_TIMEOUT (wait gave up; the crawl keeps running). The guards that asserted `crawl` is an unknown command and that no crawl skill exists are narrowed to `discover` and `monitor`. Adds the `crawl` capability, a crawl skill, README and docs sections, unit tests, and an opt-in e2e test (`npm run test:e2e`) that drives the built CLI against a live API. --- README.md | 52 +++- docs/capabilities.md | 1 + docs/contributing.md | 32 +++ package.json | 1 + registry/capabilities.json | 9 + registry/skills.json | 11 + skills/crawl/SKILL.md | 73 +++++ skills/trace-debug/SKILL.md | 8 + skills/zenrows/SKILL.md | 5 + src/cli/commands/crawl.ts | 473 +++++++++++++++++++++++++++++++ src/cli/index.ts | 3 +- src/core/crawl-api.ts | 459 ++++++++++++++++++++++++++++++ src/core/errors.ts | 6 + tests/capabilities.test.ts | 2 + tests/crawl-api.test.ts | 286 +++++++++++++++++++ tests/crawl-command.test.ts | 324 +++++++++++++++++++++ tests/e2e/crawl.e2e.test.ts | 141 +++++++++ tests/production-surface.test.ts | 5 +- tests/registry.test.ts | 7 +- 19 files changed, 1883 insertions(+), 15 deletions(-) create mode 100644 skills/crawl/SKILL.md create mode 100644 src/cli/commands/crawl.ts create mode 100644 src/core/crawl-api.ts create mode 100644 tests/crawl-api.test.ts create mode 100644 tests/crawl-command.test.ts create mode 100644 tests/e2e/crawl.e2e.test.ts diff --git a/README.md b/README.md index 4fd37d3..0973eaf 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ that endpoint with extraction parameters, not a separate product. | `zenrows fetch` | Fetch — retrieve a protected page | **available** — `GET https://api.zenrows.com/v1/` | | `zenrows extract` | Extract — `extract=auto` (domain-gated open beta; falls back to Autoparse) / CSS / Markdown | **beta** — same `/v1/` | | `zenrows batch` | Batch — fan out over many URLs | beta — cloud works with beta access; local validate/estimate always | +| `zenrows crawl` | Crawl — collect the URLs behind one start page, optionally with each page's HTML | **available** — `https://api.zenrows.com/v1/crawls` | | `zenrows browser` | Browser Sessions REST API (same backend as MCP `browser_*`) | **available** — escalation-only; bills by bandwidth + time | | `zenrows mcp` | MCP server config (remote + local) | **available** | | Zenrows CLI | this repo | available | @@ -156,7 +157,36 @@ An Extract task costs the same as a regular one (1 credit at base tier), so keys: `html` (the raw page) and `parsed` (the structured data) — validate a sample with `zenrows extract ` before running the full batch. -## 10. Browser Sessions +## 10. Crawl + +Zenrows **Crawl** (`https://api.zenrows.com/v1/crawls`) takes one start URL and +returns the URLs it finds behind it, optionally with each page's HTML. Use it +when you know the listing or section but not the item URLs. Crawl follows the +links on each page up to `--depth` hops and stays on the start URL's domain. +Without `--html` it returns URLs only. An account without Crawl access gets +`CRAWL_NOT_ENABLED` ("Crawl is not enabled for this account", HTTP 403 +`REQS008`). + +```bash +zenrows crawl start https://example.com/products/ --depth 1 \ + --max-items 20 --include /product/ --follow # start and wait for the end +zenrows crawl results --out urls.jsonl # every kept URL (paginated) +zenrows crawl start --depth 2 --html # also fetch each kept page's HTML +zenrows crawl content # one page's HTML +zenrows crawl results --download # NDJSON export (URLs + HTML) → .jsonl +zenrows crawl status # status, coverage, stop reason +zenrows crawl list # your crawls, newest first +zenrows crawl stop # stop a running crawl +``` + +`--depth` (link hops) is required; `--max-items` and `--max-pages` cap the +crawl (API default 10 each), and `--max-pages` bounds the cost, since each page +is one fetch. `--include` / `--exclude` are repeatable substring filters on the +URL. When the account has too many crawls running, a start gets +`CRAWL_QUOTA_EXCEEDED` and creates nothing; retry after `Retry-After`. Set +`ZENROWS_CRAWL_API_BASE` to point the CLI at another deployment. + +## 11. Browser Sessions Escalation only — **prefer `fetch`/`extract` for the vast majority of cases**; they cost less. Use the browser for logins, forms, and multi-step JS flows that @@ -170,7 +200,7 @@ bring-your-own Playwright/Puppeteer. close interactive sessions with `zenrows browser close`. On by default; opt out with `zenrows policy set allow_browser false`. -## 11. MCP +## 12. MCP ```bash zenrows mcp status @@ -188,7 +218,7 @@ both servers in `mcp.json` (no secrets) and ships MCP-native skills under `agent-plugin/skills/`. It is included in the published `@zenrows/cli` npm tarball (`node_modules/@zenrows/cli/agent-plugin`). -## 12. Plugins +## 13. Plugins ```bash zenrows plugin list @@ -201,7 +231,7 @@ MCP-native skills). See [`agent-plugin/README.md`](agent-plugin/README.md). Legacy per-client snippets from `zenrows plugin install` remain available. CLI-oriented skills stay under repo-root `skills/`. -## 13. Skills +## 14. Skills Agent-readable playbooks that teach agents how to choose primitives. @@ -213,28 +243,28 @@ zenrows skill validate zenrows The master skill `skills/zenrows/SKILL.md` teaches the full decision tree. -## 14. Templates +## 15. Templates ```bash zenrows template list zenrows template create protected-fetch-node --output ./my-project ``` -## 15. Workflows +## 16. Workflows ```bash zenrows workflow list zenrows workflow explain competitor-intelligence ``` -## 16. Recipes +## 17. Recipes ```bash zenrows recipe list zenrows recipe run fetch-protected-page ``` -## 17. Evals +## 18. Evals Reproducible, transparent benchmarks. **No competitor keys are bundled** and nothing is hardcoded to make Zenrows win — comparison evals require you to supply @@ -249,7 +279,7 @@ zenrows eval report protected-fetch-smoke Reports write `input.json`, `results.json`, `report.md`, `failures.jsonl`, `cost.json`, and `traces/` under `.zenrows/evals//`. -## 18. Security +## 19. Security - API keys are never printed or written into run artifacts/assets. - Secrets live in `.zenrows/secrets.json` (0600, gitignored); logs are redacted. @@ -259,12 +289,12 @@ Reports write `input.json`, `results.json`, `report.md`, `failures.jsonl`, - Destructive `uninstall` requires `--yes`. Browser is on by default; opt out with `zenrows policy set allow_browser false`. Experimental features are off by default. -## 19. Capability matrix +## 20. Capability matrix See [`registry/capabilities.json`](registry/capabilities.json). `zenrows status --json` emits it. -## 20. Contributing +## 21. Contributing The CLI is TypeScript with **zero runtime dependencies** (native `fetch`, `node:util` `parseArgs`, `node:test`). Node 20+ runs the published build; diff --git a/docs/capabilities.md b/docs/capabilities.md index fd7939b..2557ee7 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -22,5 +22,6 @@ Classification is based on the public Zenrows documentation: | `protected_fetch` | Fetch `GET https://api.zenrows.com/v1/` with `mode`, `js_render`, `premium_proxy`, `proxy_country`, `wait`/`wait_for`, `js_instructions`, `response_type`, `screenshot`, `original_status`, … | available | | `extract` | Extract `GET https://api.zenrows.com/v1/` via `extract=auto` (domain-gated open beta; CLI falls back to `autoparse`), plus `autoparse`, `css_extractor`, `outputs`, `response_type=markdown\|plaintext` | beta | | `batch` | Batch `https://async.api.zenrows.com/v1` (separate host, `X-API-Key` header) — open beta. Local JSONL spec validation + credit estimation work with no key. | beta | +| `crawl` | Crawl `https://api.zenrows.com/v1/crawls` (`X-API-Key` header; same host as Fetch/Extract). Follows the links on each page from one start URL up to `depth`, on the start URL's domain; returns URLs, and with `output_format: "html"` each page's HTML. Without Crawl access the API returns 403 `REQS008` → `CRAWL_NOT_ENABLED`. | available | | `browser` | Browser Sessions REST API `https://mcp.zenrows.com/browser/sessions/*` (Bearer; same backend as `@zenrows/mcp` `browser_*`); CDP via `zenrows browser connect`. Escalation-only (prefer fetch/extract); on by default, opt out via `policy.allow_browser`; bills by bandwidth + session time (15-min max) | available | | `mcp` | Hosted `https://mcp.zenrows.com/mcp` + local `npx -y @zenrows/mcp` | available | diff --git a/docs/contributing.md b/docs/contributing.md index bcd9989..cc3a519 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -37,6 +37,38 @@ npm run build # tsc → dist/ (for publishing) 4. Add a skill + recipe/eval and declare `requires_backend_capabilities`. 5. Add tests. Never fake backend behavior; return a normalized error instead. +## Running the Crawl e2e test + +`tests/e2e/crawl.e2e.test.ts` drives the built CLI (`zenrows crawl …`) against a +live Zenrows API: it starts one small crawl of the target you give it (depth 1, +3 items, 5 pages, HTML output), waits for it, reads its results, one page's +HTML and the NDJSON export, lists crawls, stops the ended crawl and checks a +missing id. It is not part of `npm test`, and it skips unless `ZENROWS_E2E=1`, +`ZENROWS_API_KEY` and `ZENROWS_E2E_CRAWL_URL` are all set. It bills a few +pages on the key's account. + +| Variable | Value | +| --- | --- | +| `ZENROWS_E2E` | `1` to run the test | +| `ZENROWS_API_KEY` | a key with Crawl access | +| `ZENROWS_E2E_CRAWL_URL` | the start URL to crawl, e.g. `https://example.com/products/` | +| `ZENROWS_E2E_CRAWL_INCLUDE` | optional include pattern, e.g. `/product/`; every result URL must contain it | +| `ZENROWS_CRAWL_API_BASE` | optional API base, default `https://api.zenrows.com/v1` | + +Read the key from a file rather than typing it, so it stays out of your shell +history: + +```bash +export ZENROWS_API_KEY="$(tr -d '\n' < path/to/api.key)" +ZENROWS_E2E=1 ZENROWS_E2E_CRAWL_URL=https://example.com/products/ \ + ZENROWS_E2E_CRAWL_INCLUDE=/product/ npm run test:e2e +``` + +To target a local or staging deployment, set `ZENROWS_CRAWL_API_BASE` to its +`/v1` base. When the account has too many crawls running, the test waits +`Retry-After` and retries the start for up to 5 minutes, so run concurrent e2e +runs on one account one at a time. + ## Rules - Never print or persist API keys. Redact secrets in logs and artifacts. diff --git a/package.json b/package.json index ef1173b..8b10701 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "zenrows": "node bin/zenrows.ts", "dev": "node bin/zenrows.ts", "test": "npm run build && node --test --import ./dist/tests/setup.js dist/tests/*.test.js", + "test:e2e": "npm run build && node --test dist/tests/e2e/*.test.js", "typecheck": "tsc --noEmit", "build": "rm -rf dist && tsc && chmod +x dist/bin/zenrows.js", "prepack": "npm run build", diff --git a/registry/capabilities.json b/registry/capabilities.json index 231d875..36c8a1f 100644 --- a/registry/capabilities.json +++ b/registry/capabilities.json @@ -28,6 +28,15 @@ "requiresAuth": true, "notes": "Open beta. Cloud subcommands (create/status/results/cancel/wait/retry-failed) call the Batch API; `zenrows batch estimate` works locally with no API key." }, + "crawl": { + "key": "crawl", + "label": "Crawl", + "status": "available", + "command": "zenrows crawl", + "backend": "Crawl — https://api.zenrows.com/v1/crawls (X-API-Key header; same host as Fetch/Extract)", + "requiresAuth": true, + "notes": "Follows the links on each page from one start URL up to depth, on the start URL's domain, and returns the URLs it keeps, optionally with each page's HTML (start/status/wait/results/content/list/stop). An account without Crawl access gets 403 REQS008 (CRAWL_NOT_ENABLED). Too many crawls running on the account gives 429 too_many_crawls (CRAWL_QUOTA_EXCEEDED); retry after Retry-After." + }, "browser": { "key": "browser", "label": "Browser Sessions", diff --git a/registry/skills.json b/registry/skills.json index 458cea6..2658341 100644 --- a/registry/skills.json +++ b/registry/skills.json @@ -44,6 +44,17 @@ "path": "skills/batch-jobs", "tags": ["batch", "scale", "jsonl"] }, + { + "name": "crawl", + "type": "skill", + "description": "Collect the URLs behind one start page with Crawl: start, wait, read results and page HTML, stop.", + "status": "available", + "requires_backend_capabilities": ["crawl"], + "requires_auth": true, + "version": "0.1.0", + "path": "skills/crawl", + "tags": ["crawl", "links", "urls"] + }, { "name": "interact-browser", "type": "skill", diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md new file mode 100644 index 0000000..0d8136e --- /dev/null +++ b/skills/crawl/SKILL.md @@ -0,0 +1,73 @@ +--- +name: crawl +description: Collect the URLs behind one start page with Crawl — start a crawl, wait for it, read the kept URLs and optionally each page's HTML, stop it. Use when the user has a listing or section URL but not the item URLs. +version: 0.1.0 +requires_backend_capabilities: [crawl] +--- + +# Crawl + +Give Crawl one start URL; it follows the links on that page (and, with a +higher depth, on the pages behind it) and returns the URLs it keeps. Use it +when you know where the items live (a category page, a blog index, a docs +section) but not the item URLs themselves. Once you have the URLs, read pages +with `zenrows fetch` / `zenrows extract`, or ask Crawl for each page's HTML +with `--html`. + +Crawl stays on the start URL's domain. An account without Crawl access gets +403 `REQS008` → `CRAWL_NOT_ENABLED` ("Crawl is not enabled for this account"); +do not retry it. + +## Start, wait, read + +``` +zenrows crawl start https://example.com/products/ --depth 1 \ + --max-items 20 --max-pages 25 --include /product/ --follow --json +zenrows crawl results --json # every kept URL (crawl must have ended) +zenrows crawl results --out urls.jsonl # same, as JSONL +``` + +- `--depth N` is required: link hops from the start URL. `1` = the start + page's links; `2` = also the links on each of those pages. +- `--max-items` (kept URLs) and `--max-pages` (fetched pages) cap the crawl; + both default to 10. `--max-pages` bounds the cost: each page is one fetch. +- `--include` / `--exclude` are substring matches on the normalized URL and + repeat: `--include /product/ --exclude ?add-to-cart`. +- `start` returns at once. `--follow` (or `zenrows crawl wait `) polls + until the crawl ends; on `--timeout` the crawl keeps running (it is not + stopped) and you get `CRAWL_TIMEOUT`. + +## Page HTML + +``` +zenrows crawl start --depth 1 --html --follow +zenrows crawl results --json # fetched results carry content_url +zenrows crawl content # one page's HTML (or pass the content_url) +zenrows crawl results --download --out pages.jsonl # every URL + HTML, NDJSON +``` + +A result's `content_status` is `pending`, `fetched` or `failed`; only +`fetched` results have content. + +## While it runs, and after + +``` +zenrows crawl status # status, coverage, stop_reason / error +zenrows crawl results --cursor --json # one page; keep polling with next_cursor +zenrows crawl list # your crawls, newest first +zenrows crawl stop # stop it; kept URLs stay readable +``` + +A crawl ends `completed` (nothing left, or a cap hit: `stop_reason` +`max_items` / `max_pages`), `stopped` (you stopped it) or `failed` (`error.code` +and `error.detail` say why). Only `failed` exits non-zero (`CRAWL_FAILED`). + +## Limits + +When the account has too many crawls running, a start gets 429 +`too_many_crawls` → `CRAWL_QUOTA_EXCEEDED`; nothing was created. Retry after +`Retry-After` (about 30s), or stop one of your crawls first. + +Rules: start small (`--depth 1`, low caps) and check the results before a +deep crawl. Mind the cost ([[cost-control]]). Read errors with +[[trace-debug]]. diff --git a/skills/trace-debug/SKILL.md b/skills/trace-debug/SKILL.md index 11d0b01..e786019 100644 --- a/skills/trace-debug/SKILL.md +++ b/skills/trace-debug/SKILL.md @@ -45,6 +45,14 @@ zenrows trace export # JSON for sharing auto-created Free plan. - `BATCH_FAILED` → the batch run ended `failed`; `failure_reason` and `failure_detail` in `zenrows batch status --json` say why. +- `CRAWL_NOT_ENABLED` → the API answered 403 `REQS008`: Crawl is not + enabled for this account. Do not retry; ask Zenrows support for access. +- `CRAWL_QUOTA_EXCEEDED` → 429 `too_many_crawls`: the account has too many + crawls running. Nothing was created; retry after `Retry-After` (see + `zenrows crawl list`). +- `CRAWL_FAILED` → the crawl ended `failed`; `error.code` in + `zenrows crawl status --json` says why, with `error.detail`. See + [[crawl]]. - `PARAM_CONFLICT_AUTO_MANUAL` → drop the managed flags or add `--manual`. - `CAPABILITY_UNAVAILABLE` → the primitive is not available on this account (e.g. beta/invite-only); use the local-spec path where offered. diff --git a/skills/zenrows/SKILL.md b/skills/zenrows/SKILL.md index d328eb8..58f87df 100644 --- a/skills/zenrows/SKILL.md +++ b/skills/zenrows/SKILL.md @@ -34,6 +34,10 @@ If the user has a known URL and wants page content: If the user has a known URL and wants structured data: → Use Extract. (zenrows extract | --autoparse | --css) +If the user has a start page (listing, index, section) but not the item URLs: + → Use Crawl. (zenrows crawl start --depth 1 --follow, + then `zenrows crawl results `) + If the user has many URLs (bulk): → Use Batch. (validate one page with fetch/extract first, then `zenrows batch create `) @@ -70,6 +74,7 @@ Run `zenrows status` for the live capability matrix. As of this toolkit: | Protected Fetch | `zenrows fetch` | available (`GET /v1/`) | | Extract (extract=auto / Autoparse/CSS/Markdown) | `zenrows extract` | beta (same `/v1/`; extract=auto falls back to autoparse) | | Batch | `zenrows batch` | beta — runs cloud jobs (create/status/results); cheapest path at scale. Estimate specs locally with no key | +| Crawl | `zenrows crawl` | available — collect the URLs behind one start page, optionally with each page's HTML | | Browser | `zenrows browser` | available (Browser Sessions REST API / MCP) | | MCP | `zenrows mcp` | available (remote + local server) | diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts new file mode 100644 index 0000000..a21775b --- /dev/null +++ b/src/cli/commands/crawl.ts @@ -0,0 +1,473 @@ +/** + * `zenrows crawl` — Crawl API. + * + * Give Crawl one start URL and read back the URLs it finds behind it, optionally + * with each page's HTML. A crawl is an async job: `start` returns at once + * (or polls with `--follow`), then `status` / `wait` / `results` / `content` + * read it and `stop` ends it. An account without Crawl access gets 403 REQS008 + * → CRAWL_NOT_ENABLED. + */ +import { log } from "../../core/logger.ts"; +import { requireApiKey } from "../../core/auth.ts"; +import { ensureApiKey } from "../../core/ensure-key.ts"; +import { assertUsable } from "../../core/capabilities.ts"; +import { assertDomainAllowed, assertWithinLimits, loadPolicy } from "../../core/policy.ts"; +import { newRunId, writeRun } from "../../core/artifacts.ts"; +import { + contentIdOf, + createCrawl, + downloadCrawl, + getCrawl, + getCrawlContent, + listAllResults, + listCrawls, + stopCrawl, + waitForCrawl, + type Crawl, + type CrawlStop, + type CreateCrawlParams, +} from "../../core/crawl-api.ts"; +import { asNumber, asString, parse, type Command, type RunContext } from "../command.ts"; +import { ToolkitError } from "../../core/errors.ts"; +import { printError, writeOut } from "../output.ts"; +import { normalizeTimeout } from "./fetch.ts"; + +/** The API's default for `max_pages` (and `max_items`) when the flag is unset. */ +const DEFAULT_MAX_PAGES = 10; +/** Largest `depth` / `max_items` / `max_pages` the API takes. */ +const MAX_LIMIT = 100_000; + +export const crawl: Command = { + name: "crawl", + summary: "Crawl a site from one start URL and collect the URLs behind it.", + usage: + "zenrows crawl --depth N|status |results |content |list|stop |wait >", + help: [ + "Cloud (needs a key with Crawl access):", + " start --depth [flags] start a crawl from one URL (returns at once)", + " --depth link hops to follow from the start URL (1-100000, required)", + " --max-items stop after keeping n URLs (API default 10)", + " --max-pages stop after fetching n pages (API default 10; bounds cost)", + " --include keep only URLs containing this substring (repeatable)", + " --exclude drop URLs containing this substring (repeatable)", + " --html also fetch each kept URL's page HTML (read with `content`)", + " --follow poll until the crawl ends (alias: --wait)", + " --timeout with --follow: give up waiting after ms (default 600000)", + " --idempotency-key make a retried start create no second crawl", + " --no-signup do not auto-create a Free plan account if no key exists", + " status show status, coverage, stop reason / error", + " wait [--timeout ] poll until the crawl ends (it is not stopped on timeout)", + " results all kept URLs (crawl must have ended); paginated", + " --out write results as JSONL instead of printing", + " --cursor [--limit ] read one page only (works while running; prints next_cursor)", + " --download fetch the NDJSON export (with page HTML for --html crawls)", + " into --out, or .jsonl", + " content print one kept URL's page (content_id or content_url)", + " --out write it to a file instead", + " list [--limit ] [--cursor c] list your crawls, newest first", + " stop stop a running crawl (idempotent)", + " --json print structured output", + ].join("\n"), + async run(argv: string[], ctx: RunContext): Promise { + const [sub, ...rest] = argv; + try { + switch (sub) { + case "start": + return await startCmd(rest, ctx); + case "status": + return await statusCmd(rest, ctx); + case "wait": + return await waitCmd(rest, ctx); + case "results": + return await resultsCmd(rest, ctx); + case "content": + return await contentCmd(rest, ctx); + case "list": + return await listCmd(rest, ctx); + case "stop": + return await stopCmd(rest, ctx); + default: + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `Unknown crawl subcommand: ${sub ?? "(none)"}`, + likely_cause: "Subcommand not recognized.", + next_action: + "Use start --depth N | status | wait | results | content | list | stop .", + suggested_commands: ["zenrows crawl --help"], + }); + } + } catch (err) { + printError(err, ctx.json); + return 1; + } + }, +}; + +async function startCmd(rest: string[], ctx: RunContext): Promise { + const { values, positionals } = parse(rest, { + depth: { type: "string" }, + "max-items": { type: "string" }, + "max-pages": { type: "string" }, + include: { type: "string", multiple: true }, + exclude: { type: "string", multiple: true }, + html: { type: "boolean" }, + follow: { type: "boolean" }, + wait: { type: "boolean" }, // alias for --follow + timeout: { type: "string" }, + "idempotency-key": { type: "string" }, + "no-signup": { type: "boolean" }, + json: { type: "boolean" }, + }); + const json = ctx.json || values.json === true; + const follow = values.follow === true || values.wait === true; + const url = positionals[0]; + if (!url) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: "Provide the start URL.", + likely_cause: "No positional was given.", + next_action: "Usage: zenrows crawl start --depth 1", + suggested_commands: ["zenrows crawl start https://example.com/ --depth 1"], + }); + } + const depth = limitFlag("depth", values.depth); + if (depth === undefined) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: "--depth is required.", + likely_cause: "Crawl has no default depth yet; every crawl states how many link hops to follow.", + next_action: "Pass --depth 1 to collect the start page's links, --depth 2 to also open each of those.", + suggested_commands: [`zenrows crawl start ${url} --depth 1`], + }); + } + const params: CreateCrawlParams = { url, depth }; + const maxItems = limitFlag("max-items", values["max-items"]); + if (maxItems !== undefined) params.max_items = maxItems; + const maxPages = limitFlag("max-pages", values["max-pages"]); + if (maxPages !== undefined) params.max_pages = maxPages; + const include = patterns(values.include); + if (include.length) params.include_patterns = include; + const exclude = patterns(values.exclude); + if (exclude.length) params.exclude_patterns = exclude; + if (values.html === true) params.output_format = "html"; + const timeoutMs = normalizeTimeout(values.timeout); + const idempotencyKey = asString(values["idempotency-key"]); + + assertUsable("crawl"); + + // Pre-flight governance, all local: the start URL must pass the domain policy, + // and max_pages (each page is one fetch) must fit the per-run page cap. + const policy = loadPolicy(); + assertDomainAllowed(url, policy); + assertWithinLimits({ pages: maxPages ?? DEFAULT_MAX_PAGES }, policy, "crawl"); + + const apiKey = await ensureApiKey(values["no-signup"] ? { ...policy, auto_signup: false } : policy, { + onProvision: (a) => { + log.info("No API key found — created a Zenrows Free plan account for you."); + log.dim(`Claim it anytime (keeps your usage): ${a.claimUrl}`); + }, + }); + + const runId = newRunId(); + const startedAt = new Date().toISOString(); + log.step(`Starting crawl of ${url} (depth ${depth})…`); + try { + const created = await createCrawl(params, { apiKey, idempotencyKey }); + const finished: Crawl = follow ? await waitForCrawl(created.crawl_id, { apiKey, timeoutMs }) : created; + const failure = crawlFailure(finished); + const runDir = writeRun({ + runId, + command: "zenrows crawl start", + capability: "crawl", + url, + startedAt, + finishedAt: new Date().toISOString(), + status: failure ? "error" : "ok", + request: { ...params }, + result: { crawlId: created.crawl_id, status: finished.status, coverage: finished.coverage }, + ...(failure ? { error: failure.toJSON() } : {}), + }); + const code = printCrawl(finished, json, `Started crawl ${created.crawl_id}`); + if (!json) { + if (runDir) log.dim(` artifact: ${runDir}`); + if (!follow) log.dim(` next: zenrows crawl wait ${created.crawl_id}`); + } + return code; + } catch (err) { + writeRun({ + runId, + command: "zenrows crawl start", + capability: "crawl", + url, + startedAt, + finishedAt: new Date().toISOString(), + status: "error", + request: { ...params }, + error: err instanceof ToolkitError ? err.toJSON() : { message: String(err) }, + }); + throw err; + } +} + +async function statusCmd(rest: string[], ctx: RunContext): Promise { + const { values, positionals } = parse(rest, { json: { type: "boolean" } }); + const id = requireId(positionals[0], "status"); + assertUsable("crawl"); + const apiKey = requireApiKey(); + const c = await getCrawl(id, { apiKey, limit: 1 }); + return printCrawl(c, ctx.json || values.json === true, `Crawl ${id}`); +} + +async function waitCmd(rest: string[], ctx: RunContext): Promise { + const { values, positionals } = parse(rest, { timeout: { type: "string" }, json: { type: "boolean" } }); + const id = requireId(positionals[0], "wait"); + const timeoutMs = normalizeTimeout(values.timeout); + assertUsable("crawl"); + const apiKey = requireApiKey(); + const c = await waitForCrawl(id, { apiKey, timeoutMs }); + return printCrawl(c, ctx.json || values.json === true, `Crawl ${id} finished`); +} + +async function resultsCmd(rest: string[], ctx: RunContext): Promise { + const { values, positionals } = parse(rest, { + out: { type: "string" }, + cursor: { type: "string" }, + limit: { type: "string" }, + download: { type: "boolean" }, + json: { type: "boolean" }, + }); + const json = ctx.json || values.json === true; + const id = requireId(positionals[0], "results"); + const outFile = asString(values.out); + const cursor = asString(values.cursor); + const limit = pageLimit(values.limit, 10_000); + assertUsable("crawl"); + const apiKey = requireApiKey(); + + if (values.download === true) { + const file = outFile ?? `${id.replace(/[^A-Za-z0-9._-]+/g, "_")}.jsonl`; + const { crawlStatus, ndjson } = await downloadCrawl(id, { apiKey }); + writeOut(file, ndjson); + const lines = ndjson.split("\n").filter((l) => l.trim()).length; + const partial = crawlStatus === "running"; + if (json) { + log.out(JSON.stringify({ ok: true, crawlId: id, crawlStatus, partial, count: lines, file }, null, 2)); + } else { + log.success(`Downloaded ${lines} result(s) → ${file}`); + if (partial) log.warn(" The crawl is still running: this file is partial. Download again once it ends."); + } + return 0; + } + + // One page, as the API returns it — the way to follow a crawl while it runs. + if (cursor !== undefined || limit !== undefined) { + const page = await getCrawl(id, { apiKey, cursor, limit }); + const body = { ok: true, crawlId: id, status: page.status, count: page.results.length, results: page.results, next_cursor: page.next_cursor }; + if (outFile) { + writeOut(outFile, toJsonl(page.results)); + log.success(`Wrote ${page.results.length} result(s) → ${outFile}`); + log.info(`next_cursor: ${page.next_cursor ?? "null (last page)"}`); + } else if (json) { + log.out(JSON.stringify(body, null, 2)); + } else { + log.info(`${page.results.length} result(s) for crawl ${id} (status: ${page.status}):`); + log.out(toJsonl(page.results).trimEnd()); + log.info(`next_cursor: ${page.next_cursor ?? "null (last page)"}`); + } + return 0; + } + + const { crawl: c, results } = await listAllResults(id, { apiKey }); + if (outFile) { + writeOut(outFile, toJsonl(results)); + log.success(`Wrote ${results.length} result(s) → ${outFile}`); + } else if (json) { + log.out(JSON.stringify({ ok: true, crawlId: id, status: c.status, count: results.length, results }, null, 2)); + } else { + log.info(`${results.length} result(s) for crawl ${id} (status: ${c.status}):`); + log.out(toJsonl(results).trimEnd()); + } + return 0; +} + +async function contentCmd(rest: string[], ctx: RunContext): Promise { + const { values, positionals } = parse(rest, { out: { type: "string" }, json: { type: "boolean" } }); + const json = ctx.json || values.json === true; + const id = requireId(positionals[0], "content"); + const ref = positionals[1]; + // Accept the content id or the result's whole content_url. + const contentId = ref ? (contentIdOf({ url: "", content_url: ref }) ?? ref) : undefined; + if (!contentId) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: "Missing content id.", + likely_cause: "No positional was provided.", + next_action: + "Pass the id from a fetched result's content_url (the last path segment), e.g. `zenrows crawl content ct_…`. Only crawls started with --html have content.", + suggested_commands: [`zenrows crawl results ${id}`], + }); + } + assertUsable("crawl"); + const apiKey = requireApiKey(); + const { contentType, body } = await getCrawlContent(id, contentId, { apiKey }); + const outFile = asString(values.out); + if (outFile) { + writeOut(outFile, body); + if (json) { + log.out(JSON.stringify({ ok: true, crawlId: id, contentId, contentType, bytes: Buffer.byteLength(body), file: outFile }, null, 2)); + } else { + log.success(`Wrote ${Buffer.byteLength(body)} bytes (${contentType || "unknown type"}) → ${outFile}`); + } + } else if (json) { + log.out(JSON.stringify({ ok: true, crawlId: id, contentId, contentType, bytes: Buffer.byteLength(body), content: body }, null, 2)); + } else { + log.out(body); + } + return 0; +} + +async function listCmd(rest: string[], ctx: RunContext): Promise { + const { values } = parse(rest, { limit: { type: "string" }, cursor: { type: "string" }, json: { type: "boolean" } }); + const json = ctx.json || values.json === true; + const limit = pageLimit(values.limit, 100); + const cursor = asString(values.cursor); + assertUsable("crawl"); + const apiKey = requireApiKey(); + const page = await listCrawls({ apiKey, limit, cursor }); + const crawls = page.crawls ?? []; + if (json) { + log.out(JSON.stringify({ ok: true, count: crawls.length, crawls, next_cursor: page.next_cursor ?? null }, null, 2)); + return 0; + } + if (!crawls.length) log.info("No crawls."); + for (const c of crawls) { + const cov = c.coverage; + log.out( + `${c.crawl_id} ${c.status.padEnd(9)} ${c.created_at} ${cov ? `${cov.items_found} items/${cov.pages_fetched} pages` : ""} ${c.url}`, + ); + } + if (page.next_cursor) log.dim(` more: zenrows crawl list --cursor ${page.next_cursor}`); + return 0; +} + +async function stopCmd(rest: string[], ctx: RunContext): Promise { + const { values, positionals } = parse(rest, { json: { type: "boolean" } }); + const id = requireId(positionals[0], "stop"); + assertUsable("crawl"); + const apiKey = requireApiKey(); + const s = await stopCrawl(id, { apiKey }); + return printCrawl(s, ctx.json || values.json === true, `Stopped crawl ${id}`); +} + +/** + * The error for a crawl that ended `failed`, or null for any other state. The + * crawl's own `error.code` picks the next action. + */ +export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null { + if (c.status !== "failed") return null; + const code = c.error?.code; + const why = [code, c.error?.detail].filter(Boolean).join(": "); + const next: Record = { + insufficient_credits: "The account ran out of credits. Check `zenrows usage`, add credits, then start a new crawl.", + seed_unreachable: "The start URL could not be fetched (the site may be blocking it). Check it with `zenrows fetch `.", + no_items_found: + "Nothing matched. Loosen --include / --exclude, raise --depth, or check the start page with `zenrows fetch `.", + }; + return new ToolkitError({ + code: "CRAWL_FAILED", + message: `Crawl ${c.crawl_id} failed.`, + likely_cause: why || "The crawl ended in status failed without a reason.", + next_action: (code && next[code]) || "Read the error, fix the cause, then start a new crawl.", + suggested_commands: [`zenrows crawl status ${c.crawl_id}`], + }); +} + +/** + * Print a crawl (or a stop answer), structured under --json, and return the exit + * code: 1 when it ended `failed`, else 0. `stopped` is a deliberate outcome, so + * it exits 0 but prints as a warning. + */ +function printCrawl(c: Crawl | CrawlStop, json: boolean, headline: string): number { + const err = crawlFailure(c); + const crawlFields = { ...c } as Record; + // A status read asks for one result to keep polls cheap; that page is not "the results". + delete crawlFields.results; + delete crawlFields.next_cursor; + if (json) { + log.out( + JSON.stringify( + { + ok: !err, + crawlId: c.crawl_id, + status: c.status, + ...(c.stop_reason ? { stop_reason: c.stop_reason } : {}), + crawl: crawlFields, + ...(err ? { error: err.toJSON() } : {}), + }, + null, + 2, + ), + ); + return err ? 1 : 0; + } + const line = `${headline} · status: ${c.status ?? "unknown"}${c.stop_reason ? ` (${c.stop_reason})` : ""}`; + if (err) log.error(line); + else if (c.status === "stopped") log.warn(line); + else log.success(line); + const cov = (c as Crawl).coverage; + if (cov) { + log.info(` ${cov.items_found} item(s) found · ${cov.pages_fetched} page(s) fetched · ${cov.pages_failed} failed`); + } + if (err) printError(err, false); + return err ? 1 : 0; +} + +/** A positive integer flag up to the API's 100,000 ceiling; undefined when unset. */ +function limitFlag(name: string, v: unknown): number | undefined { + if (v === undefined) return undefined; + const n = asNumber(v); + if (n === undefined || !Number.isInteger(n) || n < 1 || n > MAX_LIMIT) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `Invalid --${name} value '${String(v)}'.`, + likely_cause: `--${name} takes a whole number from 1 to ${MAX_LIMIT}.`, + next_action: `Pass e.g. --${name} ${name === "depth" ? 1 : 10}.`, + }); + } + return n; +} + +/** `--limit` for a paged read: a whole number from 1 to `max`. */ +function pageLimit(v: unknown, max: number): number | undefined { + if (v === undefined) return undefined; + const n = asNumber(v); + if (n === undefined || !Number.isInteger(n) || n < 1 || n > max) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `Invalid --limit value '${String(v)}'.`, + likely_cause: `--limit takes a whole number from 1 to ${max}.`, + next_action: `Pass e.g. --limit ${Math.min(max, 100)}.`, + }); + } + return n; +} + +function patterns(v: unknown): string[] { + const list = Array.isArray(v) ? v : typeof v === "string" ? [v] : []; + return list.filter((p): p is string => typeof p === "string" && p.length > 0); +} + +function toJsonl(rows: unknown[]): string { + return rows.map((r) => JSON.stringify(r)).join("\n") + (rows.length ? "\n" : ""); +} + +function requireId(id: string | undefined, sub: string): string { + if (id) return id; + throw new ToolkitError({ + code: "INVALID_USAGE", + message: "Missing crawl id.", + likely_cause: "No positional was provided.", + next_action: `Usage: zenrows crawl ${sub} . List your crawls with \`zenrows crawl list\`.`, + suggested_commands: ["zenrows crawl list"], + }); +} diff --git a/src/cli/index.ts b/src/cli/index.ts index e8e2bdc..2185bcb 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -27,6 +27,7 @@ import { policy } from "./commands/policy.ts"; import { fetch_ } from "./commands/fetch.ts"; import { extract } from "./commands/extract.ts"; import { batch } from "./commands/batch.ts"; +import { crawl } from "./commands/crawl.ts"; import { browser } from "./commands/browser.ts"; import { mcp } from "./commands/mcp.ts"; import { plugin } from "./commands/plugin.ts"; @@ -43,7 +44,7 @@ const evalCmd = makeAssetCommand("eval", "Reproducible benchmark/test suites."); /** Ordered groups for help output. */ const GROUPS: Array<{ title: string; commands: Command[] }> = [ { title: "Setup", commands: [init, signup, account, login, logout, status, usage, update, uninstall, config, policy] }, - { title: "Primitives", commands: [fetch_, extract, batch, browser] }, + { title: "Primitives", commands: [fetch_, extract, batch, crawl, browser] }, { title: "Distribution", commands: [mcp, plugin, skill, template, workflow, recipe, evalCmd, trace] }, ]; diff --git a/src/core/crawl-api.ts b/src/core/crawl-api.ts new file mode 100644 index 0000000..f434f51 --- /dev/null +++ b/src/core/crawl-api.ts @@ -0,0 +1,459 @@ +/** + * Client for the Zenrows Crawl API. + * + * Crawl lives on the same host as Fetch and Extract (`https://api.zenrows.com/v1`) + * under `/crawls`. Auth is the `X-API-Key` header, never the `apikey` query + * param: Crawl answers any query parameter it does not know with 400. Bodies are + * JSON; errors come back as `application/problem+json` (RFC 9457) and we branch + * on `status` + `code` only. A crawl is an async job: create it, then poll it + * until its status leaves `running`. `fetchImpl` / `sleepImpl` are injectable + * for tests. + * + * `output_format` takes `html` only (absent = URLs only). Response fields we do + * not model pass through untouched. + */ +import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "./errors.ts"; +import { readAccount } from "./agent-account.ts"; +import { registerSecret } from "./logger.ts"; +import { CLI_USER_AGENT, loadConfig } from "./config.ts"; +import { agentClientHeader } from "./agent-client.ts"; + +/** Production Crawl API base (no trailing slash): the Fetch/Extract host. */ +export const DEFAULT_CRAWL_API_BASE = "https://api.zenrows.com/v1"; +/** Env var to override the Crawl API base (local/staging testing). */ +export const CRAWL_API_BASE_ENV = "ZENROWS_CRAWL_API_BASE"; + +/** + * Resolve the Crawl API base, trimmed of trailing `/`. `ZENROWS_CRAWL_API_BASE` + * wins; otherwise Crawl follows the Fetch/Extract `apiBase` (config or + * `ZENROWS_API_BASE`), since both live on the same API host. + */ +export function crawlBase(): string { + const env = process.env[CRAWL_API_BASE_ENV]; + const base = env && env.trim() ? env.trim() : loadConfig().apiBase || DEFAULT_CRAWL_API_BASE; + return base.replace(/\/+$/, ""); +} + +export interface CrawlCoverage { + pages_fetched: number; + pages_failed: number; + items_found: number; +} + +/** Why a crawl ended `failed` (part of the crawl, not an error response). */ +export interface CrawlRunError { + code: string; + detail: string; +} + +/** One crawl. Enum fields are extensible: expect values not listed here. */ +export interface Crawl { + crawl_id: string; + /** running | completed | stopped | failed. Terminal = anything but `running`. */ + status: string; + url: string; + depth: number; + max_items: number; + max_pages: number; + coverage: CrawlCoverage; + created_at: string; + /** max_items | max_pages | user. Absent when nothing was left to open. */ + stop_reason?: string; + error?: CrawlRunError; + include_patterns?: string[]; + exclude_patterns?: string[]; + output_format?: string; + duplicates_removed?: number; + finished_at?: string; + [k: string]: unknown; +} + +/** One URL the crawl kept. */ +export interface CrawlResult { + url: string; + /** pending | fetched | failed — only when the crawl has an output format. */ + content_status?: string; + /** e.g. `/v1/crawls/c_x/contents/ct_y`, present once fetched. */ + content_url?: string; + [k: string]: unknown; +} + +/** `GET /crawls/{id}`: the crawl plus one page of results. */ +export interface CrawlWithResults extends Crawl { + results: CrawlResult[]; + /** Never null while the crawl runs; null once it ended and this page is the last. */ + next_cursor: string | null; +} + +export interface CrawlList { + crawls: Crawl[]; + /** Absent on the last page. */ + next_cursor?: string; +} + +/** `POST /crawls/{id}/stop`: where the crawl stands, without counts. */ +export interface CrawlStop { + crawl_id: string; + status: string; + stop_reason?: string; + error?: CrawlRunError; + finished_at?: string; + [k: string]: unknown; +} + +/** Create body. Only the fields the caller set are sent. */ +export interface CreateCrawlParams { + url: string; + depth: number; + max_items?: number; + max_pages?: number; + include_patterns?: string[]; + exclude_patterns?: string[]; + /** `html` returns each kept URL's page; absent = URLs only. */ + output_format?: "html"; +} + +/** RFC 9457 problem body. */ +interface ProblemJson { + type?: string; + title?: string; + status?: number; + detail?: string; + code?: string; +} + +/** The API's "Crawl is not enabled for this account" code (403). */ +export const CRAWL_NOT_ENABLED_CODE = "REQS008"; + +/** A crawl stops progressing once its status is anything but `running`. */ +export function isTerminal(status: string | undefined): boolean { + return !!status && status !== "running"; +} + +interface RequestOpts { + apiKey: string; + body?: unknown; + query?: Record; + headers?: Record; + accept?: string; + fetchImpl?: typeof fetch; + timeoutMs?: number; +} + +interface RawResponse { + status: number; + headers: Headers; + text: string; +} + +/** + * Perform a Crawl API request and return the raw body. Sets `X-API-Key`, adds a + * JSON content-type when a body is present, and on a non-2xx parses the + * problem+json body into a normalized ToolkitError. + */ +async function crawlFetch(method: string, path: string, opts: RequestOpts): Promise { + registerSecret(opts.apiKey); + const url = new URL(crawlBase() + path); + for (const [k, v] of Object.entries(opts.query ?? {})) { + if (v !== undefined && v !== null) url.searchParams.set(k, String(v)); + } + const doFetch = opts.fetchImpl ?? fetch; + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), opts.timeoutMs ?? 60_000); + + const headers: Record = { + "X-API-Key": opts.apiKey, + Accept: opts.accept ?? "application/json", + "User-Agent": CLI_USER_AGENT, + ...agentClientHeader(), + ...(opts.headers ?? {}), + }; + if (opts.body !== undefined) headers["Content-Type"] = "application/json"; + + let res: Response; + let text: string; + try { + res = await doFetch(url.toString(), { + method, + headers, + body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined, + signal: controller.signal, + }); + text = await res.text(); + } catch (err) { + throw new ToolkitError({ + code: "BACKEND_UNAVAILABLE", + message: "Could not reach the Zenrows Crawl API.", + likely_cause: err instanceof Error ? err.message : String(err), + next_action: + "Check connectivity and retry. Override the host with ZENROWS_CRAWL_API_BASE if you are testing against staging.", + suggested_commands: ["zenrows status"], + }); + } finally { + clearTimeout(timeout); + } + + if (res.status < 200 || res.status >= 300) { + throw problemToError(res.status, text, method, path, res.headers.get("retry-after")); + } + return { status: res.status, headers: res.headers, text }; +} + +/** `crawlFetch` + JSON parse. */ +async function crawlJson(method: string, path: string, opts: RequestOpts): Promise { + const { text } = await crawlFetch(method, path, opts); + try { + return JSON.parse(text) as T; + } catch { + throw new ToolkitError({ + code: "CRAWL_FAILED", + message: "The Crawl API response was not valid JSON.", + likely_cause: text.slice(0, 240) || "Empty response body.", + next_action: "Retry, or check the crawl with `zenrows crawl status `.", + }); + } +} + +/** Map a problem+json body + HTTP status to a normalized ToolkitError. */ +function problemToError( + status: number, + body: string, + method: string, + path: string, + retryAfter: string | null, +): ToolkitError { + let problem: ProblemJson = {}; + try { + problem = JSON.parse(body) as ProblemJson; + } catch { + // non-JSON error body — fall through with an empty problem + } + const serverCode = problem.code ?? ""; + const detail = problem.detail || problem.title || body.slice(0, 240) || `HTTP ${status}`; + const cause = `HTTP ${status}${serverCode ? ` (${serverCode})` : ""} for ${method} ${path}: ${detail}`; + + if (status === 403 && serverCode === CRAWL_NOT_ENABLED_CODE) { + return new ToolkitError({ + code: "CRAWL_NOT_ENABLED", + message: "Crawl is not enabled for this account.", + likely_cause: `${cause}. Crawl is not enabled for this account yet.`, + next_action: + "Ask Zenrows support to enable Crawl for this account. Meanwhile fetch known URLs with `zenrows fetch`.", + suggested_commands: ["zenrows fetch "], + }); + } + if (status === 401) { + return new ToolkitError({ + code: "AUTH_INVALID", + message: "Zenrows rejected the API key for the Crawl API.", + likely_cause: cause, + next_action: "Re-check your key and log in again.", + suggested_commands: ["zenrows login --api-key "], + }); + } + if (status === 404) { + const content = serverCode === "content_not_found"; + return new ToolkitError({ + code: "CRAWL_NOT_FOUND", + message: content ? "Crawl content not found." : "Crawl not found.", + likely_cause: content + ? `${cause}. The page was not fetched (yet), its fetch failed, or the crawl ran without --html.` + : `${cause}. The id may be wrong or the crawl is not owned by this account.`, + next_action: content + ? "Read the result's content_status with `zenrows crawl results `; only `fetched` results have content." + : "Check the crawl id, or list your crawls with `zenrows crawl list`.", + suggested_commands: ["zenrows crawl list"], + }); + } + if (status === 429) { + const wait = retryAfter && /^\d+$/.test(retryAfter.trim()) ? ` Retry after ${retryAfter.trim()}s.` : ""; + return new ToolkitError({ + code: "CRAWL_QUOTA_EXCEEDED", + message: "Too many crawls running.", + likely_cause: `${cause}. The account has too many crawls running.${wait}`, + next_action: + "Retry after Retry-After, or stop one of your crawls with `zenrows crawl stop ` first. Nothing was created.", + suggested_commands: ["zenrows crawl list"], + }); + } + if (status === 402 && isKeyCapReached(serverCode)) { + return keyCapReached(`${method} ${path}`, { status: 402, detail: problem.detail || problem.title || undefined }); + } + if (status === 402) { + const acct = readAccount(); + return quotaExhausted(`${method} ${path}`, acct?.unclaimed ? acct.claimUrl : undefined, { + status: 402, + detail: problem.detail || problem.title || undefined, + }); + } + if (status === 400 || status === 409 || status === 422) { + return new ToolkitError({ + code: "CRAWL_INVALID", + message: `Crawl rejected the request (HTTP ${status}).`, + likely_cause: `${cause}.`, + next_action: + status === 409 + ? "A request with the same Idempotency-Key is still in flight; retry once it has finished." + : "Fix the reported parameter and retry. See `zenrows crawl --help` for the accepted flags and ranges.", + }); + } + return new ToolkitError({ + code: "CRAWL_FAILED", + message: `Crawl request failed (HTTP ${status}).`, + likely_cause: `${cause}.`, + next_action: + status >= 500 ? "Retry with a short backoff." : "Fix the reported problem and retry.", + }); +} + +interface CallOpts { + apiKey: string; + fetchImpl?: typeof fetch; + timeoutMs?: number; +} + +/** Start a crawl (202). Sends only the fields set in `params`. */ +export function createCrawl(params: CreateCrawlParams, opts: CallOpts & { idempotencyKey?: string }): Promise { + const body: Record = { url: params.url, depth: params.depth }; + if (params.max_items !== undefined) body.max_items = params.max_items; + if (params.max_pages !== undefined) body.max_pages = params.max_pages; + if (params.include_patterns?.length) body.include_patterns = params.include_patterns; + if (params.exclude_patterns?.length) body.exclude_patterns = params.exclude_patterns; + if (params.output_format !== undefined) body.output_format = params.output_format; + return crawlJson("POST", "/crawls", { + apiKey: opts.apiKey, + body, + headers: opts.idempotencyKey ? { "Idempotency-Key": opts.idempotencyKey } : undefined, + fetchImpl: opts.fetchImpl, + timeoutMs: opts.timeoutMs, + }); +} + +/** Read a crawl and one page of its results. */ +export function getCrawl(id: string, opts: CallOpts & { cursor?: string; limit?: number }): Promise { + return crawlJson("GET", `/crawls/${encodeURIComponent(id)}`, { + apiKey: opts.apiKey, + query: { cursor: opts.cursor, limit: opts.limit }, + fetchImpl: opts.fetchImpl, + timeoutMs: opts.timeoutMs, + }); +} + +/** One page of the account's crawls, newest first. */ +export function listCrawls(opts: CallOpts & { cursor?: string; limit?: number }): Promise { + return crawlJson("GET", "/crawls", { + apiKey: opts.apiKey, + query: { cursor: opts.cursor, limit: opts.limit }, + fetchImpl: opts.fetchImpl, + timeoutMs: opts.timeoutMs, + }); +} + +/** Stop a running crawl. Idempotent: an ended crawl answers as it ended. */ +export function stopCrawl(id: string, opts: CallOpts): Promise { + return crawlJson("POST", `/crawls/${encodeURIComponent(id)}/stop`, { + apiKey: opts.apiKey, + fetchImpl: opts.fetchImpl, + timeoutMs: opts.timeoutMs, + }); +} + +/** The page of one kept URL (HTML for `output_format: html`). */ +export async function getCrawlContent( + id: string, + contentId: string, + opts: CallOpts, +): Promise<{ contentType: string; body: string }> { + const res = await crawlFetch("GET", `/crawls/${encodeURIComponent(id)}/contents/${encodeURIComponent(contentId)}`, { + apiKey: opts.apiKey, + accept: "text/html, application/json;q=0.9, application/problem+json;q=0.8", + fetchImpl: opts.fetchImpl, + timeoutMs: opts.timeoutMs, + }); + return { contentType: res.headers.get("content-type") ?? "", body: res.text }; +} + +/** + * Every result in one NDJSON file (`{url, content_status?, content?}` per line). + * `crawlStatus` is the `X-Crawl-Status` header: `running` means the file is partial. + */ +export async function downloadCrawl(id: string, opts: CallOpts): Promise<{ crawlStatus?: string; ndjson: string }> { + const res = await crawlFetch("GET", `/crawls/${encodeURIComponent(id)}/download`, { + apiKey: opts.apiKey, + accept: "application/x-ndjson, application/problem+json;q=0.8", + fetchImpl: opts.fetchImpl, + timeoutMs: opts.timeoutMs ?? 300_000, + }); + return { crawlStatus: res.headers.get("x-crawl-status") ?? undefined, ndjson: res.text }; +} + +/** + * The content id of a fetched result: the last segment of its `content_url`. + * Undefined when the result has no content yet. + */ +export function contentIdOf(result: CrawlResult): string | undefined { + if (!result.content_url) return undefined; + const last = result.content_url.split("?")[0]!.replace(/\/+$/, "").split("/").pop(); + return last ? decodeURIComponent(last) : undefined; +} + +/** + * Read every result, following `next_cursor` until it is null. Only call this + * on an ended crawl: while a crawl runs `next_cursor` is never null, so this + * refuses a `running` crawl instead of looping forever (wait first). + */ +export async function listAllResults( + id: string, + opts: CallOpts & { pageSize?: number }, +): Promise<{ crawl: CrawlWithResults; results: CrawlResult[] }> { + const results: CrawlResult[] = []; + let cursor: string | undefined; + let first: CrawlWithResults | undefined; + for (;;) { + const page = await getCrawl(id, { ...opts, cursor, limit: opts.pageSize }); + if (!isTerminal(page.status)) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `Crawl ${id} is still running.`, + likely_cause: "Reading all results needs an ended crawl: while it runs, more results keep arriving.", + next_action: "Wait for it first (`zenrows crawl wait `), or stop it with `zenrows crawl stop `.", + suggested_commands: [`zenrows crawl wait ${id}`, `zenrows crawl status ${id}`], + }); + } + first ??= page; + results.push(...(page.results ?? [])); + if (page.next_cursor === null || page.next_cursor === undefined || page.next_cursor === cursor) break; + cursor = page.next_cursor; + } + return { crawl: first!, results }; +} + +/** + * Poll `GET /crawls/{id}?limit=1` until the status leaves `running`, backing off + * 2s → ×1.5 → capped at 15s. On timeout throws CRAWL_TIMEOUT + * and leaves the crawl running. `sleepImpl` is injectable for tests. + */ +export async function waitForCrawl( + id: string, + opts: CallOpts & { sleepImpl?: (ms: number) => Promise }, +): Promise { + const sleep = opts.sleepImpl ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); + const timeoutMs = opts.timeoutMs ?? 600_000; + const deadline = Date.now() + timeoutMs; + let delay = 2000; + for (;;) { + const crawl = await getCrawl(id, { apiKey: opts.apiKey, fetchImpl: opts.fetchImpl, limit: 1 }); + if (isTerminal(crawl.status)) return crawl; + if (Date.now() + delay > deadline) { + throw new ToolkitError({ + code: "CRAWL_TIMEOUT", + message: `Timed out waiting for crawl ${id} to finish.`, + likely_cause: `The crawl was still running after ${Math.round(timeoutMs / 1000)}s. It keeps running; it was not stopped.`, + next_action: "Re-check progress with `zenrows crawl status `, wait again with a larger --timeout, or stop it.", + suggested_commands: [`zenrows crawl status ${id}`, `zenrows crawl wait ${id}`, `zenrows crawl stop ${id}`], + }); + } + await sleep(delay); + delay = Math.min(delay * 1.5, 15_000); + } +} diff --git a/src/core/errors.ts b/src/core/errors.ts index b36f109..95d3a22 100644 --- a/src/core/errors.ts +++ b/src/core/errors.ts @@ -35,6 +35,12 @@ export type ErrorCode = | "BATCH_QUOTA_EXCEEDED" | "BATCH_NOT_FOUND" | "BATCH_FAILED" + | "CRAWL_NOT_ENABLED" + | "CRAWL_QUOTA_EXCEEDED" + | "CRAWL_NOT_FOUND" + | "CRAWL_INVALID" + | "CRAWL_FAILED" + | "CRAWL_TIMEOUT" | "PLUGIN_UNSUPPORTED" | "MCP_CLIENT_UNSUPPORTED" | "ASSET_NOT_FOUND" diff --git a/tests/capabilities.test.ts b/tests/capabilities.test.ts index 1aa8e9d..73c7da6 100644 --- a/tests/capabilities.test.ts +++ b/tests/capabilities.test.ts @@ -11,6 +11,7 @@ test("capability matrix loads with the honest classifications", () => { assert.equal(caps.mcp?.status, "available"); // Batch is a real product in open beta. assert.equal(caps.batch?.status, "beta"); + assert.equal(caps.crawl?.status, "available"); }); test("isUsable true for available + open beta, false for absent capabilities", () => { @@ -25,6 +26,7 @@ test("assertUsable passes for available + experimental + beta", () => { assert.doesNotThrow(() => assertUsable("browser")); // available (billing/escalation handled by docs, opt-out via policy) assert.doesNotThrow(() => assertUsable("extract")); assert.doesNotThrow(() => assertUsable("batch")); + assert.doesNotThrow(() => assertUsable("crawl")); assert.throws( () => assertUsable("nope"), (e: unknown) => e instanceof ToolkitError && e.code === "CAPABILITY_UNAVAILABLE", diff --git a/tests/crawl-api.test.ts b/tests/crawl-api.test.ts new file mode 100644 index 0000000..2dc80b3 --- /dev/null +++ b/tests/crawl-api.test.ts @@ -0,0 +1,286 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + CRAWL_API_BASE_ENV, + DEFAULT_CRAWL_API_BASE, + contentIdOf, + crawlBase, + createCrawl, + downloadCrawl, + getCrawl, + getCrawlContent, + listAllResults, + listCrawls, + stopCrawl, + waitForCrawl, + type CrawlWithResults, +} from "../src/core/crawl-api.ts"; +import { ToolkitError } from "../src/core/errors.ts"; + +const crawlBody = { + crawl_id: "c_1", + status: "running", + url: "https://example.com/products/", + depth: 1, + max_items: 10, + max_pages: 10, + coverage: { pages_fetched: 0, pages_failed: 0, items_found: 0 }, + created_at: "2026-10-15T09:00:00Z", +}; + +/** Build a fetch stub that returns a fixed body, recording each call. */ +function stubFetch(status: number, payload: unknown, headers: Record = {}) { + const calls: Array<{ url: string; init?: RequestInit }> = []; + const impl = (async (url: string, init?: RequestInit) => { + calls.push({ url, init }); + return new Response(typeof payload === "string" ? payload : JSON.stringify(payload), { + status, + headers: { "content-type": "application/json", ...headers }, + }); + }) as unknown as typeof fetch; + return { impl, calls }; +} + +function problem(status: number, code: string, detail = "nope", headers: Record = {}) { + return stubFetch(status, { code, title: code, detail, status, instance: "urn:zenrows:request:x" }, { + "content-type": "application/problem+json", + ...headers, + }); +} + +test("crawlBase defaults to the production API and honors ZENROWS_CRAWL_API_BASE", () => { + assert.equal(crawlBase(), DEFAULT_CRAWL_API_BASE); + process.env[CRAWL_API_BASE_ENV] = "https://example.com/v1/"; + try { + assert.equal(crawlBase(), "https://example.com/v1"); + } finally { + delete process.env[CRAWL_API_BASE_ENV]; + } +}); + +test("createCrawl posts /crawls with X-API-Key and sends only the fields the caller set", async () => { + const { impl, calls } = stubFetch(202, crawlBody, { location: "/v1/crawls/c_1" }); + const c = await createCrawl( + { url: "https://example.com/products/", depth: 1, max_items: 3, include_patterns: ["/product/"], exclude_patterns: [] }, + { apiKey: "zr-key", fetchImpl: impl }, + ); + assert.equal(c.crawl_id, "c_1"); + const call = calls[0]!; + assert.equal(call.url, `${DEFAULT_CRAWL_API_BASE}/crawls`); + assert.equal(call.init?.method, "POST"); + const headers = call.init?.headers as Record; + assert.equal(headers["X-API-Key"], "zr-key"); + assert.equal(headers["Content-Type"], "application/json"); + assert.equal(headers["Idempotency-Key"], undefined); + assert.ok(!call.url.includes("apikey"), "the key never rides the query string"); + const body = JSON.parse(String(call.init?.body)); + assert.deepEqual(body, { url: "https://example.com/products/", depth: 1, max_items: 3, include_patterns: ["/product/"] }); + assert.ok(!("discovery" in body)); + assert.ok(!("output_format" in body)); +}); + +test("createCrawl sends output_format html and Idempotency-Key when asked", async () => { + const { impl, calls } = stubFetch(202, crawlBody); + await createCrawl({ url: "https://example.com/products/", depth: 2, max_pages: 5, output_format: "html" }, { + apiKey: "k", + fetchImpl: impl, + idempotencyKey: "idem-1", + }); + const body = JSON.parse(String(calls[0]!.init?.body)); + assert.deepEqual(body, { url: "https://example.com/products/", depth: 2, max_pages: 5, output_format: "html" }); + assert.equal((calls[0]!.init?.headers as Record)["Idempotency-Key"], "idem-1"); +}); + +test("getCrawl passes cursor + limit and parses results, tolerating unknown fields", async () => { + const { impl, calls } = stubFetch(200, { + ...crawlBody, + status: "a_future_status", + brand_new_field: { x: 1 }, + results: [{ url: "https://example.com/product/a", content_status: "fetched", content_url: "/v1/crawls/c_1/contents/ct_9" }], + next_cursor: "cur_2", + }); + const page = await getCrawl("c_1", { apiKey: "k", cursor: "cur_1", limit: 50, fetchImpl: impl }); + const u = new URL(calls[0]!.url); + assert.equal(u.pathname, "/v1/crawls/c_1"); + assert.equal(u.searchParams.get("cursor"), "cur_1"); + assert.equal(u.searchParams.get("limit"), "50"); + assert.equal(calls[0]!.init?.method, "GET"); + assert.equal(page.results[0]!.content_status, "fetched"); + assert.equal(contentIdOf(page.results[0]!), "ct_9"); + assert.equal(page.next_cursor, "cur_2"); + assert.equal(page.brand_new_field && typeof page.brand_new_field, "object"); +}); + +test("listCrawls reads one page; next_cursor is absent on the last page", async () => { + const { impl, calls } = stubFetch(200, { crawls: [{ ...crawlBody, status: "completed" }] }); + const page = await listCrawls({ apiKey: "k", limit: 5, fetchImpl: impl }); + assert.equal(new URL(calls[0]!.url).pathname, "/v1/crawls"); + assert.equal(new URL(calls[0]!.url).searchParams.get("limit"), "5"); + assert.equal(new URL(calls[0]!.url).searchParams.get("cursor"), null); + assert.equal(page.crawls.length, 1); + assert.equal(page.next_cursor, undefined); +}); + +test("stopCrawl posts /crawls/{id}/stop with no body", async () => { + const { impl, calls } = stubFetch(200, { crawl_id: "c_1", status: "stopped", stop_reason: "user" }); + const s = await stopCrawl("c_1", { apiKey: "k", fetchImpl: impl }); + assert.equal(s.status, "stopped"); + assert.equal(calls[0]!.init?.method, "POST"); + assert.equal(calls[0]!.init?.body, undefined); + assert.match(calls[0]!.url, /\/crawls\/c_1\/stop$/); +}); + +test("getCrawlContent returns the HTML body and its type", async () => { + const { impl, calls } = stubFetch(200, "hi", { "content-type": "text/html; charset=utf-8" }); + const out = await getCrawlContent("c_1", "ct_9", { apiKey: "k", fetchImpl: impl }); + assert.match(calls[0]!.url, /\/crawls\/c_1\/contents\/ct_9$/); + assert.match(out.contentType, /text\/html/); + assert.equal(out.body, "hi"); +}); + +test("downloadCrawl returns the NDJSON and X-Crawl-Status", async () => { + const { impl, calls } = stubFetch(200, '{"url":"https://example.com/product/a"}\n{"url":"https://example.com/product/b"}\n', { + "content-type": "application/x-ndjson", + "x-crawl-status": "running", + }); + const out = await downloadCrawl("c_1", { apiKey: "k", fetchImpl: impl }); + assert.match(calls[0]!.url, /\/crawls\/c_1\/download$/); + assert.equal(out.crawlStatus, "running"); + assert.equal(out.ndjson.trim().split("\n").length, 2); +}); + +test("403 REQS008 maps to CRAWL_NOT_ENABLED with the API's wording", async () => { + const { impl } = problem(403, "REQS008", "Crawl is not enabled for this account."); + await assert.rejects( + () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: impl }), + (e: unknown) => + e instanceof ToolkitError && e.code === "CRAWL_NOT_ENABLED" && e.message === "Crawl is not enabled for this account.", + ); +}); + +test("404 crawl_not_found maps to CRAWL_NOT_FOUND", async () => { + const { impl } = problem(404, "crawl_not_found"); + await assert.rejects( + () => getCrawl("c_nope", { apiKey: "k", fetchImpl: impl }), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_NOT_FOUND" && /crawl_not_found/.test(e.likely_cause), + ); +}); + +test("404 content_not_found maps to CRAWL_NOT_FOUND with content guidance", async () => { + const { impl } = problem(404, "content_not_found"); + await assert.rejects( + () => getCrawlContent("c_1", "ct_x", { apiKey: "k", fetchImpl: impl }), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_NOT_FOUND" && /content_status/.test(e.next_action), + ); +}); + +test("422 invalid_parameter and 400 unknown_parameter map to CRAWL_INVALID", async () => { + for (const [status, code] of [ + [422, "invalid_parameter"], + [400, "unknown_parameter"], + [409, "idempotency_request_in_flight"], + ] as const) { + const { impl } = problem(status, code, "'depth' is out of range."); + await assert.rejects( + () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: impl }), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_INVALID" && e.likely_cause.includes(code), + ); + } +}); + +test("429 too_many_crawls maps to CRAWL_QUOTA_EXCEEDED and carries Retry-After", async () => { + const { impl } = problem(429, "too_many_crawls", "Too many crawls.", { "retry-after": "30" }); + await assert.rejects( + () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: impl }), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_QUOTA_EXCEEDED" && /Retry after 30s/.test(e.likely_cause), + ); +}); + +test("401 maps to AUTH_INVALID and 500 to CRAWL_FAILED", async () => { + const a = problem(401, "AUTH002"); + await assert.rejects(() => listCrawls({ apiKey: "k", fetchImpl: a.impl }), (e: unknown) => e instanceof ToolkitError && e.code === "AUTH_INVALID"); + const b = problem(500, "internal_error"); + await assert.rejects(() => getCrawl("c", { apiKey: "k", fetchImpl: b.impl }), (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_FAILED"); +}); + +test("a transport failure maps to BACKEND_UNAVAILABLE", async () => { + const impl = (async () => { + throw new TypeError("fetch failed"); + }) as unknown as typeof fetch; + await assert.rejects(() => getCrawl("c", { apiKey: "k", fetchImpl: impl }), (e: unknown) => e instanceof ToolkitError && e.code === "BACKEND_UNAVAILABLE"); +}); + +/** A stub that serves `pages` in order, recording the cursor each call sent. */ +function pagedFetch(pages: Array>) { + const cursors: Array = []; + const limits: Array = []; + let i = 0; + const impl = (async (url: string) => { + const u = new URL(url); + cursors.push(u.searchParams.get("cursor")); + limits.push(u.searchParams.get("limit")); + const page = pages[Math.min(i++, pages.length - 1)]!; + return new Response(JSON.stringify({ ...crawlBody, ...page }), { status: 200, headers: { "content-type": "application/json" } }); + }) as unknown as typeof fetch; + return { impl, cursors, limits, calls: () => i }; +} + +test("listAllResults follows next_cursor and stops on null", async () => { + const { impl, cursors } = pagedFetch([ + { status: "completed", results: [{ url: "https://example.com/product/1" }], next_cursor: "c1" }, + { status: "completed", results: [{ url: "https://example.com/product/2" }], next_cursor: "c2" }, + { status: "completed", results: [{ url: "https://example.com/product/3" }], next_cursor: null }, + ]); + const { results, crawl } = await listAllResults("c_1", { apiKey: "k", fetchImpl: impl }); + assert.deepEqual(results.map((r) => r.url), ["https://example.com/product/1", "https://example.com/product/2", "https://example.com/product/3"]); + assert.deepEqual(cursors, [null, "c1", "c2"]); + assert.equal(crawl.status, "completed"); +}); + +test("listAllResults refuses a running crawl instead of polling forever", async () => { + const { impl, calls } = pagedFetch([{ status: "running", results: [], next_cursor: "c1" }]); + await assert.rejects( + () => listAllResults("c_1", { apiKey: "k", fetchImpl: impl }), + (e: unknown) => e instanceof ToolkitError && e.code === "INVALID_USAGE" && /still running/.test(e.message), + ); + assert.equal(calls(), 1); +}); + +test("waitForCrawl polls with limit=1 until the status leaves running, backing off", async () => { + const { impl, limits, calls } = pagedFetch([ + { status: "running", results: [], next_cursor: "c" }, + { status: "running", results: [], next_cursor: "c" }, + { status: "running", results: [], next_cursor: "c" }, + { status: "completed", results: [], next_cursor: null }, + ]); + const sleeps: number[] = []; + const c = await waitForCrawl("c_1", { apiKey: "k", fetchImpl: impl, sleepImpl: async (ms) => void sleeps.push(ms) }); + assert.equal(c.status, "completed"); + assert.equal(calls(), 4); + assert.ok(limits.every((l) => l === "1")); + assert.deepEqual(sleeps, [2000, 3000, 4500]); +}); + +test("waitForCrawl treats failed and stopped as terminal", async () => { + for (const status of ["failed", "stopped"]) { + const { impl } = pagedFetch([{ status, results: [], next_cursor: null }]); + const c = await waitForCrawl("c_1", { apiKey: "k", fetchImpl: impl, sleepImpl: async () => {} }); + assert.equal(c.status, status); + } +}); + +test("waitForCrawl times out with CRAWL_TIMEOUT and does not stop the crawl", async () => { + const methods: string[] = []; + const impl = (async (_url: string, init?: RequestInit) => { + methods.push(init?.method ?? "GET"); + return new Response(JSON.stringify({ ...crawlBody, results: [], next_cursor: "c" }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }) as unknown as typeof fetch; + await assert.rejects( + () => waitForCrawl("c_1", { apiKey: "k", fetchImpl: impl, timeoutMs: 1, sleepImpl: async () => {} }), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_TIMEOUT" && /not stopped/.test(e.likely_cause), + ); + assert.ok(methods.every((m) => m === "GET"), "no stop call on timeout"); +}); diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts new file mode 100644 index 0000000..768fb98 --- /dev/null +++ b/tests/crawl-command.test.ts @@ -0,0 +1,324 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { crawl } from "../src/cli/commands/crawl.ts"; +import { createWorkspace } from "../src/core/workspace.ts"; +import { savePolicy, defaultPolicy } from "../src/core/policy.ts"; +import { saveApiKey } from "../src/core/auth.ts"; +import { tempRoot } from "./helpers.ts"; + +const ctx = { json: true, yes: false }; + +const crawlBody = { + crawl_id: "c_1", + status: "running", + url: "https://example.com/products/", + depth: 1, + max_items: 10, + max_pages: 10, + coverage: { pages_fetched: 0, pages_failed: 0, items_found: 0 }, + created_at: "2026-10-15T09:00:00Z", +}; + +type Handler = (url: URL, init?: RequestInit) => Response; + +/** + * Run `fn` in a fresh workspace with a saved key and the given policy; every + * network call goes to `handler` and is recorded. + */ +function withCrawlWorkspace( + handler: Handler, + fn: (calls: Array<{ url: URL; init?: RequestInit }>) => Promise, + policy: Partial> = {}, +): Promise { + const { root, cleanup } = tempRoot(); + const cwd = process.cwd(); + createWorkspace(root); + savePolicy({ ...defaultPolicy(), ...policy }, root); + saveApiKey("0".repeat(41), root); + process.chdir(root); + const calls: Array<{ url: URL; init?: RequestInit }> = []; + const orig = globalThis.fetch; + globalThis.fetch = (async (u: string, init?: RequestInit) => { + const url = new URL(u); + calls.push({ url, init }); + return handler(url, init); + }) as unknown as typeof fetch; + return fn(calls).finally(() => { + globalThis.fetch = orig; + process.chdir(cwd); + cleanup(); + }); +} + +function json(body: unknown, status = 200, headers: Record = {}): Response { + return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json", ...headers } }); +} + +/** Capture stdout (log.out) for the duration of `fn`. */ +async function captureOut(fn: () => unknown): Promise { + const orig = process.stdout.write.bind(process.stdout); + let buf = ""; + process.stdout.write = ((s: string | Uint8Array) => { + buf += typeof s === "string" ? s : Buffer.from(s).toString(); + return true; + }) as typeof process.stdout.write; + try { + await fn(); + } finally { + process.stdout.write = orig; + } + return buf; +} + +async function run(args: string[]): Promise<{ code: number; out: Record }> { + let code = -1; + const raw = await captureOut(async () => { + code = await crawl.run(args, ctx); + }); + return { code, out: JSON.parse(raw) as Record }; +} + +test("crawl start rejects flags it does not declare, before any network call", async () => { + await withCrawlWorkspace( + () => json(crawlBody, 202), + async (calls) => { + for (const flag of ["--pagination", "--discovery", "--json-output"]) { + const { code, out } = await run(["start", "https://example.com/products/", "--depth", "1", flag]); + assert.equal(code, 1, flag); + assert.equal(out.error.code, "UNKNOWN_FLAG", flag); + } + assert.equal(calls.length, 0); + }, + ); +}); + +test("crawl start requires --depth and a whole number in range", async () => { + await withCrawlWorkspace( + () => json(crawlBody, 202), + async (calls) => { + for (const args of [[], ["--depth", "0"], ["--depth", "1.5"], ["--depth", "100001"], ["--depth", "x"]]) { + const { code, out } = await run(["start", "https://example.com/products/", ...args]); + assert.equal(code, 1, args.join(" ")); + assert.equal(out.error.code, "INVALID_USAGE", args.join(" ")); + } + assert.equal(calls.length, 0); + }, + ); +}); + +test("crawl start sends only the flags the caller set as the create body", async () => { + await withCrawlWorkspace( + () => json(crawlBody, 202, { location: "/v1/crawls/c_1" }), + async (calls) => { + const { code, out } = await run([ + "start", + "https://example.com/products/", + "--depth", + "1", + "--max-items", + "3", + "--max-pages", + "5", + "--include", + "/product/", + "--include", + "/item/", + "--exclude", + "?add", + "--html", + ]); + assert.equal(code, 0); + assert.equal(out.ok, true); + assert.equal(out.crawlId, "c_1"); + assert.equal(out.status, "running"); + assert.equal(calls.length, 1); + assert.equal(calls[0]!.url.pathname, "/v1/crawls"); + assert.deepEqual(JSON.parse(String(calls[0]!.init?.body)), { + url: "https://example.com/products/", + depth: 1, + max_items: 3, + max_pages: 5, + include_patterns: ["/product/", "/item/"], + exclude_patterns: ["?add"], + output_format: "html", + }); + }, + ); +}); + +test("crawl start honors the domain policy and the page cap before any network call", async () => { + await withCrawlWorkspace( + () => json(crawlBody, 202), + async (calls) => { + const blocked = await run(["start", "https://blocked.example.com/", "--depth", "1"]); + assert.equal(blocked.code, 1); + const overCap = await run(["start", "https://example.com/products/", "--depth", "1", "--max-pages", "50"]); + assert.equal(overCap.code, 1); + assert.equal(overCap.out.error.code, "POLICY_LIMIT_EXCEEDED"); + assert.equal(calls.length, 0); + }, + { blocked_domains: ["blocked.example.com"], max_pages_per_run: 20 }, + ); +}); + +test("crawl start --follow waits for the end and exits 1 with CRAWL_FAILED on a failed crawl", async () => { + await withCrawlWorkspace( + (url, init) => + init?.method === "POST" + ? json(crawlBody, 202) + : json({ + ...crawlBody, + status: "failed", + error: { code: "seed_unreachable", detail: "The site may be blocking it; check the URL." }, + results: [], + next_cursor: null, + }), + async (calls) => { + const { code, out } = await run(["start", "https://example.com/products/", "--depth", "1", "--follow"]); + assert.equal(code, 1); + assert.equal(out.ok, false); + assert.equal(out.status, "failed"); + assert.equal(out.error.code, "CRAWL_FAILED"); + assert.match(out.error.likely_cause, /seed_unreachable/); + assert.equal(calls[1]!.url.searchParams.get("limit"), "1"); + assert.equal(out.crawl.results, undefined, "status output carries no partial result page"); + }, + ); +}); + +test("crawl start surfaces 403 REQS008 as CRAWL_NOT_ENABLED", async () => { + await withCrawlWorkspace( + () => + json( + { code: "REQS008", title: "Crawl is not enabled for this account.", detail: "Contact support.", status: 403 }, + 403, + { "content-type": "application/problem+json" }, + ), + async () => { + const { code, out } = await run(["start", "https://example.com/products/", "--depth", "1"]); + assert.equal(code, 1); + assert.equal(out.error.code, "CRAWL_NOT_ENABLED"); + assert.equal(out.error.message, "Crawl is not enabled for this account."); + }, + ); +}); + +test("crawl status exits 0 for completed and stopped crawls", async () => { + for (const [status, stop_reason] of [ + ["completed", "max_items"], + ["stopped", "user"], + ]) { + await withCrawlWorkspace( + () => json({ ...crawlBody, status, stop_reason, results: [{ url: "https://example.com/product/a" }], next_cursor: "c" }), + async () => { + const { code, out } = await run(["status", "c_1"]); + assert.equal(code, 0, status); + assert.equal(out.ok, true); + assert.equal(out.stop_reason, stop_reason); + }, + ); + } +}); + +test("crawl status on an unknown id exits 1 with CRAWL_NOT_FOUND", async () => { + await withCrawlWorkspace( + () => json({ code: "crawl_not_found", title: "Crawl not found", detail: "No crawl c_x.", status: 404 }, 404), + async () => { + const { code, out } = await run(["status", "c_x"]); + assert.equal(code, 1); + assert.equal(out.error.code, "CRAWL_NOT_FOUND"); + }, + ); +}); + +test("crawl results reads every page of an ended crawl and writes JSONL with --out", async () => { + const pages: Record = { + "": { ...crawlBody, status: "completed", results: [{ url: "https://example.com/product/1" }], next_cursor: "p2" }, + p2: { ...crawlBody, status: "completed", results: [{ url: "https://example.com/product/2" }], next_cursor: null }, + }; + await withCrawlWorkspace( + (url) => json(pages[url.searchParams.get("cursor") ?? ""]), + async () => { + const { code, out } = await run(["results", "c_1"]); + assert.equal(code, 0); + assert.equal(out.count, 2); + assert.deepEqual(out.results.map((r: { url: string }) => r.url), ["https://example.com/product/1", "https://example.com/product/2"]); + + const file = join(process.cwd(), "urls.jsonl"); + await captureOut(() => crawl.run(["results", "c_1", "--out", file], ctx)); + assert.equal(readFileSync(file, "utf8"), '{"url":"https://example.com/product/1"}\n{"url":"https://example.com/product/2"}\n'); + }, + ); +}); + +test("crawl results --cursor reads one page and returns next_cursor while running", async () => { + await withCrawlWorkspace( + () => json({ ...crawlBody, results: [{ url: "https://example.com/product/3" }], next_cursor: "p4" }), + async (calls) => { + const { code, out } = await run(["results", "c_1", "--cursor", "p3", "--limit", "500"]); + assert.equal(code, 0); + assert.equal(out.next_cursor, "p4"); + assert.equal(calls[0]!.url.searchParams.get("cursor"), "p3"); + assert.equal(calls[0]!.url.searchParams.get("limit"), "500"); + }, + ); +}); + +test("crawl results --download writes the NDJSON export to .jsonl", async () => { + await withCrawlWorkspace( + () => + new Response('{"url":"https://example.com/product/1","content_status":"fetched","content":""}\n', { + status: 200, + headers: { "content-type": "application/x-ndjson", "x-crawl-status": "completed" }, + }), + async (calls) => { + const { code, out } = await run(["results", "c_1", "--download"]); + assert.equal(code, 0); + assert.equal(out.count, 1); + assert.equal(out.partial, false); + assert.equal(calls[0]!.url.pathname, "/v1/crawls/c_1/download"); + assert.ok(existsSync(join(process.cwd(), "c_1.jsonl"))); + }, + ); +}); + +test("crawl content accepts a content_url and prints the page", async () => { + await withCrawlWorkspace( + () => new Response("ok", { status: 200, headers: { "content-type": "text/html" } }), + async (calls) => { + const { code, out } = await run(["content", "c_1", "/v1/crawls/c_1/contents/ct_9"]); + assert.equal(code, 0); + assert.equal(out.content, "ok"); + assert.equal(calls[0]!.url.pathname, "/v1/crawls/c_1/contents/ct_9"); + }, + ); +}); + +test("crawl list and stop call the right endpoints", async () => { + await withCrawlWorkspace( + (url, init) => + init?.method === "POST" + ? json({ crawl_id: "c_1", status: "stopped", stop_reason: "user" }) + : json({ crawls: [{ ...crawlBody, status: "completed" }], next_cursor: "n2" }), + async (calls) => { + const list = await run(["list", "--limit", "5"]); + assert.equal(list.code, 0); + assert.equal(list.out.count, 1); + assert.equal(list.out.next_cursor, "n2"); + assert.equal(calls[0]!.url.searchParams.get("limit"), "5"); + + const stop = await run(["stop", "c_1"]); + assert.equal(stop.code, 0); + assert.equal(stop.out.status, "stopped"); + assert.equal(calls[1]!.url.pathname, "/v1/crawls/c_1/stop"); + }, + ); +}); + +test("unknown crawl subcommand is INVALID_USAGE", async () => { + const { code, out } = await run(["discover"]); + assert.equal(code, 1); + assert.equal(out.error.code, "INVALID_USAGE"); +}); diff --git a/tests/e2e/crawl.e2e.test.ts b/tests/e2e/crawl.e2e.test.ts new file mode 100644 index 0000000..14d5a72 --- /dev/null +++ b/tests/e2e/crawl.e2e.test.ts @@ -0,0 +1,141 @@ +/** + * End-to-end test for `zenrows crawl` against a live Zenrows API. + * + * Drives the built CLI binary (`dist/bin/zenrows.js`) the way a user does, so it + * covers argument parsing, the HTTP client and the output together. Skipped + * unless all are set: + * + * ZENROWS_E2E=1 + * ZENROWS_API_KEY= + * ZENROWS_E2E_CRAWL_URL= + * + * Optional: `ZENROWS_E2E_CRAWL_INCLUDE` (an include pattern every result URL + * must contain) and `ZENROWS_CRAWL_API_BASE` (default https://api.zenrows.com/v1). + * Run with `npm run test:e2e`, which does NOT load tests/setup.ts (that file + * scrubs every ZENROWS_* variable to keep the unit suite hermetic). + * + * The test starts one real, small crawl (bills a few pages on the key's account). + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const START_URL = process.env.ZENROWS_E2E_CRAWL_URL; +const INCLUDE = process.env.ZENROWS_E2E_CRAWL_INCLUDE || undefined; +const enabled = process.env.ZENROWS_E2E === "1" && !!process.env.ZENROWS_API_KEY && !!START_URL; +const skip = enabled + ? false + : "set ZENROWS_E2E=1, ZENROWS_API_KEY and ZENROWS_E2E_CRAWL_URL to run the Crawl e2e test"; + +const BIN = fileURLToPath(new URL("../../bin/zenrows.js", import.meta.url)); +/** How long to keep retrying a start that hits the account's active-crawl limit. */ +const QUOTA_RETRY_MS = 5 * 60_000; + +interface CliResult { + code: number; + out: Record; +} + +/** Run `zenrows --json` in `cwd` and parse its JSON stdout. */ +function cli(cwd: string, args: string[]): Promise { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [BIN, ...args, "--json"], { + cwd, + env: { ...process.env, ZENROWS_CONFIG_HOME: cwd, ZENROWS_TELEMETRY: "off" }, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (d) => (stdout += d)); + child.stderr.on("data", (d) => (stderr += d)); + child.on("error", reject); + child.on("close", (code) => { + try { + resolve({ code: code ?? -1, out: JSON.parse(stdout) as Record }); + } catch { + reject(new Error(`zenrows ${args[0]} ${args[1] ?? ""} printed no JSON (exit ${code}): ${stdout.slice(0, 500)} ${stderr.slice(-500)}`)); + } + }); + }); +} + +function log(msg: string): void { + process.stdout.write(`${msg}\n`); +} + +test("zenrows crawl end to end against a live API", { skip, timeout: 20 * 60_000 }, async () => { + const dir = mkdtempSync(join(tmpdir(), "zr-crawl-e2e-")); + try { + // 1. Start a small crawl and wait for it. Other runs may hold the account's + // active-crawl slots: on CRAWL_QUOTA_EXCEEDED wait Retry-After and try again. + const deadline = Date.now() + QUOTA_RETRY_MS; + let start: CliResult; + for (;;) { + start = await cli(dir, [ + "crawl", "start", START_URL!, + "--depth", "1", "--max-items", "3", "--max-pages", "5", + ...(INCLUDE ? ["--include", INCLUDE] : []), + "--html", "--follow", "--timeout", "600000", + ]); + if (start.out.error?.code !== "CRAWL_QUOTA_EXCEEDED" || Date.now() > deadline) break; + const wait = Number(/Retry after (\d+)s/.exec(start.out.error.likely_cause)?.[1] ?? 30); + log(`too_many_crawls: retrying start in ${wait}s`); + await new Promise((r) => setTimeout(r, wait * 1000)); + } + assert.equal(start.code, 0, JSON.stringify(start.out.error ?? start.out)); + const id = start.out.crawlId as string; + assert.match(id, /\S/); + assert.equal(start.out.status, "completed"); + log(`crawl: ${start.out.status} (${start.out.stop_reason ?? "nothing left"})`); + + // 2. Every kept URL, across all pages. + const results = await cli(dir, ["crawl", "results", id]); + assert.equal(results.code, 0, JSON.stringify(results.out)); + const rows = results.out.results as Array<{ url: string; content_status?: string; content_url?: string }>; + assert.ok(rows.length >= 1, "at least one result"); + if (INCLUDE) for (const r of rows) assert.ok(r.url.includes(INCLUDE), `${r.url} matches --include`); + log(`results: ${rows.length} URL(s)${INCLUDE ? ", all matching the include pattern" : ""}`); + + // 3. One fetched page's HTML. + const fetched = rows.find((r) => r.content_status === "fetched" && r.content_url); + assert.ok(fetched, "at least one result has fetched content"); + const content = await cli(dir, ["crawl", "content", id, fetched.content_url!]); + assert.equal(content.code, 0, JSON.stringify(content.out)); + assert.match(content.out.contentType, /text\/html/); + assert.match(content.out.content, / l.trim()); + assert.equal(lines.length, rows.length); + assert.equal(download.out.count, rows.length); + log(`download: ${lines.length} NDJSON line(s), X-Crawl-Status ${download.out.crawlStatus}`); + + // 5. The new crawl is listed. + const list = await cli(dir, ["crawl", "list", "--limit", "100"]); + assert.equal(list.code, 0, JSON.stringify(list.out)); + assert.ok((list.out.crawls as Array<{ crawl_id: string }>).some((c) => c.crawl_id === id), "crawl is listed"); + log("list: the new crawl is listed"); + + // 6. Stopping an ended crawl answers as it ended. + const stop = await cli(dir, ["crawl", "stop", id]); + assert.equal(stop.code, 0, JSON.stringify(stop.out)); + assert.equal(stop.out.status, start.out.status); + log(`stop on ended crawl: status ${stop.out.status}`); + + // 7. An unknown id is CRAWL_NOT_FOUND. + const missing = await cli(dir, ["crawl", "status", "c_does_not_exist_e2e"]); + assert.equal(missing.code, 1); + assert.equal(missing.out.error.code, "CRAWL_NOT_FOUND"); + log("status on unknown id: CRAWL_NOT_FOUND"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/production-surface.test.ts b/tests/production-surface.test.ts index 4b6c420..db39b18 100644 --- a/tests/production-surface.test.ts +++ b/tests/production-surface.test.ts @@ -3,7 +3,8 @@ import assert from "node:assert/strict"; import { main } from "../src/cli/index.ts"; test("removed non-GA commands are not registered", async () => { - for (const cmd of ["crawl", "discover", "monitor"]) { + // `crawl` was here while it was non-GA; it is registered now. + for (const cmd of ["discover", "monitor"]) { const code = await main([cmd]); assert.equal(code, 1, `${cmd} should be unknown (exit 1)`); } @@ -15,4 +16,6 @@ test("GA commands remain registered", async () => { assert.equal(await main(["account", "--help"]), 0); // batch is registered again (beta) — --help exits 0 assert.equal(await main(["batch", "--help"]), 0); + // crawl is registered + assert.equal(await main(["crawl", "--help"]), 0); }); diff --git a/tests/registry.test.ts b/tests/registry.test.ts index b09c135..3ca7fb7 100644 --- a/tests/registry.test.ts +++ b/tests/registry.test.ts @@ -29,6 +29,7 @@ test("core skills include the GA set and the master decision-tree skill", () => "protected-fetch", "extract", "batch-jobs", + "crawl", "interact-browser", "cost-control", "trace-debug", @@ -36,8 +37,8 @@ test("core skills include the GA set and the master decision-tree skill", () => ]) { assert.ok(skills.includes(required), `skill ${required} present`); } - // Non-GA skills were trimmed from the registry. - for (const removed of ["discover", "crawl", "monitor"]) { + // Non-GA skills were trimmed from the registry (crawl is back). + for (const removed of ["discover", "monitor"]) { assert.ok(!skills.includes(removed), `skill ${removed} removed`); } assert.ok(existsSync(pkgPath("skills/zenrows/SKILL.md")), "master SKILL.md exists"); @@ -50,4 +51,6 @@ test("assetRunnable reflects backend capability status", () => { assert.equal(assetRunnable(batchSkill), true); // needs batch (open beta = usable) const extractSkill = loadRegistry("skill").find((s) => s.name === "extract")!; assert.equal(assetRunnable(extractSkill), true); // needs extract (open beta = usable) + const crawlSkill = loadRegistry("skill").find((s) => s.name === "crawl")!; + assert.equal(assetRunnable(crawlSkill), true); // needs crawl (available) }); From b0823cf5e6be0bf4b0e1e37b2da0f83ce99baafd Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Fri, 9 Oct 2026 12:39:24 +0200 Subject: [PATCH 02/19] fix(crawl): mark Crawl beta and align errors with the Crawl API Mark the crawl capability and skill as beta. Say Beta in the help, the README and the skills. Map Crawl errors to codes an agent can act on. 429 is now CRAWL_TOO_MANY_CRAWLS with a retry_after field. 400 and 422 are CRAWL_INVALID_REQUEST. 409 is CRAWL_REQUEST_IN_FLIGHT. A missing page is CRAWL_CONTENT_NOT_FOUND. A wait timeout is CRAWL_WAIT_TIMEOUT. Every error carries the API code as server_code, and crawl_id once the crawl exists. Keep --wait and remove --follow. Ctrl-C during a wait now says that the crawl keeps running, gives the resume command and exits 130. results returns what a running crawl kept, with partial: true. results --out --json prints the JSON envelope. stop on an ended crawl prints the status it ended with. The help and docs say that the local policy caps --max-pages and that --html uses up the page budget. --- README.md | 36 +++++++---- docs/capabilities.md | 2 +- docs/contributing.md | 5 +- registry/capabilities.json | 4 +- registry/skills.json | 4 +- skills/crawl/SKILL.md | 53 +++++++++++----- skills/trace-debug/SKILL.md | 5 +- skills/zenrows/SKILL.md | 4 +- src/cli/commands/crawl.ts | 104 ++++++++++++++++++++------------ src/core/crawl-api.ts | 116 ++++++++++++++++++++++-------------- src/core/errors.ts | 23 ++++++- tests/capabilities.test.ts | 2 +- tests/crawl-api.test.ts | 97 +++++++++++++++++++++++------- tests/crawl-command.test.ts | 87 +++++++++++++++++++++++++-- tests/e2e/crawl.e2e.test.ts | 8 +-- tests/registry.test.ts | 2 +- 16 files changed, 397 insertions(+), 155 deletions(-) diff --git a/README.md b/README.md index 0973eaf..64fc422 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ that endpoint with extraction parameters, not a separate product. | `zenrows fetch` | Fetch — retrieve a protected page | **available** — `GET https://api.zenrows.com/v1/` | | `zenrows extract` | Extract — `extract=auto` (domain-gated open beta; falls back to Autoparse) / CSS / Markdown | **beta** — same `/v1/` | | `zenrows batch` | Batch — fan out over many URLs | beta — cloud works with beta access; local validate/estimate always | -| `zenrows crawl` | Crawl — collect the URLs behind one start page, optionally with each page's HTML | **available** — `https://api.zenrows.com/v1/crawls` | +| `zenrows crawl` | Crawl (beta) — collect the URLs behind one start page, optionally with each page's HTML | beta — `https://api.zenrows.com/v1/crawls` | | `zenrows browser` | Browser Sessions REST API (same backend as MCP `browser_*`) | **available** — escalation-only; bills by bandwidth + time | | `zenrows mcp` | MCP server config (remote + local) | **available** | | Zenrows CLI | this repo | available | @@ -157,19 +157,19 @@ An Extract task costs the same as a regular one (1 credit at base tier), so keys: `html` (the raw page) and `parsed` (the structured data) — validate a sample with `zenrows extract ` before running the full batch. -## 10. Crawl +## 10. Crawl (beta) -Zenrows **Crawl** (`https://api.zenrows.com/v1/crawls`) takes one start URL and -returns the URLs it finds behind it, optionally with each page's HTML. Use it -when you know the listing or section but not the item URLs. Crawl follows the -links on each page up to `--depth` hops and stays on the start URL's domain. -Without `--html` it returns URLs only. An account without Crawl access gets +Zenrows **Crawl** is in beta. Crawl (`https://api.zenrows.com/v1/crawls`) +takes one start URL and returns the URLs it finds behind it, optionally with +each page's HTML. Use it when you know the listing or section but not the item +URLs. Crawl follows the links on each page up to `--depth` hops and stays on +the start URL's registrable domain (subdomains count). Without `--html` it returns URLs only. An account without Crawl access gets `CRAWL_NOT_ENABLED` ("Crawl is not enabled for this account", HTTP 403 `REQS008`). ```bash zenrows crawl start https://example.com/products/ --depth 1 \ - --max-items 20 --include /product/ --follow # start and wait for the end + --max-items 20 --include /product/ --wait # start and wait for the end zenrows crawl results --out urls.jsonl # every kept URL (paginated) zenrows crawl start --depth 2 --html # also fetch each kept page's HTML zenrows crawl content # one page's HTML @@ -181,10 +181,22 @@ zenrows crawl stop # stop a running crawl `--depth` (link hops) is required; `--max-items` and `--max-pages` cap the crawl (API default 10 each), and `--max-pages` bounds the cost, since each page -is one fetch. `--include` / `--exclude` are repeatable substring filters on the -URL. When the account has too many crawls running, a start gets -`CRAWL_QUOTA_EXCEEDED` and creates nothing; retry after `Retry-After`. Set -`ZENROWS_CRAWL_API_BASE` to point the CLI at another deployment. +is one fetch. The local policy `max_pages_per_run` caps `--max-pages` (default +1000). `--html` uses up the page budget: each kept page is one more fetch, and +`--max-pages` counts it. `--include` / `--exclude` are repeatable substring +filters on the URL. `results` on a running crawl returns the URLs kept so far, +with `partial: true`. `--wait` gives up after `--timeout` (default 600000 ms) +with `CRAWL_WAIT_TIMEOUT`; the crawl keeps running. Ctrl-C stops the wait, not +the crawl, and exits 130; resume with `zenrows crawl wait `. + +Errors carry the API's code as `server_code`, and `crawl_id` once the crawl +exists. When the account has reached its limit of active jobs (3 by default), +shared with its Batch jobs, a start gets `CRAWL_TOO_MANY_CRAWLS` and creates +nothing; retry after `retry_after` seconds. Other codes: `CRAWL_INVALID_REQUEST` +(400/422, do not retry as is), `CRAWL_REQUEST_IN_FLIGHT` (409, retry once the +first request ends), `CRAWL_NOT_FOUND`, `CRAWL_CONTENT_NOT_FOUND` and +`CRAWL_FAILED`. Set `ZENROWS_CRAWL_API_BASE` to point the CLI at another +deployment. ## 11. Browser Sessions diff --git a/docs/capabilities.md b/docs/capabilities.md index 2557ee7..96f241e 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -22,6 +22,6 @@ Classification is based on the public Zenrows documentation: | `protected_fetch` | Fetch `GET https://api.zenrows.com/v1/` with `mode`, `js_render`, `premium_proxy`, `proxy_country`, `wait`/`wait_for`, `js_instructions`, `response_type`, `screenshot`, `original_status`, … | available | | `extract` | Extract `GET https://api.zenrows.com/v1/` via `extract=auto` (domain-gated open beta; CLI falls back to `autoparse`), plus `autoparse`, `css_extractor`, `outputs`, `response_type=markdown\|plaintext` | beta | | `batch` | Batch `https://async.api.zenrows.com/v1` (separate host, `X-API-Key` header) — open beta. Local JSONL spec validation + credit estimation work with no key. | beta | -| `crawl` | Crawl `https://api.zenrows.com/v1/crawls` (`X-API-Key` header; same host as Fetch/Extract). Follows the links on each page from one start URL up to `depth`, on the start URL's domain; returns URLs, and with `output_format: "html"` each page's HTML. Without Crawl access the API returns 403 `REQS008` → `CRAWL_NOT_ENABLED`. | available | +| `crawl` | Crawl `https://api.zenrows.com/v1/crawls` (`X-API-Key` header; same host as Fetch/Extract). Follows the links on each page from one start URL up to `depth`, on the start URL's registrable domain (subdomains count); returns URLs, and with `output_format: "html"` each page's HTML. Without Crawl access the API returns 403 `REQS008` → `CRAWL_NOT_ENABLED`. | beta | | `browser` | Browser Sessions REST API `https://mcp.zenrows.com/browser/sessions/*` (Bearer; same backend as `@zenrows/mcp` `browser_*`); CDP via `zenrows browser connect`. Escalation-only (prefer fetch/extract); on by default, opt out via `policy.allow_browser`; bills by bandwidth + session time (15-min max) | available | | `mcp` | Hosted `https://mcp.zenrows.com/mcp` + local `npx -y @zenrows/mcp` | available | diff --git a/docs/contributing.md b/docs/contributing.md index cc3a519..437ef4d 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -65,8 +65,9 @@ ZENROWS_E2E=1 ZENROWS_E2E_CRAWL_URL=https://example.com/products/ \ ``` To target a local or staging deployment, set `ZENROWS_CRAWL_API_BASE` to its -`/v1` base. When the account has too many crawls running, the test waits -`Retry-After` and retries the start for up to 5 minutes, so run concurrent e2e +`/v1` base. When the account has reached its limit of active jobs (3 by +default, shared with its Batch jobs), the test waits `retry_after` seconds and +retries the start for up to 5 minutes, so run concurrent e2e runs on one account one at a time. ## Rules diff --git a/registry/capabilities.json b/registry/capabilities.json index 36c8a1f..b331569 100644 --- a/registry/capabilities.json +++ b/registry/capabilities.json @@ -31,11 +31,11 @@ "crawl": { "key": "crawl", "label": "Crawl", - "status": "available", + "status": "beta", "command": "zenrows crawl", "backend": "Crawl — https://api.zenrows.com/v1/crawls (X-API-Key header; same host as Fetch/Extract)", "requiresAuth": true, - "notes": "Follows the links on each page from one start URL up to depth, on the start URL's domain, and returns the URLs it keeps, optionally with each page's HTML (start/status/wait/results/content/list/stop). An account without Crawl access gets 403 REQS008 (CRAWL_NOT_ENABLED). Too many crawls running on the account gives 429 too_many_crawls (CRAWL_QUOTA_EXCEEDED); retry after Retry-After." + "notes": "Beta. Follows the links on each page from one start URL up to depth, on the start URL's registrable domain (subdomains count), and returns the URLs it keeps, optionally with each page's HTML (start/status/wait/results/content/list/stop). An account without Crawl access gets 403 REQS008 (CRAWL_NOT_ENABLED). When the account has reached its limit of active jobs (3 by default), shared with its Batch jobs, a start gets 429 too_many_crawls (CRAWL_TOO_MANY_CRAWLS); retry after retry_after seconds." }, "browser": { "key": "browser", diff --git a/registry/skills.json b/registry/skills.json index 2658341..7a01812 100644 --- a/registry/skills.json +++ b/registry/skills.json @@ -47,8 +47,8 @@ { "name": "crawl", "type": "skill", - "description": "Collect the URLs behind one start page with Crawl: start, wait, read results and page HTML, stop.", - "status": "available", + "description": "Collect the URLs behind one start page with Crawl (beta): start, wait, read results and page HTML, stop.", + "status": "beta", "requires_backend_capabilities": ["crawl"], "requires_auth": true, "version": "0.1.0", diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index 0d8136e..7008f59 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -1,20 +1,20 @@ --- name: crawl -description: Collect the URLs behind one start page with Crawl — start a crawl, wait for it, read the kept URLs and optionally each page's HTML, stop it. Use when the user has a listing or section URL but not the item URLs. +description: Collect the URLs behind one start page with Crawl (beta) — start a crawl, wait for it, read the kept URLs and optionally each page's HTML, stop it. Use when the user has a listing or section URL but not the item URLs. version: 0.1.0 requires_backend_capabilities: [crawl] --- -# Crawl +# Crawl (beta) -Give Crawl one start URL; it follows the links on that page (and, with a +Crawl is in beta. Give Crawl one start URL; it follows the links on that page (and, with a higher depth, on the pages behind it) and returns the URLs it keeps. Use it when you know where the items live (a category page, a blog index, a docs section) but not the item URLs themselves. Once you have the URLs, read pages with `zenrows fetch` / `zenrows extract`, or ask Crawl for each page's HTML with `--html`. -Crawl stays on the start URL's domain. An account without Crawl access gets +Crawl stays on the start URL's registrable domain; subdomains count. An account without Crawl access gets 403 `REQS008` → `CRAWL_NOT_ENABLED` ("Crawl is not enabled for this account"); do not retry it. @@ -22,8 +22,8 @@ do not retry it. ``` zenrows crawl start https://example.com/products/ --depth 1 \ - --max-items 20 --max-pages 25 --include /product/ --follow --json -zenrows crawl results --json # every kept URL (crawl must have ended) + --max-items 20 --max-pages 25 --include /product/ --wait --json +zenrows crawl results --json # every kept URL zenrows crawl results --out urls.jsonl # same, as JSONL ``` @@ -31,23 +31,31 @@ zenrows crawl results --out urls.jsonl # same, as JSONL page's links; `2` = also the links on each of those pages. - `--max-items` (kept URLs) and `--max-pages` (fetched pages) cap the crawl; both default to 10. `--max-pages` bounds the cost: each page is one fetch. + The local policy `max_pages_per_run` caps `--max-pages` (default 1000); + above it the CLI refuses with `POLICY_LIMIT_EXCEEDED` before any request. - `--include` / `--exclude` are substring matches on the normalized URL and repeat: `--include /product/ --exclude ?add-to-cart`. -- `start` returns at once. `--follow` (or `zenrows crawl wait `) polls - until the crawl ends; on `--timeout` the crawl keeps running (it is not - stopped) and you get `CRAWL_TIMEOUT`. +- `start` returns at once. `--wait` (or `zenrows crawl wait `) polls + until the crawl ends (default timeout 600000 ms). On `--timeout` the crawl + keeps running (it is not stopped) and you get `CRAWL_WAIT_TIMEOUT` with + `crawl_id`. Ctrl-C also stops the wait, not the crawl, and exits 130; + resume with `zenrows crawl wait `. +- `results` on a running crawl returns the URLs kept so far, with + `partial: true`. Read again once the crawl ends. ## Page HTML ``` -zenrows crawl start --depth 1 --html --follow +zenrows crawl start --depth 1 --html --max-pages 20 --wait zenrows crawl results --json # fetched results carry content_url zenrows crawl content # one page's HTML (or pass the content_url) zenrows crawl results --download --out pages.jsonl # every URL + HTML, NDJSON ``` +`--html` uses up the page budget: each kept page is one more fetch, and +`--max-pages` counts it. Raise `--max-pages` to keep the same number of URLs. A result's `content_status` is `pending`, `fetched` or `failed`; only -`fetched` results have content. +`fetched` results have content (else `CRAWL_CONTENT_NOT_FOUND`). ## While it runs, and after @@ -61,12 +69,29 @@ zenrows crawl stop # stop it; kept URLs stay readab A crawl ends `completed` (nothing left, or a cap hit: `stop_reason` `max_items` / `max_pages`), `stopped` (you stopped it) or `failed` (`error.code` and `error.detail` say why). Only `failed` exits non-zero (`CRAWL_FAILED`). +`stop` on a crawl that already ended prints the status it ended with. ## Limits -When the account has too many crawls running, a start gets 429 -`too_many_crawls` → `CRAWL_QUOTA_EXCEEDED`; nothing was created. Retry after -`Retry-After` (about 30s), or stop one of your crawls first. +When the account has reached its limit of active jobs (3 by default), shared +with its Batch jobs, a start gets 429 `too_many_crawls` → +`CRAWL_TOO_MANY_CRAWLS`; nothing was created. Retry after `retry_after` +seconds (about 30), or stop one of your crawls or Batch jobs first. + +## Errors + +Every Crawl error carries the API's code as `server_code` (when the API sent +one), and `crawl_id` once the crawl exists. + +| Code | Meaning | Retry? | +| --- | --- | --- | +| `CRAWL_NOT_ENABLED` | 403 `REQS008`: Crawl is not enabled for this account | no | +| `CRAWL_INVALID_REQUEST` | 400 or 422: fix the request. For `idempotency_key_reused`, send a new key or no key | no, not as is | +| `CRAWL_REQUEST_IN_FLIGHT` | 409: the same Idempotency-Key is still in flight | yes, once the first request ends | +| `CRAWL_NOT_FOUND` / `CRAWL_CONTENT_NOT_FOUND` | 404: wrong id, or the page has no content | no | +| `CRAWL_TOO_MANY_CRAWLS` | 429: the account's active-job limit | yes, after `retry_after` | +| `CRAWL_WAIT_TIMEOUT` | the wait ran out; the crawl keeps running | wait again | +| `CRAWL_FAILED` | the crawl ended `failed`, or another error | 5xx: yes, with backoff | Rules: start small (`--depth 1`, low caps) and check the results before a deep crawl. Mind the cost ([[cost-control]]). Read errors with diff --git a/skills/trace-debug/SKILL.md b/skills/trace-debug/SKILL.md index e786019..f0e35a2 100644 --- a/skills/trace-debug/SKILL.md +++ b/skills/trace-debug/SKILL.md @@ -47,8 +47,9 @@ zenrows trace export # JSON for sharing `failure_detail` in `zenrows batch status --json` say why. - `CRAWL_NOT_ENABLED` → the API answered 403 `REQS008`: Crawl is not enabled for this account. Do not retry; ask Zenrows support for access. -- `CRAWL_QUOTA_EXCEEDED` → 429 `too_many_crawls`: the account has too many - crawls running. Nothing was created; retry after `Retry-After` (see +- `CRAWL_TOO_MANY_CRAWLS` → 429 `too_many_crawls`: the account has reached + its limit of active jobs (3 by default), shared with its Batch jobs. + Nothing was created; retry after `retry_after` seconds (see `zenrows crawl list`). - `CRAWL_FAILED` → the crawl ended `failed`; `error.code` in `zenrows crawl status --json` says why, with `error.detail`. See diff --git a/skills/zenrows/SKILL.md b/skills/zenrows/SKILL.md index 58f87df..1f885d4 100644 --- a/skills/zenrows/SKILL.md +++ b/skills/zenrows/SKILL.md @@ -35,7 +35,7 @@ If the user has a known URL and wants structured data: → Use Extract. (zenrows extract | --autoparse | --css) If the user has a start page (listing, index, section) but not the item URLs: - → Use Crawl. (zenrows crawl start --depth 1 --follow, + → Use Crawl. (zenrows crawl start --depth 1 --wait, then `zenrows crawl results `) If the user has many URLs (bulk): @@ -74,7 +74,7 @@ Run `zenrows status` for the live capability matrix. As of this toolkit: | Protected Fetch | `zenrows fetch` | available (`GET /v1/`) | | Extract (extract=auto / Autoparse/CSS/Markdown) | `zenrows extract` | beta (same `/v1/`; extract=auto falls back to autoparse) | | Batch | `zenrows batch` | beta — runs cloud jobs (create/status/results); cheapest path at scale. Estimate specs locally with no key | -| Crawl | `zenrows crawl` | available — collect the URLs behind one start page, optionally with each page's HTML | +| Crawl | `zenrows crawl` | beta — collect the URLs behind one start page, optionally with each page's HTML | | Browser | `zenrows browser` | available (Browser Sessions REST API / MCP) | | MCP | `zenrows mcp` | available (remote + local server) | diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index a21775b..5579c75 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -1,9 +1,9 @@ /** - * `zenrows crawl` — Crawl API. + * `zenrows crawl` — Crawl API (status: beta). * * Give Crawl one start URL and read back the URLs it finds behind it, optionally * with each page's HTML. A crawl is an async job: `start` returns at once - * (or polls with `--follow`), then `status` / `wait` / `results` / `content` + * (or polls with `--wait`), then `status` / `wait` / `results` / `content` * read it and `stop` ends it. An account without Crawl access gets 403 REQS008 * → CRAWL_NOT_ENABLED. */ @@ -24,6 +24,7 @@ import { stopCrawl, waitForCrawl, type Crawl, + type CrawlResult, type CrawlStop, type CreateCrawlParams, } from "../../core/crawl-api.ts"; @@ -39,25 +40,27 @@ const MAX_LIMIT = 100_000; export const crawl: Command = { name: "crawl", - summary: "Crawl a site from one start URL and collect the URLs behind it.", + summary: "Crawl a site from one start URL and collect the URLs behind it (beta).", usage: "zenrows crawl --depth N|status |results |content |list|stop |wait >", help: [ - "Cloud (needs a key with Crawl access):", + "Crawl is in beta. Cloud (needs a key with Crawl access):", " start --depth [flags] start a crawl from one URL (returns at once)", " --depth link hops to follow from the start URL (1-100000, required)", " --max-items stop after keeping n URLs (API default 10)", - " --max-pages stop after fetching n pages (API default 10; bounds cost)", + " --max-pages stop after fetching n pages (API default 10; bounds cost);", + " the local policy max_pages_per_run caps it (default 1000)", " --include keep only URLs containing this substring (repeatable)", " --exclude drop URLs containing this substring (repeatable)", - " --html also fetch each kept URL's page HTML (read with `content`)", - " --follow poll until the crawl ends (alias: --wait)", - " --timeout with --follow: give up waiting after ms (default 600000)", + " --html also fetch each kept URL's page HTML (read with `content`);", + " each kept page is one more fetch counted by --max-pages", + " --wait poll until the crawl ends (Ctrl-C stops the wait, not the crawl)", + " --timeout with --wait: give up waiting after ms (default 600000)", " --idempotency-key make a retried start create no second crawl", " --no-signup do not auto-create a Free plan account if no key exists", " status show status, coverage, stop reason / error", " wait [--timeout ] poll until the crawl ends (it is not stopped on timeout)", - " results all kept URLs (crawl must have ended); paginated", + " results all kept URLs; on a running crawl, those kept so far (partial)", " --out write results as JSONL instead of printing", " --cursor [--limit ] read one page only (works while running; prints next_cursor)", " --download fetch the NDJSON export (with page HTML for --html crawls)", @@ -65,7 +68,7 @@ export const crawl: Command = { " content print one kept URL's page (content_id or content_url)", " --out write it to a file instead", " list [--limit ] [--cursor c] list your crawls, newest first", - " stop stop a running crawl (idempotent)", + " stop stop a running crawl (an ended crawl answers as it ended)", " --json print structured output", ].join("\n"), async run(argv: string[], ctx: RunContext): Promise { @@ -111,15 +114,14 @@ async function startCmd(rest: string[], ctx: RunContext): Promise { include: { type: "string", multiple: true }, exclude: { type: "string", multiple: true }, html: { type: "boolean" }, - follow: { type: "boolean" }, - wait: { type: "boolean" }, // alias for --follow + wait: { type: "boolean" }, timeout: { type: "string" }, "idempotency-key": { type: "string" }, "no-signup": { type: "boolean" }, json: { type: "boolean" }, }); const json = ctx.json || values.json === true; - const follow = values.follow === true || values.wait === true; + const wait = values.wait === true; const url = positionals[0]; if (!url) { throw new ToolkitError({ @@ -173,7 +175,7 @@ async function startCmd(rest: string[], ctx: RunContext): Promise { log.step(`Starting crawl of ${url} (depth ${depth})…`); try { const created = await createCrawl(params, { apiKey, idempotencyKey }); - const finished: Crawl = follow ? await waitForCrawl(created.crawl_id, { apiKey, timeoutMs }) : created; + const finished: Crawl = wait ? await waitUntilEnd(created.crawl_id, { apiKey, timeoutMs }) : created; const failure = crawlFailure(finished); const runDir = writeRun({ runId, @@ -190,7 +192,7 @@ async function startCmd(rest: string[], ctx: RunContext): Promise { const code = printCrawl(finished, json, `Started crawl ${created.crawl_id}`); if (!json) { if (runDir) log.dim(` artifact: ${runDir}`); - if (!follow) log.dim(` next: zenrows crawl wait ${created.crawl_id}`); + if (!wait) log.dim(` next: zenrows crawl wait ${created.crawl_id}`); } return code; } catch (err) { @@ -224,7 +226,7 @@ async function waitCmd(rest: string[], ctx: RunContext): Promise { const timeoutMs = normalizeTimeout(values.timeout); assertUsable("crawl"); const apiKey = requireApiKey(); - const c = await waitForCrawl(id, { apiKey, timeoutMs }); + const c = await waitUntilEnd(id, { apiKey, timeoutMs }); return printCrawl(c, ctx.json || values.json === true, `Crawl ${id} finished`); } @@ -262,31 +264,38 @@ async function resultsCmd(rest: string[], ctx: RunContext): Promise { // One page, as the API returns it — the way to follow a crawl while it runs. if (cursor !== undefined || limit !== undefined) { const page = await getCrawl(id, { apiKey, cursor, limit }); - const body = { ok: true, crawlId: id, status: page.status, count: page.results.length, results: page.results, next_cursor: page.next_cursor }; - if (outFile) { - writeOut(outFile, toJsonl(page.results)); - log.success(`Wrote ${page.results.length} result(s) → ${outFile}`); - log.info(`next_cursor: ${page.next_cursor ?? "null (last page)"}`); - } else if (json) { - log.out(JSON.stringify(body, null, 2)); - } else { - log.info(`${page.results.length} result(s) for crawl ${id} (status: ${page.status}):`); - log.out(toJsonl(page.results).trimEnd()); - log.info(`next_cursor: ${page.next_cursor ?? "null (last page)"}`); - } - return 0; + return printResults(id, page.status, page.results, { outFile, json, next_cursor: page.next_cursor ?? null }); } - const { crawl: c, results } = await listAllResults(id, { apiKey }); - if (outFile) { - writeOut(outFile, toJsonl(results)); - log.success(`Wrote ${results.length} result(s) → ${outFile}`); - } else if (json) { - log.out(JSON.stringify({ ok: true, crawlId: id, status: c.status, count: results.length, results }, null, 2)); + const { crawl: c, results, partial } = await listAllResults(id, { apiKey }); + return printResults(id, c.status, results, { outFile, json, partial }); +} + +/** Write or print results. Under --json, always print the envelope (`file` replaces `results` with --out). */ +function printResults( + id: string, + status: string, + results: CrawlResult[], + o: { outFile?: string; json: boolean; partial?: boolean; next_cursor?: string | null }, +): number { + const extra = { + ...(o.partial !== undefined ? { partial: o.partial } : {}), + ...(o.next_cursor !== undefined ? { next_cursor: o.next_cursor } : {}), + }; + if (o.outFile) writeOut(o.outFile, toJsonl(results)); + if (o.json) { + const rows = o.outFile ? { file: o.outFile } : { results }; + log.out(JSON.stringify({ ok: true, crawlId: id, status, count: results.length, ...rows, ...extra }, null, 2)); + return 0; + } + if (o.outFile) { + log.success(`Wrote ${results.length} result(s) → ${o.outFile}`); } else { - log.info(`${results.length} result(s) for crawl ${id} (status: ${c.status}):`); + log.info(`${results.length} result(s) for crawl ${id} (status: ${status}):`); log.out(toJsonl(results).trimEnd()); } + if (o.partial) log.warn(" The crawl is still running: these results are partial. Read them again once it ends."); + if (o.next_cursor !== undefined) log.info(`next_cursor: ${o.next_cursor ?? "null (last page)"}`); return 0; } @@ -356,7 +365,26 @@ async function stopCmd(rest: string[], ctx: RunContext): Promise { assertUsable("crawl"); const apiKey = requireApiKey(); const s = await stopCrawl(id, { apiKey }); - return printCrawl(s, ctx.json || values.json === true, `Stopped crawl ${id}`); + const headline = s.status === "stopped" ? `Stopped crawl ${id}` : `Nothing to stop: crawl ${id} had already ended`; + return printCrawl(s, ctx.json || values.json === true, headline); +} + +/** + * Wait for a crawl to end. Ctrl-C ends the wait, not the crawl: say so, give + * the command that resumes the wait, and exit 130 as shells do on SIGINT. + */ +async function waitUntilEnd(id: string, opts: { apiKey: string; timeoutMs?: number }): Promise { + const onSigint = () => { + log.warn(`Stopped waiting. Crawl ${id} is still running; it was not stopped.`); + log.dim(` resume: zenrows crawl wait ${id}`); + process.exit(130); + }; + process.once("SIGINT", onSigint); + try { + return await waitForCrawl(id, opts); + } finally { + process.off("SIGINT", onSigint); + } } /** @@ -379,6 +407,8 @@ export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null { likely_cause: why || "The crawl ended in status failed without a reason.", next_action: (code && next[code]) || "Read the error, fix the cause, then start a new crawl.", suggested_commands: [`zenrows crawl status ${c.crawl_id}`], + server_code: code, + crawl_id: c.crawl_id, }); } diff --git a/src/core/crawl-api.ts b/src/core/crawl-api.ts index f434f51..f98b248 100644 --- a/src/core/crawl-api.ts +++ b/src/core/crawl-api.ts @@ -138,6 +138,8 @@ interface RequestOpts { accept?: string; fetchImpl?: typeof fetch; timeoutMs?: number; + /** Set on every call about one crawl, so its errors carry `crawl_id`. */ + crawlId?: string; } interface RawResponse { @@ -188,13 +190,14 @@ async function crawlFetch(method: string, path: string, opts: RequestOpts): Prom next_action: "Check connectivity and retry. Override the host with ZENROWS_CRAWL_API_BASE if you are testing against staging.", suggested_commands: ["zenrows status"], + crawl_id: opts.crawlId, }); } finally { clearTimeout(timeout); } if (res.status < 200 || res.status >= 300) { - throw problemToError(res.status, text, method, path, res.headers.get("retry-after")); + throw problemToError(res.status, text, `${method} ${path}`, res.headers.get("retry-after"), opts.crawlId); } return { status: res.status, headers: res.headers, text }; } @@ -210,17 +213,21 @@ async function crawlJson(method: string, path: string, opts: RequestOpts): Pr message: "The Crawl API response was not valid JSON.", likely_cause: text.slice(0, 240) || "Empty response body.", next_action: "Retry, or check the crawl with `zenrows crawl status `.", + crawl_id: opts.crawlId, }); } } -/** Map a problem+json body + HTTP status to a normalized ToolkitError. */ +/** + * Map a problem+json body + HTTP status to a normalized ToolkitError that + * carries the API's `code` as `server_code`, and `crawl_id` when known. + */ function problemToError( status: number, body: string, - method: string, - path: string, + request: string, retryAfter: string | null, + crawlId?: string, ): ToolkitError { let problem: ProblemJson = {}; try { @@ -228,9 +235,22 @@ function problemToError( } catch { // non-JSON error body — fall through with an empty problem } - const serverCode = problem.code ?? ""; + const serverCode = problem.code || undefined; + const seconds = status === 429 && retryAfter && /^\d+$/.test(retryAfter.trim()) ? Number(retryAfter) : undefined; + const err = classifyProblem(status, problem, body, request, serverCode, seconds); + return new ToolkitError({ ...err.toJSON(), server_code: serverCode, crawl_id: crawlId, retry_after: seconds }); +} + +function classifyProblem( + status: number, + problem: ProblemJson, + body: string, + request: string, + serverCode: string | undefined, + retryAfter: number | undefined, +): ToolkitError { const detail = problem.detail || problem.title || body.slice(0, 240) || `HTTP ${status}`; - const cause = `HTTP ${status}${serverCode ? ` (${serverCode})` : ""} for ${method} ${path}: ${detail}`; + const cause = `HTTP ${status}${serverCode ? ` (${serverCode})` : ""} for ${request}: ${detail}`; if (status === 403 && serverCode === CRAWL_NOT_ENABLED_CODE) { return new ToolkitError({ @@ -251,49 +271,59 @@ function problemToError( suggested_commands: ["zenrows login --api-key "], }); } + if (status === 404 && serverCode === "content_not_found") { + return new ToolkitError({ + code: "CRAWL_CONTENT_NOT_FOUND", + message: "Crawl content not found.", + likely_cause: `${cause}. The page was not fetched (yet), its fetch failed, or the crawl ran without --html.`, + next_action: "Read the result's content_status with `zenrows crawl results `; only `fetched` results have content.", + suggested_commands: ["zenrows crawl list"], + }); + } if (status === 404) { - const content = serverCode === "content_not_found"; return new ToolkitError({ code: "CRAWL_NOT_FOUND", - message: content ? "Crawl content not found." : "Crawl not found.", - likely_cause: content - ? `${cause}. The page was not fetched (yet), its fetch failed, or the crawl ran without --html.` - : `${cause}. The id may be wrong or the crawl is not owned by this account.`, - next_action: content - ? "Read the result's content_status with `zenrows crawl results `; only `fetched` results have content." - : "Check the crawl id, or list your crawls with `zenrows crawl list`.", + message: "Crawl not found.", + likely_cause: `${cause}. The id may be wrong or the crawl is not owned by this account.`, + next_action: "Check the crawl id, or list your crawls with `zenrows crawl list`.", suggested_commands: ["zenrows crawl list"], }); } if (status === 429) { - const wait = retryAfter && /^\d+$/.test(retryAfter.trim()) ? ` Retry after ${retryAfter.trim()}s.` : ""; return new ToolkitError({ - code: "CRAWL_QUOTA_EXCEEDED", - message: "Too many crawls running.", - likely_cause: `${cause}. The account has too many crawls running.${wait}`, - next_action: - "Retry after Retry-After, or stop one of your crawls with `zenrows crawl stop ` first. Nothing was created.", + code: "CRAWL_TOO_MANY_CRAWLS", + message: "The account has reached its limit of active jobs.", + likely_cause: `${cause}. The account has reached its limit of active jobs (3 by default), shared with its Batch jobs.`, + next_action: `Nothing was created. ${retryAfter !== undefined ? `Retry after ${retryAfter} seconds` : "Retry later"}, or stop one of your crawls (\`zenrows crawl stop \`) or Batch jobs first.`, suggested_commands: ["zenrows crawl list"], }); } if (status === 402 && isKeyCapReached(serverCode)) { - return keyCapReached(`${method} ${path}`, { status: 402, detail: problem.detail || problem.title || undefined }); + return keyCapReached(request, { status: 402, detail: problem.detail || problem.title || undefined }); } if (status === 402) { const acct = readAccount(); - return quotaExhausted(`${method} ${path}`, acct?.unclaimed ? acct.claimUrl : undefined, { + return quotaExhausted(request, acct?.unclaimed ? acct.claimUrl : undefined, { status: 402, detail: problem.detail || problem.title || undefined, }); } - if (status === 400 || status === 409 || status === 422) { + if (status === 409) { return new ToolkitError({ - code: "CRAWL_INVALID", + code: "CRAWL_REQUEST_IN_FLIGHT", + message: "A request with the same Idempotency-Key is still in flight.", + likely_cause: `${cause}.`, + next_action: "Retry the same request once the first one has finished.", + }); + } + if (status === 400 || status === 422) { + return new ToolkitError({ + code: "CRAWL_INVALID_REQUEST", message: `Crawl rejected the request (HTTP ${status}).`, likely_cause: `${cause}.`, next_action: - status === 409 - ? "A request with the same Idempotency-Key is still in flight; retry once it has finished." + serverCode === "idempotency_key_reused" + ? "This Idempotency-Key was already used for a different request. Send a new key, or no key. Do not retry as is." : "Fix the reported parameter and retry. See `zenrows crawl --help` for the accepted flags and ranges.", }); } @@ -333,6 +363,7 @@ export function createCrawl(params: CreateCrawlParams, opts: CallOpts & { idempo export function getCrawl(id: string, opts: CallOpts & { cursor?: string; limit?: number }): Promise { return crawlJson("GET", `/crawls/${encodeURIComponent(id)}`, { apiKey: opts.apiKey, + crawlId: id, query: { cursor: opts.cursor, limit: opts.limit }, fetchImpl: opts.fetchImpl, timeoutMs: opts.timeoutMs, @@ -353,6 +384,7 @@ export function listCrawls(opts: CallOpts & { cursor?: string; limit?: number }) export function stopCrawl(id: string, opts: CallOpts): Promise { return crawlJson("POST", `/crawls/${encodeURIComponent(id)}/stop`, { apiKey: opts.apiKey, + crawlId: id, fetchImpl: opts.fetchImpl, timeoutMs: opts.timeoutMs, }); @@ -366,6 +398,7 @@ export async function getCrawlContent( ): Promise<{ contentType: string; body: string }> { const res = await crawlFetch("GET", `/crawls/${encodeURIComponent(id)}/contents/${encodeURIComponent(contentId)}`, { apiKey: opts.apiKey, + crawlId: id, accept: "text/html, application/json;q=0.9, application/problem+json;q=0.8", fetchImpl: opts.fetchImpl, timeoutMs: opts.timeoutMs, @@ -380,6 +413,7 @@ export async function getCrawlContent( export async function downloadCrawl(id: string, opts: CallOpts): Promise<{ crawlStatus?: string; ndjson: string }> { const res = await crawlFetch("GET", `/crawls/${encodeURIComponent(id)}/download`, { apiKey: opts.apiKey, + crawlId: id, accept: "application/x-ndjson, application/problem+json;q=0.8", fetchImpl: opts.fetchImpl, timeoutMs: opts.timeoutMs ?? 300_000, @@ -398,39 +432,30 @@ export function contentIdOf(result: CrawlResult): string | undefined { } /** - * Read every result, following `next_cursor` until it is null. Only call this - * on an ended crawl: while a crawl runs `next_cursor` is never null, so this - * refuses a `running` crawl instead of looping forever (wait first). + * Read every kept result, following `next_cursor`. While a crawl runs its + * `next_cursor` is never null, so the read also ends at the first empty page; + * `partial` is true when the crawl was still running. */ export async function listAllResults( id: string, opts: CallOpts & { pageSize?: number }, -): Promise<{ crawl: CrawlWithResults; results: CrawlResult[] }> { +): Promise<{ crawl: CrawlWithResults; results: CrawlResult[]; partial: boolean }> { const results: CrawlResult[] = []; let cursor: string | undefined; - let first: CrawlWithResults | undefined; for (;;) { const page = await getCrawl(id, { ...opts, cursor, limit: opts.pageSize }); - if (!isTerminal(page.status)) { - throw new ToolkitError({ - code: "INVALID_USAGE", - message: `Crawl ${id} is still running.`, - likely_cause: "Reading all results needs an ended crawl: while it runs, more results keep arriving.", - next_action: "Wait for it first (`zenrows crawl wait `), or stop it with `zenrows crawl stop `.", - suggested_commands: [`zenrows crawl wait ${id}`, `zenrows crawl status ${id}`], - }); + const rows = page.results ?? []; + results.push(...rows); + if (!rows.length || !page.next_cursor || page.next_cursor === cursor) { + return { crawl: page, results, partial: !isTerminal(page.status) }; } - first ??= page; - results.push(...(page.results ?? [])); - if (page.next_cursor === null || page.next_cursor === undefined || page.next_cursor === cursor) break; cursor = page.next_cursor; } - return { crawl: first!, results }; } /** * Poll `GET /crawls/{id}?limit=1` until the status leaves `running`, backing off - * 2s → ×1.5 → capped at 15s. On timeout throws CRAWL_TIMEOUT + * 2s → ×1.5 → capped at 15s. On timeout throws CRAWL_WAIT_TIMEOUT * and leaves the crawl running. `sleepImpl` is injectable for tests. */ export async function waitForCrawl( @@ -446,11 +471,12 @@ export async function waitForCrawl( if (isTerminal(crawl.status)) return crawl; if (Date.now() + delay > deadline) { throw new ToolkitError({ - code: "CRAWL_TIMEOUT", + code: "CRAWL_WAIT_TIMEOUT", message: `Timed out waiting for crawl ${id} to finish.`, likely_cause: `The crawl was still running after ${Math.round(timeoutMs / 1000)}s. It keeps running; it was not stopped.`, next_action: "Re-check progress with `zenrows crawl status `, wait again with a larger --timeout, or stop it.", suggested_commands: [`zenrows crawl status ${id}`, `zenrows crawl wait ${id}`, `zenrows crawl stop ${id}`], + crawl_id: id, }); } await sleep(delay); diff --git a/src/core/errors.ts b/src/core/errors.ts index 95d3a22..0c2c60f 100644 --- a/src/core/errors.ts +++ b/src/core/errors.ts @@ -36,11 +36,13 @@ export type ErrorCode = | "BATCH_NOT_FOUND" | "BATCH_FAILED" | "CRAWL_NOT_ENABLED" - | "CRAWL_QUOTA_EXCEEDED" + | "CRAWL_TOO_MANY_CRAWLS" | "CRAWL_NOT_FOUND" - | "CRAWL_INVALID" + | "CRAWL_CONTENT_NOT_FOUND" + | "CRAWL_INVALID_REQUEST" + | "CRAWL_REQUEST_IN_FLIGHT" | "CRAWL_FAILED" - | "CRAWL_TIMEOUT" + | "CRAWL_WAIT_TIMEOUT" | "PLUGIN_UNSUPPORTED" | "MCP_CLIENT_UNSUPPORTED" | "ASSET_NOT_FOUND" @@ -54,6 +56,12 @@ export interface ToolkitErrorShape { likely_cause: string; next_action: string; suggested_commands?: string[]; + /** The API's own `code`, when the error came from an API answer that had one. */ + server_code?: string; + /** The crawl the error is about, once it exists. */ + crawl_id?: string; + /** Seconds to wait before a retry (the `Retry-After` header). */ + retry_after?: number; } export class ToolkitError extends Error { @@ -61,6 +69,9 @@ export class ToolkitError extends Error { readonly likely_cause: string; readonly next_action: string; readonly suggested_commands: string[]; + readonly server_code?: string; + readonly crawl_id?: string; + readonly retry_after?: number; constructor(shape: ToolkitErrorShape) { super(shape.message); @@ -69,6 +80,9 @@ export class ToolkitError extends Error { this.likely_cause = shape.likely_cause; this.next_action = shape.next_action; this.suggested_commands = shape.suggested_commands ?? []; + this.server_code = shape.server_code; + this.crawl_id = shape.crawl_id; + this.retry_after = shape.retry_after; } toJSON(): ToolkitErrorShape { @@ -78,6 +92,9 @@ export class ToolkitError extends Error { likely_cause: this.likely_cause, next_action: this.next_action, suggested_commands: this.suggested_commands, + ...(this.server_code ? { server_code: this.server_code } : {}), + ...(this.crawl_id ? { crawl_id: this.crawl_id } : {}), + ...(this.retry_after !== undefined ? { retry_after: this.retry_after } : {}), }; } } diff --git a/tests/capabilities.test.ts b/tests/capabilities.test.ts index 73c7da6..3485169 100644 --- a/tests/capabilities.test.ts +++ b/tests/capabilities.test.ts @@ -11,7 +11,7 @@ test("capability matrix loads with the honest classifications", () => { assert.equal(caps.mcp?.status, "available"); // Batch is a real product in open beta. assert.equal(caps.batch?.status, "beta"); - assert.equal(caps.crawl?.status, "available"); + assert.equal(caps.crawl?.status, "beta"); }); test("isUsable true for available + open beta, false for absent capabilities", () => { diff --git a/tests/crawl-api.test.ts b/tests/crawl-api.test.ts index 2dc80b3..0ffab09 100644 --- a/tests/crawl-api.test.ts +++ b/tests/crawl-api.test.ts @@ -154,7 +154,19 @@ test("403 REQS008 maps to CRAWL_NOT_ENABLED with the API's wording", async () => await assert.rejects( () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: impl }), (e: unknown) => - e instanceof ToolkitError && e.code === "CRAWL_NOT_ENABLED" && e.message === "Crawl is not enabled for this account.", + e instanceof ToolkitError && + e.code === "CRAWL_NOT_ENABLED" && + e.message === "Crawl is not enabled for this account." && + e.server_code === "REQS008" && + e.crawl_id === undefined, + ); +}); + +test("any other 403 maps to CRAWL_FAILED", async () => { + const { impl } = problem(403, "REQS001"); + await assert.rejects( + () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: impl }), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_FAILED" && e.server_code === "REQS001", ); }); @@ -162,37 +174,71 @@ test("404 crawl_not_found maps to CRAWL_NOT_FOUND", async () => { const { impl } = problem(404, "crawl_not_found"); await assert.rejects( () => getCrawl("c_nope", { apiKey: "k", fetchImpl: impl }), - (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_NOT_FOUND" && /crawl_not_found/.test(e.likely_cause), + (e: unknown) => + e instanceof ToolkitError && + e.code === "CRAWL_NOT_FOUND" && + e.server_code === "crawl_not_found" && + e.crawl_id === "c_nope", ); }); -test("404 content_not_found maps to CRAWL_NOT_FOUND with content guidance", async () => { +test("404 content_not_found maps to CRAWL_CONTENT_NOT_FOUND with content guidance", async () => { const { impl } = problem(404, "content_not_found"); await assert.rejects( () => getCrawlContent("c_1", "ct_x", { apiKey: "k", fetchImpl: impl }), - (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_NOT_FOUND" && /content_status/.test(e.next_action), + (e: unknown) => + e instanceof ToolkitError && e.code === "CRAWL_CONTENT_NOT_FOUND" && /content_status/.test(e.next_action) && e.crawl_id === "c_1", ); }); -test("422 invalid_parameter and 400 unknown_parameter map to CRAWL_INVALID", async () => { +test("400 and 422 map to CRAWL_INVALID_REQUEST with the server code", async () => { for (const [status, code] of [ [422, "invalid_parameter"], [400, "unknown_parameter"], - [409, "idempotency_request_in_flight"], ] as const) { const { impl } = problem(status, code, "'depth' is out of range."); await assert.rejects( () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: impl }), - (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_INVALID" && e.likely_cause.includes(code), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_INVALID_REQUEST" && e.server_code === code, ); } }); -test("429 too_many_crawls maps to CRAWL_QUOTA_EXCEEDED and carries Retry-After", async () => { - const { impl } = problem(429, "too_many_crawls", "Too many crawls.", { "retry-after": "30" }); +test("422 idempotency_key_reused says to send a new key, not to retry as is", async () => { + const { impl } = problem(422, "idempotency_key_reused"); await assert.rejects( - () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: impl }), - (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_QUOTA_EXCEEDED" && /Retry after 30s/.test(e.likely_cause), + () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: impl, idempotencyKey: "k1" }), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_INVALID_REQUEST" && /new key, or no key/.test(e.next_action), + ); +}); + +test("409 maps to CRAWL_REQUEST_IN_FLIGHT", async () => { + const { impl } = problem(409, "idempotency_request_in_flight"); + await assert.rejects( + () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: impl, idempotencyKey: "k1" }), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_REQUEST_IN_FLIGHT" && /finished/.test(e.next_action), + ); +}); + +test("429 too_many_crawls maps to CRAWL_TOO_MANY_CRAWLS with retry_after and no retry", async () => { + const { impl, calls } = problem(429, "too_many_crawls", "Too many crawls.", { "retry-after": "30" }); + await assert.rejects( + () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: impl, idempotencyKey: "k1" }), + (e: unknown) => + e instanceof ToolkitError && + e.code === "CRAWL_TOO_MANY_CRAWLS" && + e.retry_after === 30 && + e.server_code === "too_many_crawls" && + /shared with its Batch jobs/.test(e.likely_cause), + ); + assert.equal(calls.length, 1); +}); + +test("an error body without a code leaves server_code unset", async () => { + const { impl } = stubFetch(500, "upstream exploded"); + await assert.rejects( + () => getCrawl("c_1", { apiKey: "k", fetchImpl: impl }), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_FAILED" && !("server_code" in e.toJSON()), ); }); @@ -207,7 +253,10 @@ test("a transport failure maps to BACKEND_UNAVAILABLE", async () => { const impl = (async () => { throw new TypeError("fetch failed"); }) as unknown as typeof fetch; - await assert.rejects(() => getCrawl("c", { apiKey: "k", fetchImpl: impl }), (e: unknown) => e instanceof ToolkitError && e.code === "BACKEND_UNAVAILABLE"); + await assert.rejects( + () => getCrawl("c", { apiKey: "k", fetchImpl: impl }), + (e: unknown) => e instanceof ToolkitError && e.code === "BACKEND_UNAVAILABLE" && e.crawl_id === "c", + ); }); /** A stub that serves `pages` in order, recording the cursor each call sent. */ @@ -231,19 +280,22 @@ test("listAllResults follows next_cursor and stops on null", async () => { { status: "completed", results: [{ url: "https://example.com/product/2" }], next_cursor: "c2" }, { status: "completed", results: [{ url: "https://example.com/product/3" }], next_cursor: null }, ]); - const { results, crawl } = await listAllResults("c_1", { apiKey: "k", fetchImpl: impl }); + const { results, crawl, partial } = await listAllResults("c_1", { apiKey: "k", fetchImpl: impl }); + assert.equal(partial, false); assert.deepEqual(results.map((r) => r.url), ["https://example.com/product/1", "https://example.com/product/2", "https://example.com/product/3"]); assert.deepEqual(cursors, [null, "c1", "c2"]); assert.equal(crawl.status, "completed"); }); -test("listAllResults refuses a running crawl instead of polling forever", async () => { - const { impl, calls } = pagedFetch([{ status: "running", results: [], next_cursor: "c1" }]); - await assert.rejects( - () => listAllResults("c_1", { apiKey: "k", fetchImpl: impl }), - (e: unknown) => e instanceof ToolkitError && e.code === "INVALID_USAGE" && /still running/.test(e.message), - ); - assert.equal(calls(), 1); +test("listAllResults returns what a running crawl kept, ending at the first empty page", async () => { + const { impl, calls } = pagedFetch([ + { status: "running", results: [{ url: "https://example.com/product/1" }], next_cursor: "c1" }, + { status: "running", results: [], next_cursor: "c1" }, + ]); + const { results, partial } = await listAllResults("c_1", { apiKey: "k", fetchImpl: impl }); + assert.deepEqual(results.map((r) => r.url), ["https://example.com/product/1"]); + assert.equal(partial, true); + assert.equal(calls(), 2); }); test("waitForCrawl polls with limit=1 until the status leaves running, backing off", async () => { @@ -269,7 +321,7 @@ test("waitForCrawl treats failed and stopped as terminal", async () => { } }); -test("waitForCrawl times out with CRAWL_TIMEOUT and does not stop the crawl", async () => { +test("waitForCrawl times out with CRAWL_WAIT_TIMEOUT, carries crawl_id and does not stop the crawl", async () => { const methods: string[] = []; const impl = (async (_url: string, init?: RequestInit) => { methods.push(init?.method ?? "GET"); @@ -280,7 +332,8 @@ test("waitForCrawl times out with CRAWL_TIMEOUT and does not stop the crawl", as }) as unknown as typeof fetch; await assert.rejects( () => waitForCrawl("c_1", { apiKey: "k", fetchImpl: impl, timeoutMs: 1, sleepImpl: async () => {} }), - (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_TIMEOUT" && /not stopped/.test(e.likely_cause), + (e: unknown) => + e instanceof ToolkitError && e.code === "CRAWL_WAIT_TIMEOUT" && e.crawl_id === "c_1" && /not stopped/.test(e.likely_cause), ); assert.ok(methods.every((m) => m === "GET"), "no stop call on timeout"); }); diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index 768fb98..631bf08 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -1,4 +1,4 @@ -import { test } from "node:test"; +import { mock, test } from "node:test"; import assert from "node:assert/strict"; import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; @@ -84,7 +84,7 @@ test("crawl start rejects flags it does not declare, before any network call", a await withCrawlWorkspace( () => json(crawlBody, 202), async (calls) => { - for (const flag of ["--pagination", "--discovery", "--json-output"]) { + for (const flag of ["--pagination", "--discovery", "--json-output", "--follow"]) { const { code, out } = await run(["start", "https://example.com/products/", "--depth", "1", flag]); assert.equal(code, 1, flag); assert.equal(out.error.code, "UNKNOWN_FLAG", flag); @@ -163,7 +163,7 @@ test("crawl start honors the domain policy and the page cap before any network c ); }); -test("crawl start --follow waits for the end and exits 1 with CRAWL_FAILED on a failed crawl", async () => { +test("crawl start --wait waits for the end and exits 1 with CRAWL_FAILED on a failed crawl", async () => { await withCrawlWorkspace( (url, init) => init?.method === "POST" @@ -176,12 +176,14 @@ test("crawl start --follow waits for the end and exits 1 with CRAWL_FAILED on a next_cursor: null, }), async (calls) => { - const { code, out } = await run(["start", "https://example.com/products/", "--depth", "1", "--follow"]); + const { code, out } = await run(["start", "https://example.com/products/", "--depth", "1", "--wait"]); assert.equal(code, 1); assert.equal(out.ok, false); assert.equal(out.status, "failed"); assert.equal(out.error.code, "CRAWL_FAILED"); assert.match(out.error.likely_cause, /seed_unreachable/); + assert.equal(out.error.server_code, "seed_unreachable"); + assert.equal(out.error.crawl_id, "c_1"); assert.equal(calls[1]!.url.searchParams.get("limit"), "1"); assert.equal(out.crawl.results, undefined, "status output carries no partial result page"); }, @@ -246,13 +248,33 @@ test("crawl results reads every page of an ended crawl and writes JSONL with --o assert.equal(out.count, 2); assert.deepEqual(out.results.map((r: { url: string }) => r.url), ["https://example.com/product/1", "https://example.com/product/2"]); + assert.equal(out.partial, false); + const file = join(process.cwd(), "urls.jsonl"); - await captureOut(() => crawl.run(["results", "c_1", "--out", file], ctx)); + const written = await run(["results", "c_1", "--out", file]); + assert.deepEqual(written.out, { ok: true, crawlId: "c_1", status: "completed", count: 2, file, partial: false }); assert.equal(readFileSync(file, "utf8"), '{"url":"https://example.com/product/1"}\n{"url":"https://example.com/product/2"}\n'); }, ); }); +test("crawl results on a running crawl returns what is kept with partial: true", async () => { + const pages: Record = { + "": { ...crawlBody, results: [{ url: "https://example.com/product/1" }], next_cursor: "p2" }, + p2: { ...crawlBody, results: [], next_cursor: "p2" }, + }; + await withCrawlWorkspace( + (url) => json(pages[url.searchParams.get("cursor") ?? ""]), + async () => { + const { code, out } = await run(["results", "c_1"]); + assert.equal(code, 0); + assert.equal(out.status, "running"); + assert.equal(out.partial, true); + assert.equal(out.count, 1); + }, + ); +}); + test("crawl results --cursor reads one page and returns next_cursor while running", async () => { await withCrawlWorkspace( () => json({ ...crawlBody, results: [{ url: "https://example.com/product/3" }], next_cursor: "p4" }), @@ -317,6 +339,61 @@ test("crawl list and stop call the right endpoints", async () => { ); }); +/** Capture stderr (human output) for the duration of `fn`. */ +async function captureErr(fn: () => unknown): Promise { + const orig = process.stderr.write.bind(process.stderr); + let buf = ""; + process.stderr.write = ((s: string | Uint8Array) => { + buf += typeof s === "string" ? s : Buffer.from(s).toString(); + return true; + }) as typeof process.stderr.write; + try { + await fn(); + } finally { + process.stderr.write = orig; + } + return buf; +} + +test("crawl stop on an ended crawl prints its real status, not Stopped", async () => { + await withCrawlWorkspace( + () => json({ crawl_id: "c_1", status: "completed", stop_reason: "max_items" }), + async () => { + let code = -1; + const err = await captureErr(async () => { + code = await crawl.run(["stop", "c_1"], { json: false, yes: false }); + }); + assert.equal(code, 0); + assert.match(err, /already ended · status: completed/); + assert.doesNotMatch(err, /Stopped crawl/); + }, + ); +}); + +test("Ctrl-C during crawl wait says the crawl keeps running and exits 130", async () => { + let interrupted = false; + const exit = mock.method(process, "exit", () => { + interrupted = true; + }); + try { + await withCrawlWorkspace( + () => { + if (!interrupted) setImmediate(() => process.emit("SIGINT")); + return json({ ...crawlBody, status: interrupted ? "completed" : "running", results: [], next_cursor: null }); + }, + async () => { + const err = await captureErr(() => crawl.run(["wait", "c_1"], { json: false, yes: false })); + assert.deepEqual(exit.mock.calls[0]?.arguments, [130]); + assert.match(err, /c_1 is still running/); + assert.match(err, /zenrows crawl wait c_1/); + assert.equal(process.listenerCount("SIGINT"), 0, "the wait removes its SIGINT handler"); + }, + ); + } finally { + exit.mock.restore(); + } +}); + test("unknown crawl subcommand is INVALID_USAGE", async () => { const { code, out } = await run(["discover"]); assert.equal(code, 1); diff --git a/tests/e2e/crawl.e2e.test.ts b/tests/e2e/crawl.e2e.test.ts index 14d5a72..dddd43c 100644 --- a/tests/e2e/crawl.e2e.test.ts +++ b/tests/e2e/crawl.e2e.test.ts @@ -71,7 +71,7 @@ test("zenrows crawl end to end against a live API", { skip, timeout: 20 * 60_000 const dir = mkdtempSync(join(tmpdir(), "zr-crawl-e2e-")); try { // 1. Start a small crawl and wait for it. Other runs may hold the account's - // active-crawl slots: on CRAWL_QUOTA_EXCEEDED wait Retry-After and try again. + // active-job slots: on CRAWL_TOO_MANY_CRAWLS wait retry_after and try again. const deadline = Date.now() + QUOTA_RETRY_MS; let start: CliResult; for (;;) { @@ -79,10 +79,10 @@ test("zenrows crawl end to end against a live API", { skip, timeout: 20 * 60_000 "crawl", "start", START_URL!, "--depth", "1", "--max-items", "3", "--max-pages", "5", ...(INCLUDE ? ["--include", INCLUDE] : []), - "--html", "--follow", "--timeout", "600000", + "--html", "--wait", "--timeout", "600000", ]); - if (start.out.error?.code !== "CRAWL_QUOTA_EXCEEDED" || Date.now() > deadline) break; - const wait = Number(/Retry after (\d+)s/.exec(start.out.error.likely_cause)?.[1] ?? 30); + if (start.out.error?.code !== "CRAWL_TOO_MANY_CRAWLS" || Date.now() > deadline) break; + const wait = Number(start.out.error.retry_after ?? 30); log(`too_many_crawls: retrying start in ${wait}s`); await new Promise((r) => setTimeout(r, wait * 1000)); } diff --git a/tests/registry.test.ts b/tests/registry.test.ts index 3ca7fb7..145b479 100644 --- a/tests/registry.test.ts +++ b/tests/registry.test.ts @@ -52,5 +52,5 @@ test("assetRunnable reflects backend capability status", () => { const extractSkill = loadRegistry("skill").find((s) => s.name === "extract")!; assert.equal(assetRunnable(extractSkill), true); // needs extract (open beta = usable) const crawlSkill = loadRegistry("skill").find((s) => s.name === "crawl")!; - assert.equal(assetRunnable(crawlSkill), true); // needs crawl (available) + assert.equal(assetRunnable(crawlSkill), true); // needs crawl (open beta = usable) }); From 76b063a05bcdee19a8b33ad5be956bfb8689cc2c Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Fri, 9 Oct 2026 16:47:13 +0200 Subject: [PATCH 03/19] fix(crawl): wait with --follow, as batch does fetch and extract use --wait for a different meaning, so a boolean --wait on crawl start can confuse people. crawl start now takes --follow only. The crawl wait subcommand stays. --- README.md | 4 ++-- skills/crawl/SKILL.md | 6 +++--- skills/zenrows/SKILL.md | 2 +- src/cli/commands/crawl.ts | 10 +++++----- tests/crawl-command.test.ts | 6 +++--- tests/e2e/crawl.e2e.test.ts | 2 +- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 64fc422..e748cbc 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ the start URL's registrable domain (subdomains count). Without `--html` it retur ```bash zenrows crawl start https://example.com/products/ --depth 1 \ - --max-items 20 --include /product/ --wait # start and wait for the end + --max-items 20 --include /product/ --follow # start and wait for the end zenrows crawl results --out urls.jsonl # every kept URL (paginated) zenrows crawl start --depth 2 --html # also fetch each kept page's HTML zenrows crawl content # one page's HTML @@ -185,7 +185,7 @@ is one fetch. The local policy `max_pages_per_run` caps `--max-pages` (default 1000). `--html` uses up the page budget: each kept page is one more fetch, and `--max-pages` counts it. `--include` / `--exclude` are repeatable substring filters on the URL. `results` on a running crawl returns the URLs kept so far, -with `partial: true`. `--wait` gives up after `--timeout` (default 600000 ms) +with `partial: true`. `--follow` gives up after `--timeout` (default 600000 ms) with `CRAWL_WAIT_TIMEOUT`; the crawl keeps running. Ctrl-C stops the wait, not the crawl, and exits 130; resume with `zenrows crawl wait `. diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index 7008f59..e6aac46 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -22,7 +22,7 @@ do not retry it. ``` zenrows crawl start https://example.com/products/ --depth 1 \ - --max-items 20 --max-pages 25 --include /product/ --wait --json + --max-items 20 --max-pages 25 --include /product/ --follow --json zenrows crawl results --json # every kept URL zenrows crawl results --out urls.jsonl # same, as JSONL ``` @@ -35,7 +35,7 @@ zenrows crawl results --out urls.jsonl # same, as JSONL above it the CLI refuses with `POLICY_LIMIT_EXCEEDED` before any request. - `--include` / `--exclude` are substring matches on the normalized URL and repeat: `--include /product/ --exclude ?add-to-cart`. -- `start` returns at once. `--wait` (or `zenrows crawl wait `) polls +- `start` returns at once. `--follow` (or `zenrows crawl wait `) polls until the crawl ends (default timeout 600000 ms). On `--timeout` the crawl keeps running (it is not stopped) and you get `CRAWL_WAIT_TIMEOUT` with `crawl_id`. Ctrl-C also stops the wait, not the crawl, and exits 130; @@ -46,7 +46,7 @@ zenrows crawl results --out urls.jsonl # same, as JSONL ## Page HTML ``` -zenrows crawl start --depth 1 --html --max-pages 20 --wait +zenrows crawl start --depth 1 --html --max-pages 20 --follow zenrows crawl results --json # fetched results carry content_url zenrows crawl content # one page's HTML (or pass the content_url) zenrows crawl results --download --out pages.jsonl # every URL + HTML, NDJSON diff --git a/skills/zenrows/SKILL.md b/skills/zenrows/SKILL.md index 1f885d4..595c9c9 100644 --- a/skills/zenrows/SKILL.md +++ b/skills/zenrows/SKILL.md @@ -35,7 +35,7 @@ If the user has a known URL and wants structured data: → Use Extract. (zenrows extract | --autoparse | --css) If the user has a start page (listing, index, section) but not the item URLs: - → Use Crawl. (zenrows crawl start --depth 1 --wait, + → Use Crawl. (zenrows crawl start --depth 1 --follow, then `zenrows crawl results `) If the user has many URLs (bulk): diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index 5579c75..1c3a7cc 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -3,7 +3,7 @@ * * Give Crawl one start URL and read back the URLs it finds behind it, optionally * with each page's HTML. A crawl is an async job: `start` returns at once - * (or polls with `--wait`), then `status` / `wait` / `results` / `content` + * (or polls with `--follow`), then `status` / `wait` / `results` / `content` * read it and `stop` ends it. An account without Crawl access gets 403 REQS008 * → CRAWL_NOT_ENABLED. */ @@ -54,8 +54,8 @@ export const crawl: Command = { " --exclude drop URLs containing this substring (repeatable)", " --html also fetch each kept URL's page HTML (read with `content`);", " each kept page is one more fetch counted by --max-pages", - " --wait poll until the crawl ends (Ctrl-C stops the wait, not the crawl)", - " --timeout with --wait: give up waiting after ms (default 600000)", + " --follow poll until the crawl ends (Ctrl-C stops the wait, not the crawl)", + " --timeout with --follow: give up waiting after ms (default 600000)", " --idempotency-key make a retried start create no second crawl", " --no-signup do not auto-create a Free plan account if no key exists", " status show status, coverage, stop reason / error", @@ -114,14 +114,14 @@ async function startCmd(rest: string[], ctx: RunContext): Promise { include: { type: "string", multiple: true }, exclude: { type: "string", multiple: true }, html: { type: "boolean" }, - wait: { type: "boolean" }, + follow: { type: "boolean" }, timeout: { type: "string" }, "idempotency-key": { type: "string" }, "no-signup": { type: "boolean" }, json: { type: "boolean" }, }); const json = ctx.json || values.json === true; - const wait = values.wait === true; + const wait = values.follow === true; const url = positionals[0]; if (!url) { throw new ToolkitError({ diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index 631bf08..cff1d62 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -84,7 +84,7 @@ test("crawl start rejects flags it does not declare, before any network call", a await withCrawlWorkspace( () => json(crawlBody, 202), async (calls) => { - for (const flag of ["--pagination", "--discovery", "--json-output", "--follow"]) { + for (const flag of ["--pagination", "--discovery", "--json-output", "--wait"]) { const { code, out } = await run(["start", "https://example.com/products/", "--depth", "1", flag]); assert.equal(code, 1, flag); assert.equal(out.error.code, "UNKNOWN_FLAG", flag); @@ -163,7 +163,7 @@ test("crawl start honors the domain policy and the page cap before any network c ); }); -test("crawl start --wait waits for the end and exits 1 with CRAWL_FAILED on a failed crawl", async () => { +test("crawl start --follow waits for the end and exits 1 with CRAWL_FAILED on a failed crawl", async () => { await withCrawlWorkspace( (url, init) => init?.method === "POST" @@ -176,7 +176,7 @@ test("crawl start --wait waits for the end and exits 1 with CRAWL_FAILED on a fa next_cursor: null, }), async (calls) => { - const { code, out } = await run(["start", "https://example.com/products/", "--depth", "1", "--wait"]); + const { code, out } = await run(["start", "https://example.com/products/", "--depth", "1", "--follow"]); assert.equal(code, 1); assert.equal(out.ok, false); assert.equal(out.status, "failed"); diff --git a/tests/e2e/crawl.e2e.test.ts b/tests/e2e/crawl.e2e.test.ts index dddd43c..c5fe6a5 100644 --- a/tests/e2e/crawl.e2e.test.ts +++ b/tests/e2e/crawl.e2e.test.ts @@ -79,7 +79,7 @@ test("zenrows crawl end to end against a live API", { skip, timeout: 20 * 60_000 "crawl", "start", START_URL!, "--depth", "1", "--max-items", "3", "--max-pages", "5", ...(INCLUDE ? ["--include", INCLUDE] : []), - "--html", "--wait", "--timeout", "600000", + "--html", "--follow", "--timeout", "600000", ]); if (start.out.error?.code !== "CRAWL_TOO_MANY_CRAWLS" || Date.now() > deadline) break; const wait = Number(start.out.error.retry_after ?? 30); From efb7f26adbcf198de0cf7a8f6ed78c0196f984fc Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Fri, 9 Oct 2026 17:37:13 +0200 Subject: [PATCH 04/19] fix(crawl): match the shared Crawl client interface Rename the subcommands to the shared operation names: create, get, list, results, content, download, stop and wait. start and status are gone, and results --download is now download. Rename the flags to the API names: --include-pattern, --exclude-pattern and --output-format html. get and list take --cursor and --limit. results takes --limit as the page size and follows every page. create takes --follow to wait for the end. --timeout is in seconds, default 600. A wait that runs out is not an error. It exits 0, prints the crawl with status running and the hint to wait again. CRAWL_WAIT_TIMEOUT is gone. A 402 is now CRAWL_QUOTA_EXCEEDED, or CRAWL_KEY_CAP_REACHED for AUTH014. The messages keep the claim, top-up and key-cap advice. download reports status and partial. results reports partial when the crawl still runs or a cursor remains. --- README.md | 62 +++++---- docs/capabilities.md | 2 +- docs/contributing.md | 4 +- registry/capabilities.json | 2 +- registry/skills.json | 2 +- skills/crawl/SKILL.md | 57 ++++---- skills/trace-debug/SKILL.md | 5 +- skills/zenrows/SKILL.md | 2 +- src/cli/commands/crawl.ts | 262 +++++++++++++++++++++--------------- src/core/crawl-api.ts | 48 +++---- src/core/errors.ts | 3 +- tests/crawl-api.test.ts | 38 ++++-- tests/crawl-command.test.ts | 111 +++++++++++---- tests/e2e/crawl.e2e.test.ts | 24 ++-- 14 files changed, 379 insertions(+), 243 deletions(-) diff --git a/README.md b/README.md index e748cbc..7902411 100644 --- a/README.md +++ b/README.md @@ -163,40 +163,52 @@ Zenrows **Crawl** is in beta. Crawl (`https://api.zenrows.com/v1/crawls`) takes one start URL and returns the URLs it finds behind it, optionally with each page's HTML. Use it when you know the listing or section but not the item URLs. Crawl follows the links on each page up to `--depth` hops and stays on -the start URL's registrable domain (subdomains count). Without `--html` it returns URLs only. An account without Crawl access gets +the start URL's registrable domain (subdomains count). Without +`--output-format html` it returns URLs only. An account without Crawl access gets `CRAWL_NOT_ENABLED` ("Crawl is not enabled for this account", HTTP 403 `REQS008`). ```bash -zenrows crawl start https://example.com/products/ --depth 1 \ - --max-items 20 --include /product/ --follow # start and wait for the end -zenrows crawl results --out urls.jsonl # every kept URL (paginated) -zenrows crawl start --depth 2 --html # also fetch each kept page's HTML -zenrows crawl content # one page's HTML -zenrows crawl results --download # NDJSON export (URLs + HTML) → .jsonl -zenrows crawl status # status, coverage, stop reason -zenrows crawl list # your crawls, newest first -zenrows crawl stop # stop a running crawl +zenrows crawl create https://example.com/products/ --depth 1 \ + --max-items 20 --include-pattern /product/ --follow # create and wait for the end +zenrows crawl results --out urls.jsonl # every kept URL (paginated) +zenrows crawl create --depth 2 --output-format html # also fetch each kept page's HTML +zenrows crawl content # one page's HTML +zenrows crawl download # NDJSON export (URLs + HTML) → .jsonl +zenrows crawl get # status, coverage, stop reason, one page of results +zenrows crawl wait # wait for the end +zenrows crawl list # your crawls, newest first +zenrows crawl stop # stop a running crawl ``` `--depth` (link hops) is required; `--max-items` and `--max-pages` cap the crawl (API default 10 each), and `--max-pages` bounds the cost, since each page is one fetch. The local policy `max_pages_per_run` caps `--max-pages` (default -1000). `--html` uses up the page budget: each kept page is one more fetch, and -`--max-pages` counts it. `--include` / `--exclude` are repeatable substring -filters on the URL. `results` on a running crawl returns the URLs kept so far, -with `partial: true`. `--follow` gives up after `--timeout` (default 600000 ms) -with `CRAWL_WAIT_TIMEOUT`; the crawl keeps running. Ctrl-C stops the wait, not -the crawl, and exits 130; resume with `zenrows crawl wait `. - -Errors carry the API's code as `server_code`, and `crawl_id` once the crawl -exists. When the account has reached its limit of active jobs (3 by default), -shared with its Batch jobs, a start gets `CRAWL_TOO_MANY_CRAWLS` and creates -nothing; retry after `retry_after` seconds. Other codes: `CRAWL_INVALID_REQUEST` -(400/422, do not retry as is), `CRAWL_REQUEST_IN_FLIGHT` (409, retry once the -first request ends), `CRAWL_NOT_FOUND`, `CRAWL_CONTENT_NOT_FOUND` and -`CRAWL_FAILED`. Set `ZENROWS_CRAWL_API_BASE` to point the CLI at another -deployment. +1000). `--output-format html` uses up the page budget: each kept page is one +more fetch, and `--max-pages` counts it. `--include-pattern` / +`--exclude-pattern` are repeatable substring filters on the URL. `get` and +`list` read one page and take `--cursor` and `--limit`. `results` follows every +page (`--limit` is the page size); on a running crawl it returns the URLs kept +so far, with `partial: true`. `download` reports the crawl's `status`, and +`partial: true` while it runs. + +`wait`, and `create --follow`, poll until the crawl ends or `--timeout` seconds +run out (default 600). When the time runs out, the command exits 0 and prints +the crawl with status `running` and the hint `zenrows crawl wait `; +the crawl keeps running. Ctrl-C stops the wait, not the crawl, and exits 130; +resume with `zenrows crawl wait `. A crawl that ends `failed` exits 1 +with `CRAWL_FAILED`. + +Errors carry the API's code as `server_code`, `crawl_id` on any call about one +crawl, and `retry_after` on a 429. When the account has reached its limit of +active jobs (3 by default), shared with its Batch jobs, a create gets +`CRAWL_TOO_MANY_CRAWLS` and creates nothing; retry after `retry_after` seconds. +Other codes: `CRAWL_QUOTA_EXCEEDED` and `CRAWL_KEY_CAP_REACHED` (402, with the +same claim, top-up and key-cap advice as other commands), +`CRAWL_INVALID_REQUEST` (400/422, do not retry as is), `CRAWL_REQUEST_IN_FLIGHT` +(409, retry once the first request ends), `CRAWL_NOT_FOUND`, +`CRAWL_CONTENT_NOT_FOUND` and `CRAWL_FAILED`. Set `ZENROWS_CRAWL_API_BASE` to +point the CLI at another deployment. ## 11. Browser Sessions diff --git a/docs/capabilities.md b/docs/capabilities.md index 96f241e..514b8b9 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -22,6 +22,6 @@ Classification is based on the public Zenrows documentation: | `protected_fetch` | Fetch `GET https://api.zenrows.com/v1/` with `mode`, `js_render`, `premium_proxy`, `proxy_country`, `wait`/`wait_for`, `js_instructions`, `response_type`, `screenshot`, `original_status`, … | available | | `extract` | Extract `GET https://api.zenrows.com/v1/` via `extract=auto` (domain-gated open beta; CLI falls back to `autoparse`), plus `autoparse`, `css_extractor`, `outputs`, `response_type=markdown\|plaintext` | beta | | `batch` | Batch `https://async.api.zenrows.com/v1` (separate host, `X-API-Key` header) — open beta. Local JSONL spec validation + credit estimation work with no key. | beta | -| `crawl` | Crawl `https://api.zenrows.com/v1/crawls` (`X-API-Key` header; same host as Fetch/Extract). Follows the links on each page from one start URL up to `depth`, on the start URL's registrable domain (subdomains count); returns URLs, and with `output_format: "html"` each page's HTML. Without Crawl access the API returns 403 `REQS008` → `CRAWL_NOT_ENABLED`. | beta | +| `crawl` | Crawl `https://api.zenrows.com/v1/crawls` (`X-API-Key` header; same host as Fetch/Extract). Follows the links on each page from one start URL up to `depth`, on the start URL's registrable domain (subdomains count); returns URLs, and with `output_format: "html"` each page's HTML. Subcommands: `create`, `get`, `list`, `results`, `content`, `download`, `stop`, `wait`. Without Crawl access the API returns 403 `REQS008` → `CRAWL_NOT_ENABLED`. | beta | | `browser` | Browser Sessions REST API `https://mcp.zenrows.com/browser/sessions/*` (Bearer; same backend as `@zenrows/mcp` `browser_*`); CDP via `zenrows browser connect`. Escalation-only (prefer fetch/extract); on by default, opt out via `policy.allow_browser`; bills by bandwidth + session time (15-min max) | available | | `mcp` | Hosted `https://mcp.zenrows.com/mcp` + local `npx -y @zenrows/mcp` | available | diff --git a/docs/contributing.md b/docs/contributing.md index 437ef4d..761bc02 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -40,7 +40,7 @@ npm run build # tsc → dist/ (for publishing) ## Running the Crawl e2e test `tests/e2e/crawl.e2e.test.ts` drives the built CLI (`zenrows crawl …`) against a -live Zenrows API: it starts one small crawl of the target you give it (depth 1, +live Zenrows API: it creates one small crawl of the target you give it (depth 1, 3 items, 5 pages, HTML output), waits for it, reads its results, one page's HTML and the NDJSON export, lists crawls, stops the ended crawl and checks a missing id. It is not part of `npm test`, and it skips unless `ZENROWS_E2E=1`, @@ -67,7 +67,7 @@ ZENROWS_E2E=1 ZENROWS_E2E_CRAWL_URL=https://example.com/products/ \ To target a local or staging deployment, set `ZENROWS_CRAWL_API_BASE` to its `/v1` base. When the account has reached its limit of active jobs (3 by default, shared with its Batch jobs), the test waits `retry_after` seconds and -retries the start for up to 5 minutes, so run concurrent e2e +retries the create for up to 5 minutes, so run concurrent e2e runs on one account one at a time. ## Rules diff --git a/registry/capabilities.json b/registry/capabilities.json index b331569..1f0b076 100644 --- a/registry/capabilities.json +++ b/registry/capabilities.json @@ -35,7 +35,7 @@ "command": "zenrows crawl", "backend": "Crawl — https://api.zenrows.com/v1/crawls (X-API-Key header; same host as Fetch/Extract)", "requiresAuth": true, - "notes": "Beta. Follows the links on each page from one start URL up to depth, on the start URL's registrable domain (subdomains count), and returns the URLs it keeps, optionally with each page's HTML (start/status/wait/results/content/list/stop). An account without Crawl access gets 403 REQS008 (CRAWL_NOT_ENABLED). When the account has reached its limit of active jobs (3 by default), shared with its Batch jobs, a start gets 429 too_many_crawls (CRAWL_TOO_MANY_CRAWLS); retry after retry_after seconds." + "notes": "Beta. Follows the links on each page from one start URL up to depth, on the start URL's registrable domain (subdomains count), and returns the URLs it keeps, optionally with each page's HTML (create/get/list/results/content/download/stop/wait). An account without Crawl access gets 403 REQS008 (CRAWL_NOT_ENABLED). When the account has reached its limit of active jobs (3 by default), shared with its Batch jobs, a create gets 429 too_many_crawls (CRAWL_TOO_MANY_CRAWLS); retry after retry_after seconds." }, "browser": { "key": "browser", diff --git a/registry/skills.json b/registry/skills.json index 7a01812..7b6e955 100644 --- a/registry/skills.json +++ b/registry/skills.json @@ -47,7 +47,7 @@ { "name": "crawl", "type": "skill", - "description": "Collect the URLs behind one start page with Crawl (beta): start, wait, read results and page HTML, stop.", + "description": "Collect the URLs behind one start page with Crawl (beta): create, wait, read results and page HTML, stop.", "status": "beta", "requires_backend_capabilities": ["crawl"], "requires_auth": true, diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index e6aac46..2aba854 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -1,6 +1,6 @@ --- name: crawl -description: Collect the URLs behind one start page with Crawl (beta) — start a crawl, wait for it, read the kept URLs and optionally each page's HTML, stop it. Use when the user has a listing or section URL but not the item URLs. +description: Collect the URLs behind one start page with Crawl (beta): create a crawl, wait for it, read the kept URLs and optionally each page's HTML, stop it. Use when the user has a listing or section URL but not the item URLs. version: 0.1.0 requires_backend_capabilities: [crawl] --- @@ -12,17 +12,17 @@ higher depth, on the pages behind it) and returns the URLs it keeps. Use it when you know where the items live (a category page, a blog index, a docs section) but not the item URLs themselves. Once you have the URLs, read pages with `zenrows fetch` / `zenrows extract`, or ask Crawl for each page's HTML -with `--html`. +with `--output-format html`. Crawl stays on the start URL's registrable domain; subdomains count. An account without Crawl access gets 403 `REQS008` → `CRAWL_NOT_ENABLED` ("Crawl is not enabled for this account"); do not retry it. -## Start, wait, read +## Create, wait, read ``` -zenrows crawl start https://example.com/products/ --depth 1 \ - --max-items 20 --max-pages 25 --include /product/ --follow --json +zenrows crawl create https://example.com/products/ --depth 1 \ + --max-items 20 --max-pages 25 --include-pattern /product/ --follow --json zenrows crawl results --json # every kept URL zenrows crawl results --out urls.jsonl # same, as JSONL ``` @@ -33,37 +33,41 @@ zenrows crawl results --out urls.jsonl # same, as JSONL both default to 10. `--max-pages` bounds the cost: each page is one fetch. The local policy `max_pages_per_run` caps `--max-pages` (default 1000); above it the CLI refuses with `POLICY_LIMIT_EXCEEDED` before any request. -- `--include` / `--exclude` are substring matches on the normalized URL and - repeat: `--include /product/ --exclude ?add-to-cart`. -- `start` returns at once. `--follow` (or `zenrows crawl wait `) polls - until the crawl ends (default timeout 600000 ms). On `--timeout` the crawl - keeps running (it is not stopped) and you get `CRAWL_WAIT_TIMEOUT` with - `crawl_id`. Ctrl-C also stops the wait, not the crawl, and exits 130; +- `--include-pattern` / `--exclude-pattern` are substring matches on the + normalized URL and repeat: + `--include-pattern /product/ --exclude-pattern ?add-to-cart`. +- `create` returns at once. `--follow` (or `zenrows crawl wait `) polls + until the crawl ends or `--timeout` seconds run out (default 600). When the + time runs out, the command exits 0 and prints the crawl with status + `running` and the hint `zenrows crawl wait `. The crawl keeps running; + it is not stopped. Ctrl-C also stops the wait, not the crawl, and exits 130; resume with `zenrows crawl wait `. - `results` on a running crawl returns the URLs kept so far, with - `partial: true`. Read again once the crawl ends. + `partial: true`. Read again once the crawl ends. `--limit` sets the page size + of each request. ## Page HTML ``` -zenrows crawl start --depth 1 --html --max-pages 20 --follow +zenrows crawl create --depth 1 --output-format html --max-pages 20 --follow zenrows crawl results --json # fetched results carry content_url zenrows crawl content # one page's HTML (or pass the content_url) -zenrows crawl results --download --out pages.jsonl # every URL + HTML, NDJSON +zenrows crawl download --out pages.jsonl # every URL + HTML, NDJSON ``` -`--html` uses up the page budget: each kept page is one more fetch, and -`--max-pages` counts it. Raise `--max-pages` to keep the same number of URLs. -A result's `content_status` is `pending`, `fetched` or `failed`; only -`fetched` results have content (else `CRAWL_CONTENT_NOT_FOUND`). +`--output-format html` uses up the page budget: each kept page is one more +fetch, and `--max-pages` counts it. Raise `--max-pages` to keep the same number +of URLs. A result's `content_status` is `pending`, `fetched` or `failed`; only +`fetched` results have content (else `CRAWL_CONTENT_NOT_FOUND`). `download` +reports the crawl's `status`, and `partial: true` while it runs. ## While it runs, and after ``` -zenrows crawl status # status, coverage, stop_reason / error -zenrows crawl results --cursor --json # one page; keep polling with next_cursor -zenrows crawl list # your crawls, newest first -zenrows crawl stop # stop it; kept URLs stay readable +zenrows crawl get # status, coverage, stop_reason / error, one page of results +zenrows crawl get --cursor --json # the next page; keep polling with next_cursor +zenrows crawl list # your crawls, newest first (--cursor, --limit) +zenrows crawl stop # stop it; kept URLs stay readable ``` A crawl ends `completed` (nothing left, or a cap hit: `stop_reason` @@ -74,24 +78,27 @@ and `error.detail` say why). Only `failed` exits non-zero (`CRAWL_FAILED`). ## Limits When the account has reached its limit of active jobs (3 by default), shared -with its Batch jobs, a start gets 429 `too_many_crawls` → +with its Batch jobs, a create gets 429 `too_many_crawls` → `CRAWL_TOO_MANY_CRAWLS`; nothing was created. Retry after `retry_after` seconds (about 30), or stop one of your crawls or Batch jobs first. ## Errors Every Crawl error carries the API's code as `server_code` (when the API sent -one), and `crawl_id` once the crawl exists. +one), and `crawl_id` on any call about one crawl. | Code | Meaning | Retry? | | --- | --- | --- | | `CRAWL_NOT_ENABLED` | 403 `REQS008`: Crawl is not enabled for this account | no | +| `AUTH_INVALID` | 401: the API key was rejected | no, log in again | +| `CRAWL_QUOTA_EXCEEDED` | 402: the account is out of credits (claim the account or add credits) | after credits renew or are added | +| `CRAWL_KEY_CAP_REACHED` | 402 `AUTH014`: this API key reached its credit cap | after the cap resets or is raised | | `CRAWL_INVALID_REQUEST` | 400 or 422: fix the request. For `idempotency_key_reused`, send a new key or no key | no, not as is | | `CRAWL_REQUEST_IN_FLIGHT` | 409: the same Idempotency-Key is still in flight | yes, once the first request ends | | `CRAWL_NOT_FOUND` / `CRAWL_CONTENT_NOT_FOUND` | 404: wrong id, or the page has no content | no | | `CRAWL_TOO_MANY_CRAWLS` | 429: the account's active-job limit | yes, after `retry_after` | -| `CRAWL_WAIT_TIMEOUT` | the wait ran out; the crawl keeps running | wait again | | `CRAWL_FAILED` | the crawl ended `failed`, or another error | 5xx: yes, with backoff | +| `BACKEND_UNAVAILABLE` | the API could not be reached | yes | Rules: start small (`--depth 1`, low caps) and check the results before a deep crawl. Mind the cost ([[cost-control]]). Read errors with diff --git a/skills/trace-debug/SKILL.md b/skills/trace-debug/SKILL.md index f0e35a2..65c0da2 100644 --- a/skills/trace-debug/SKILL.md +++ b/skills/trace-debug/SKILL.md @@ -47,12 +47,15 @@ zenrows trace export # JSON for sharing `failure_detail` in `zenrows batch status --json` say why. - `CRAWL_NOT_ENABLED` → the API answered 403 `REQS008`: Crawl is not enabled for this account. Do not retry; ask Zenrows support for access. +- `CRAWL_QUOTA_EXCEEDED` / `CRAWL_KEY_CAP_REACHED` → Crawl's 402: the + account is out of credits, or this key reached its credit cap (`AUTH014`). + Same advice as `POLICY_MAX_CREDITS_EXCEEDED` / `KEY_CREDIT_CAP_REACHED`. - `CRAWL_TOO_MANY_CRAWLS` → 429 `too_many_crawls`: the account has reached its limit of active jobs (3 by default), shared with its Batch jobs. Nothing was created; retry after `retry_after` seconds (see `zenrows crawl list`). - `CRAWL_FAILED` → the crawl ended `failed`; `error.code` in - `zenrows crawl status --json` says why, with `error.detail`. See + `zenrows crawl get --json` says why, with `error.detail`. See [[crawl]]. - `PARAM_CONFLICT_AUTO_MANUAL` → drop the managed flags or add `--manual`. - `CAPABILITY_UNAVAILABLE` → the primitive is not available on this account (e.g. beta/invite-only); use the local-spec path where offered. diff --git a/skills/zenrows/SKILL.md b/skills/zenrows/SKILL.md index 595c9c9..350d6e4 100644 --- a/skills/zenrows/SKILL.md +++ b/skills/zenrows/SKILL.md @@ -35,7 +35,7 @@ If the user has a known URL and wants structured data: → Use Extract. (zenrows extract | --autoparse | --css) If the user has a start page (listing, index, section) but not the item URLs: - → Use Crawl. (zenrows crawl start --depth 1 --follow, + → Use Crawl. (zenrows crawl create --depth 1 --follow, then `zenrows crawl results `) If the user has many URLs (bulk): diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index 1c3a7cc..e3bd52d 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -2,10 +2,10 @@ * `zenrows crawl` — Crawl API (status: beta). * * Give Crawl one start URL and read back the URLs it finds behind it, optionally - * with each page's HTML. A crawl is an async job: `start` returns at once - * (or polls with `--follow`), then `status` / `wait` / `results` / `content` - * read it and `stop` ends it. An account without Crawl access gets 403 REQS008 - * → CRAWL_NOT_ENABLED. + * with each page's HTML. A crawl is an async job: `create` returns at once + * (or waits with `--follow`), then `get` / `wait` / `results` / `content` / + * `download` read it and `stop` ends it. An account without Crawl access gets + * 403 REQS008 → CRAWL_NOT_ENABLED. */ import { log } from "../../core/logger.ts"; import { requireApiKey } from "../../core/auth.ts"; @@ -19,6 +19,7 @@ import { downloadCrawl, getCrawl, getCrawlContent, + isTerminal, listAllResults, listCrawls, stopCrawl, @@ -31,7 +32,6 @@ import { import { asNumber, asString, parse, type Command, type RunContext } from "../command.ts"; import { ToolkitError } from "../../core/errors.ts"; import { printError, writeOut } from "../output.ts"; -import { normalizeTimeout } from "./fetch.ts"; /** The API's default for `max_pages` (and `max_items`) when the flag is unset. */ const DEFAULT_MAX_PAGES = 10; @@ -42,60 +42,64 @@ export const crawl: Command = { name: "crawl", summary: "Crawl a site from one start URL and collect the URLs behind it (beta).", usage: - "zenrows crawl --depth N|status |results |content |list|stop |wait >", + "zenrows crawl --depth N|get |list|results |content |download |stop |wait >", help: [ "Crawl is in beta. Cloud (needs a key with Crawl access):", - " start --depth [flags] start a crawl from one URL (returns at once)", - " --depth link hops to follow from the start URL (1-100000, required)", - " --max-items stop after keeping n URLs (API default 10)", - " --max-pages stop after fetching n pages (API default 10; bounds cost);", - " the local policy max_pages_per_run caps it (default 1000)", - " --include keep only URLs containing this substring (repeatable)", - " --exclude drop URLs containing this substring (repeatable)", - " --html also fetch each kept URL's page HTML (read with `content`);", - " each kept page is one more fetch counted by --max-pages", - " --follow poll until the crawl ends (Ctrl-C stops the wait, not the crawl)", - " --timeout with --follow: give up waiting after ms (default 600000)", - " --idempotency-key make a retried start create no second crawl", - " --no-signup do not auto-create a Free plan account if no key exists", - " status show status, coverage, stop reason / error", - " wait [--timeout ] poll until the crawl ends (it is not stopped on timeout)", - " results all kept URLs; on a running crawl, those kept so far (partial)", - " --out write results as JSONL instead of printing", - " --cursor [--limit ] read one page only (works while running; prints next_cursor)", - " --download fetch the NDJSON export (with page HTML for --html crawls)", - " into --out, or .jsonl", - " content print one kept URL's page (content_id or content_url)", - " --out write it to a file instead", - " list [--limit ] [--cursor c] list your crawls, newest first", - " stop stop a running crawl (an ended crawl answers as it ended)", - " --json print structured output", + " create --depth [flags] start a crawl from one URL (returns at once)", + " --depth link hops to follow from the start URL (1-100000, required)", + " --max-items stop after keeping n URLs (API default 10)", + " --max-pages stop after fetching n pages (API default 10; bounds cost);", + " the local policy max_pages_per_run caps it (default 1000)", + " --include-pattern

keep only URLs containing this substring (repeatable)", + " --exclude-pattern

drop URLs containing this substring (repeatable)", + " --output-format html also fetch each kept URL's page HTML (read with content/download);", + " each kept page is one more fetch counted by --max-pages", + " --follow wait until the crawl ends (Ctrl-C stops the wait, not the crawl)", + " --timeout with --follow: stop waiting after s seconds (default 600)", + " --idempotency-key so a retried create starts no second crawl", + " --no-signup do not auto-create a Free plan account if no key exists", + " get [--cursor ] [--limit ]", + " the crawl and one page of its results (limit up to 10000)", + " list [--cursor ] [--limit ] your crawls, newest first (limit up to 100)", + " results [--limit ] every kept URL as JSONL; on a running crawl, those kept so far", + " (partial). --limit is the page size per request", + " --out write the results to a file instead of printing", + " content print one kept URL's page (content_id or content_url)", + " --out write it to a file instead", + " download [--out ] save the NDJSON export (with page HTML for html crawls),", + " default .jsonl; partial while the crawl runs", + " stop stop a running crawl (an ended crawl answers as it ended)", + " wait [--timeout ] poll until the crawl ends or s seconds (default 600) run out;", + " a crawl still running then is printed, not stopped", + " --json print structured output", ].join("\n"), async run(argv: string[], ctx: RunContext): Promise { const [sub, ...rest] = argv; try { switch (sub) { - case "start": - return await startCmd(rest, ctx); - case "status": - return await statusCmd(rest, ctx); - case "wait": - return await waitCmd(rest, ctx); + case "create": + return await createCmd(rest, ctx); + case "get": + return await getCmd(rest, ctx); + case "list": + return await listCmd(rest, ctx); case "results": return await resultsCmd(rest, ctx); case "content": return await contentCmd(rest, ctx); - case "list": - return await listCmd(rest, ctx); + case "download": + return await downloadCmd(rest, ctx); case "stop": return await stopCmd(rest, ctx); + case "wait": + return await waitCmd(rest, ctx); default: throw new ToolkitError({ code: "INVALID_USAGE", message: `Unknown crawl subcommand: ${sub ?? "(none)"}`, likely_cause: "Subcommand not recognized.", next_action: - "Use start --depth N | status | wait | results | content | list | stop .", + "Use create --depth N | get | list | results | content | download | stop | wait .", suggested_commands: ["zenrows crawl --help"], }); } @@ -106,14 +110,14 @@ export const crawl: Command = { }, }; -async function startCmd(rest: string[], ctx: RunContext): Promise { +async function createCmd(rest: string[], ctx: RunContext): Promise { const { values, positionals } = parse(rest, { depth: { type: "string" }, "max-items": { type: "string" }, "max-pages": { type: "string" }, - include: { type: "string", multiple: true }, - exclude: { type: "string", multiple: true }, - html: { type: "boolean" }, + "include-pattern": { type: "string", multiple: true }, + "exclude-pattern": { type: "string", multiple: true }, + "output-format": { type: "string" }, follow: { type: "boolean" }, timeout: { type: "string" }, "idempotency-key": { type: "string" }, @@ -121,15 +125,14 @@ async function startCmd(rest: string[], ctx: RunContext): Promise { json: { type: "boolean" }, }); const json = ctx.json || values.json === true; - const wait = values.follow === true; const url = positionals[0]; if (!url) { throw new ToolkitError({ code: "INVALID_USAGE", message: "Provide the start URL.", likely_cause: "No positional was given.", - next_action: "Usage: zenrows crawl start --depth 1", - suggested_commands: ["zenrows crawl start https://example.com/ --depth 1"], + next_action: "Usage: zenrows crawl create --depth 1", + suggested_commands: ["zenrows crawl create https://example.com/ --depth 1"], }); } const depth = limitFlag("depth", values.depth); @@ -139,7 +142,7 @@ async function startCmd(rest: string[], ctx: RunContext): Promise { message: "--depth is required.", likely_cause: "Crawl has no default depth yet; every crawl states how many link hops to follow.", next_action: "Pass --depth 1 to collect the start page's links, --depth 2 to also open each of those.", - suggested_commands: [`zenrows crawl start ${url} --depth 1`], + suggested_commands: [`zenrows crawl create ${url} --depth 1`], }); } const params: CreateCrawlParams = { url, depth }; @@ -147,12 +150,23 @@ async function startCmd(rest: string[], ctx: RunContext): Promise { if (maxItems !== undefined) params.max_items = maxItems; const maxPages = limitFlag("max-pages", values["max-pages"]); if (maxPages !== undefined) params.max_pages = maxPages; - const include = patterns(values.include); + const include = patterns(values["include-pattern"]); if (include.length) params.include_patterns = include; - const exclude = patterns(values.exclude); + const exclude = patterns(values["exclude-pattern"]); if (exclude.length) params.exclude_patterns = exclude; - if (values.html === true) params.output_format = "html"; - const timeoutMs = normalizeTimeout(values.timeout); + const outputFormat = asString(values["output-format"]); + if (outputFormat !== undefined) { + if (outputFormat !== "html") { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `Invalid --output-format value '${outputFormat}'.`, + likely_cause: "--output-format takes html only. Without it, a crawl returns URLs only.", + next_action: "Pass --output-format html, or leave it out.", + }); + } + params.output_format = outputFormat; + } + const timeoutMs = timeoutFlag(values.timeout); const idempotencyKey = asString(values["idempotency-key"]); assertUsable("crawl"); @@ -175,11 +189,12 @@ async function startCmd(rest: string[], ctx: RunContext): Promise { log.step(`Starting crawl of ${url} (depth ${depth})…`); try { const created = await createCrawl(params, { apiKey, idempotencyKey }); - const finished: Crawl = wait ? await waitUntilEnd(created.crawl_id, { apiKey, timeoutMs }) : created; + const follow = values.follow === true; + const finished: Crawl = follow ? await waitUntilEnd(created.crawl_id, { apiKey, timeoutMs }) : created; const failure = crawlFailure(finished); const runDir = writeRun({ runId, - command: "zenrows crawl start", + command: "zenrows crawl create", capability: "crawl", url, startedAt, @@ -189,16 +204,14 @@ async function startCmd(rest: string[], ctx: RunContext): Promise { result: { crawlId: created.crawl_id, status: finished.status, coverage: finished.coverage }, ...(failure ? { error: failure.toJSON() } : {}), }); - const code = printCrawl(finished, json, `Started crawl ${created.crawl_id}`); - if (!json) { - if (runDir) log.dim(` artifact: ${runDir}`); - if (!wait) log.dim(` next: zenrows crawl wait ${created.crawl_id}`); - } + const code = printCrawl(finished, json, follow ? waitHeadline(finished) : `Started crawl ${created.crawl_id}`); + if (!json && runDir) log.dim(` artifact: ${runDir}`); + printWaitHint(finished, json); return code; } catch (err) { writeRun({ runId, - command: "zenrows crawl start", + command: "zenrows crawl create", capability: "crawl", url, startedAt, @@ -211,64 +224,85 @@ async function startCmd(rest: string[], ctx: RunContext): Promise { } } -async function statusCmd(rest: string[], ctx: RunContext): Promise { - const { values, positionals } = parse(rest, { json: { type: "boolean" } }); - const id = requireId(positionals[0], "status"); +async function getCmd(rest: string[], ctx: RunContext): Promise { + const { values, positionals } = parse(rest, { + cursor: { type: "string" }, + limit: { type: "string" }, + json: { type: "boolean" }, + }); + const json = ctx.json || values.json === true; + const id = requireId(positionals[0], "get"); + const cursor = asString(values.cursor); + const limit = pageLimit(values.limit, 10_000); assertUsable("crawl"); const apiKey = requireApiKey(); - const c = await getCrawl(id, { apiKey, limit: 1 }); - return printCrawl(c, ctx.json || values.json === true, `Crawl ${id}`); + const page = await getCrawl(id, { apiKey, cursor, limit }); + const results = page.results ?? []; + const nextCursor = page.next_cursor ?? null; + const code = printCrawl(page, json, `Crawl ${id}`, { results, next_cursor: nextCursor }); + if (!json) { + log.info(`${results.length} result(s) on this page:`); + if (results.length) log.out(toJsonl(results).trimEnd()); + log.info(`next_cursor: ${nextCursor ?? "null (last page)"}`); + } + return code; } async function waitCmd(rest: string[], ctx: RunContext): Promise { const { values, positionals } = parse(rest, { timeout: { type: "string" }, json: { type: "boolean" } }); + const json = ctx.json || values.json === true; const id = requireId(positionals[0], "wait"); - const timeoutMs = normalizeTimeout(values.timeout); + const timeoutMs = timeoutFlag(values.timeout); assertUsable("crawl"); const apiKey = requireApiKey(); const c = await waitUntilEnd(id, { apiKey, timeoutMs }); - return printCrawl(c, ctx.json || values.json === true, `Crawl ${id} finished`); + const code = printCrawl(c, json, waitHeadline(c)); + printWaitHint(c, json); + return code; +} + +function waitHeadline(c: Crawl): string { + return isTerminal(c.status) ? `Crawl ${c.crawl_id} finished` : `Stopped waiting: crawl ${c.crawl_id} is still running`; +} + +function printWaitHint(c: Crawl, json: boolean): void { + if (!json && !isTerminal(c.status)) log.dim(` next: zenrows crawl wait ${c.crawl_id}`); } async function resultsCmd(rest: string[], ctx: RunContext): Promise { const { values, positionals } = parse(rest, { out: { type: "string" }, - cursor: { type: "string" }, limit: { type: "string" }, - download: { type: "boolean" }, json: { type: "boolean" }, }); const json = ctx.json || values.json === true; const id = requireId(positionals[0], "results"); const outFile = asString(values.out); - const cursor = asString(values.cursor); - const limit = pageLimit(values.limit, 10_000); + const pageSize = pageLimit(values.limit, 10_000); assertUsable("crawl"); const apiKey = requireApiKey(); + const { crawl: c, results, partial } = await listAllResults(id, { apiKey, pageSize }); + return printResults(id, c.status, results, { outFile, json, partial }); +} - if (values.download === true) { - const file = outFile ?? `${id.replace(/[^A-Za-z0-9._-]+/g, "_")}.jsonl`; - const { crawlStatus, ndjson } = await downloadCrawl(id, { apiKey }); - writeOut(file, ndjson); - const lines = ndjson.split("\n").filter((l) => l.trim()).length; - const partial = crawlStatus === "running"; - if (json) { - log.out(JSON.stringify({ ok: true, crawlId: id, crawlStatus, partial, count: lines, file }, null, 2)); - } else { - log.success(`Downloaded ${lines} result(s) → ${file}`); - if (partial) log.warn(" The crawl is still running: this file is partial. Download again once it ends."); - } - return 0; - } - - // One page, as the API returns it — the way to follow a crawl while it runs. - if (cursor !== undefined || limit !== undefined) { - const page = await getCrawl(id, { apiKey, cursor, limit }); - return printResults(id, page.status, page.results, { outFile, json, next_cursor: page.next_cursor ?? null }); +async function downloadCmd(rest: string[], ctx: RunContext): Promise { + const { values, positionals } = parse(rest, { out: { type: "string" }, json: { type: "boolean" } }); + const json = ctx.json || values.json === true; + const id = requireId(positionals[0], "download"); + const file = asString(values.out) ?? `${id.replace(/[^A-Za-z0-9._-]+/g, "_")}.jsonl`; + assertUsable("crawl"); + const apiKey = requireApiKey(); + const { status, ndjson } = await downloadCrawl(id, { apiKey }); + writeOut(file, ndjson); + const count = ndjson.split("\n").filter((l) => l.trim()).length; + const partial = status === "running"; + if (json) { + log.out(JSON.stringify({ ok: true, crawlId: id, status, partial, count, file }, null, 2)); + } else { + log.success(`Downloaded ${count} result(s) → ${file}`); + if (partial) log.warn(" The crawl is still running: this file is partial. Download again once it ends."); } - - const { crawl: c, results, partial } = await listAllResults(id, { apiKey }); - return printResults(id, c.status, results, { outFile, json, partial }); + return 0; } /** Write or print results. Under --json, always print the envelope (`file` replaces `results` with --out). */ @@ -276,16 +310,12 @@ function printResults( id: string, status: string, results: CrawlResult[], - o: { outFile?: string; json: boolean; partial?: boolean; next_cursor?: string | null }, + o: { outFile?: string; json: boolean; partial: boolean }, ): number { - const extra = { - ...(o.partial !== undefined ? { partial: o.partial } : {}), - ...(o.next_cursor !== undefined ? { next_cursor: o.next_cursor } : {}), - }; if (o.outFile) writeOut(o.outFile, toJsonl(results)); if (o.json) { const rows = o.outFile ? { file: o.outFile } : { results }; - log.out(JSON.stringify({ ok: true, crawlId: id, status, count: results.length, ...rows, ...extra }, null, 2)); + log.out(JSON.stringify({ ok: true, crawlId: id, status, count: results.length, ...rows, partial: o.partial }, null, 2)); return 0; } if (o.outFile) { @@ -295,7 +325,6 @@ function printResults( log.out(toJsonl(results).trimEnd()); } if (o.partial) log.warn(" The crawl is still running: these results are partial. Read them again once it ends."); - if (o.next_cursor !== undefined) log.info(`next_cursor: ${o.next_cursor ?? "null (last page)"}`); return 0; } @@ -312,7 +341,7 @@ async function contentCmd(rest: string[], ctx: RunContext): Promise { message: "Missing content id.", likely_cause: "No positional was provided.", next_action: - "Pass the id from a fetched result's content_url (the last path segment), e.g. `zenrows crawl content ct_…`. Only crawls started with --html have content.", + "Pass the id from a fetched result's content_url (the last path segment), e.g. `zenrows crawl content ct_…`. Only crawls created with --output-format html have content.", suggested_commands: [`zenrows crawl results ${id}`], }); } @@ -399,14 +428,14 @@ export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null { insufficient_credits: "The account ran out of credits. Check `zenrows usage`, add credits, then start a new crawl.", seed_unreachable: "The start URL could not be fetched (the site may be blocking it). Check it with `zenrows fetch `.", no_items_found: - "Nothing matched. Loosen --include / --exclude, raise --depth, or check the start page with `zenrows fetch `.", + "Nothing matched. Loosen --include-pattern / --exclude-pattern, raise --depth, or check the start page with `zenrows fetch `.", }; return new ToolkitError({ code: "CRAWL_FAILED", message: `Crawl ${c.crawl_id} failed.`, likely_cause: why || "The crawl ended in status failed without a reason.", next_action: (code && next[code]) || "Read the error, fix the cause, then start a new crawl.", - suggested_commands: [`zenrows crawl status ${c.crawl_id}`], + suggested_commands: [`zenrows crawl get ${c.crawl_id}`], server_code: code, crawl_id: c.crawl_id, }); @@ -415,12 +444,17 @@ export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null { /** * Print a crawl (or a stop answer), structured under --json, and return the exit * code: 1 when it ended `failed`, else 0. `stopped` is a deliberate outcome, so - * it exits 0 but prints as a warning. + * it exits 0 but prints as a warning. `page` carries the results a `get` read. */ -function printCrawl(c: Crawl | CrawlStop, json: boolean, headline: string): number { +function printCrawl( + c: Crawl | CrawlStop, + json: boolean, + headline: string, + page?: { results: CrawlResult[]; next_cursor: string | null }, +): number { const err = crawlFailure(c); const crawlFields = { ...c } as Record; - // A status read asks for one result to keep polls cheap; that page is not "the results". + // A wait polls with one result to stay cheap; that page is not "the results". delete crawlFields.results; delete crawlFields.next_cursor; if (json) { @@ -432,6 +466,7 @@ function printCrawl(c: Crawl | CrawlStop, json: boolean, headline: string): numb status: c.status, ...(c.stop_reason ? { stop_reason: c.stop_reason } : {}), crawl: crawlFields, + ...page, ...(err ? { error: err.toJSON() } : {}), }, null, @@ -467,6 +502,21 @@ function limitFlag(name: string, v: unknown): number | undefined { return n; } +/** `--timeout` in seconds, returned in milliseconds; undefined when unset. */ +function timeoutFlag(v: unknown): number | undefined { + if (v === undefined) return undefined; + const seconds = asNumber(v); + if (seconds === undefined || seconds <= 0) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `Invalid --timeout value '${String(v)}'.`, + likely_cause: "--timeout takes a positive number of seconds.", + next_action: "Pass seconds, e.g. --timeout 600 for ten minutes.", + }); + } + return seconds * 1000; +} + /** `--limit` for a paged read: a whole number from 1 to `max`. */ function pageLimit(v: unknown, max: number): number | undefined { if (v === undefined) return undefined; diff --git a/src/core/crawl-api.ts b/src/core/crawl-api.ts index f98b248..6ec4f0e 100644 --- a/src/core/crawl-api.ts +++ b/src/core/crawl-api.ts @@ -212,7 +212,7 @@ async function crawlJson(method: string, path: string, opts: RequestOpts): Pr code: "CRAWL_FAILED", message: "The Crawl API response was not valid JSON.", likely_cause: text.slice(0, 240) || "Empty response body.", - next_action: "Retry, or check the crawl with `zenrows crawl status `.", + next_action: "Retry, or check the crawl with `zenrows crawl get `.", crawl_id: opts.crawlId, }); } @@ -275,7 +275,7 @@ function classifyProblem( return new ToolkitError({ code: "CRAWL_CONTENT_NOT_FOUND", message: "Crawl content not found.", - likely_cause: `${cause}. The page was not fetched (yet), its fetch failed, or the crawl ran without --html.`, + likely_cause: `${cause}. The page was not fetched (yet), its fetch failed, or the crawl ran without --output-format html.`, next_action: "Read the result's content_status with `zenrows crawl results `; only `fetched` results have content.", suggested_commands: ["zenrows crawl list"], }); @@ -298,15 +298,15 @@ function classifyProblem( suggested_commands: ["zenrows crawl list"], }); } - if (status === 402 && isKeyCapReached(serverCode)) { - return keyCapReached(request, { status: 402, detail: problem.detail || problem.title || undefined }); - } if (status === 402) { + // Same advice as the shared credit errors, under Crawl's own codes. + const detail = problem.detail || problem.title || undefined; + if (isKeyCapReached(serverCode)) { + return new ToolkitError({ ...keyCapReached(request, { status, detail }).toJSON(), code: "CRAWL_KEY_CAP_REACHED" }); + } const acct = readAccount(); - return quotaExhausted(request, acct?.unclaimed ? acct.claimUrl : undefined, { - status: 402, - detail: problem.detail || problem.title || undefined, - }); + const quota = quotaExhausted(request, acct?.unclaimed ? acct.claimUrl : undefined, { status, detail }); + return new ToolkitError({ ...quota.toJSON(), code: "CRAWL_QUOTA_EXCEEDED" }); } if (status === 409) { return new ToolkitError({ @@ -408,9 +408,9 @@ export async function getCrawlContent( /** * Every result in one NDJSON file (`{url, content_status?, content?}` per line). - * `crawlStatus` is the `X-Crawl-Status` header: `running` means the file is partial. + * `status` is the `X-Crawl-Status` header: `running` means the file is partial. */ -export async function downloadCrawl(id: string, opts: CallOpts): Promise<{ crawlStatus?: string; ndjson: string }> { +export async function downloadCrawl(id: string, opts: CallOpts): Promise<{ status?: string; ndjson: string }> { const res = await crawlFetch("GET", `/crawls/${encodeURIComponent(id)}/download`, { apiKey: opts.apiKey, crawlId: id, @@ -418,7 +418,7 @@ export async function downloadCrawl(id: string, opts: CallOpts): Promise<{ crawl fetchImpl: opts.fetchImpl, timeoutMs: opts.timeoutMs ?? 300_000, }); - return { crawlStatus: res.headers.get("x-crawl-status") ?? undefined, ndjson: res.text }; + return { status: res.headers.get("x-crawl-status") ?? undefined, ndjson: res.text }; } /** @@ -434,7 +434,7 @@ export function contentIdOf(result: CrawlResult): string | undefined { /** * Read every kept result, following `next_cursor`. While a crawl runs its * `next_cursor` is never null, so the read also ends at the first empty page; - * `partial` is true when the crawl was still running. + * `partial` is true when the crawl was still running or a cursor remains. */ export async function listAllResults( id: string, @@ -447,38 +447,28 @@ export async function listAllResults( const rows = page.results ?? []; results.push(...rows); if (!rows.length || !page.next_cursor || page.next_cursor === cursor) { - return { crawl: page, results, partial: !isTerminal(page.status) }; + return { crawl: page, results, partial: !isTerminal(page.status) || !!page.next_cursor }; } cursor = page.next_cursor; } } /** - * Poll `GET /crawls/{id}?limit=1` until the status leaves `running`, backing off - * 2s → ×1.5 → capped at 15s. On timeout throws CRAWL_WAIT_TIMEOUT - * and leaves the crawl running. `sleepImpl` is injectable for tests. + * Poll `GET /crawls/{id}?limit=1` until the status leaves `running` or + * `timeoutMs` (default 600 s) runs out, backing off 2s → ×1.5 → capped at 15s. + * A timeout is not an error: it returns the crawl, still `running`. */ export async function waitForCrawl( id: string, opts: CallOpts & { sleepImpl?: (ms: number) => Promise }, ): Promise { const sleep = opts.sleepImpl ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); - const timeoutMs = opts.timeoutMs ?? 600_000; - const deadline = Date.now() + timeoutMs; + const deadline = Date.now() + (opts.timeoutMs ?? 600_000); let delay = 2000; for (;;) { const crawl = await getCrawl(id, { apiKey: opts.apiKey, fetchImpl: opts.fetchImpl, limit: 1 }); if (isTerminal(crawl.status)) return crawl; - if (Date.now() + delay > deadline) { - throw new ToolkitError({ - code: "CRAWL_WAIT_TIMEOUT", - message: `Timed out waiting for crawl ${id} to finish.`, - likely_cause: `The crawl was still running after ${Math.round(timeoutMs / 1000)}s. It keeps running; it was not stopped.`, - next_action: "Re-check progress with `zenrows crawl status `, wait again with a larger --timeout, or stop it.", - suggested_commands: [`zenrows crawl status ${id}`, `zenrows crawl wait ${id}`, `zenrows crawl stop ${id}`], - crawl_id: id, - }); - } + if (Date.now() + delay > deadline) return crawl; await sleep(delay); delay = Math.min(delay * 1.5, 15_000); } diff --git a/src/core/errors.ts b/src/core/errors.ts index 0c2c60f..de8ad6c 100644 --- a/src/core/errors.ts +++ b/src/core/errors.ts @@ -36,13 +36,14 @@ export type ErrorCode = | "BATCH_NOT_FOUND" | "BATCH_FAILED" | "CRAWL_NOT_ENABLED" + | "CRAWL_QUOTA_EXCEEDED" + | "CRAWL_KEY_CAP_REACHED" | "CRAWL_TOO_MANY_CRAWLS" | "CRAWL_NOT_FOUND" | "CRAWL_CONTENT_NOT_FOUND" | "CRAWL_INVALID_REQUEST" | "CRAWL_REQUEST_IN_FLIGHT" | "CRAWL_FAILED" - | "CRAWL_WAIT_TIMEOUT" | "PLUGIN_UNSUPPORTED" | "MCP_CLIENT_UNSUPPORTED" | "ASSET_NOT_FOUND" diff --git a/tests/crawl-api.test.ts b/tests/crawl-api.test.ts index 0ffab09..f325630 100644 --- a/tests/crawl-api.test.ts +++ b/tests/crawl-api.test.ts @@ -145,7 +145,7 @@ test("downloadCrawl returns the NDJSON and X-Crawl-Status", async () => { }); const out = await downloadCrawl("c_1", { apiKey: "k", fetchImpl: impl }); assert.match(calls[0]!.url, /\/crawls\/c_1\/download$/); - assert.equal(out.crawlStatus, "running"); + assert.equal(out.status, "running"); assert.equal(out.ndjson.trim().split("\n").length, 2); }); @@ -234,6 +234,25 @@ test("429 too_many_crawls maps to CRAWL_TOO_MANY_CRAWLS with retry_after and no assert.equal(calls.length, 1); }); +test("402 maps to CRAWL_QUOTA_EXCEEDED, and AUTH014 to CRAWL_KEY_CAP_REACHED, with the shared advice", async () => { + const quota = problem(402, "AUTH004", "Usage limit reached."); + await assert.rejects( + () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: quota.impl }), + (e: unknown) => + e instanceof ToolkitError && e.code === "CRAWL_QUOTA_EXCEEDED" && e.server_code === "AUTH004" && /zenrows usage/.test(e.next_action), + ); + const cap = problem(402, "AUTH014", "Key cap reached."); + await assert.rejects( + () => getCrawl("c_1", { apiKey: "k", fetchImpl: cap.impl }), + (e: unknown) => + e instanceof ToolkitError && + e.code === "CRAWL_KEY_CAP_REACHED" && + e.server_code === "AUTH014" && + e.crawl_id === "c_1" && + /settings\/api-keys/.test(e.next_action), + ); +}); + test("an error body without a code leaves server_code unset", async () => { const { impl } = stubFetch(500, "upstream exploded"); await assert.rejects( @@ -274,14 +293,15 @@ function pagedFetch(pages: Array>) { return { impl, cursors, limits, calls: () => i }; } -test("listAllResults follows next_cursor and stops on null", async () => { - const { impl, cursors } = pagedFetch([ +test("listAllResults follows next_cursor with the page size and stops on null", async () => { + const { impl, cursors, limits } = pagedFetch([ { status: "completed", results: [{ url: "https://example.com/product/1" }], next_cursor: "c1" }, { status: "completed", results: [{ url: "https://example.com/product/2" }], next_cursor: "c2" }, { status: "completed", results: [{ url: "https://example.com/product/3" }], next_cursor: null }, ]); - const { results, crawl, partial } = await listAllResults("c_1", { apiKey: "k", fetchImpl: impl }); + const { results, crawl, partial } = await listAllResults("c_1", { apiKey: "k", fetchImpl: impl, pageSize: 500 }); assert.equal(partial, false); + assert.ok(limits.every((l) => l === "500")); assert.deepEqual(results.map((r) => r.url), ["https://example.com/product/1", "https://example.com/product/2", "https://example.com/product/3"]); assert.deepEqual(cursors, [null, "c1", "c2"]); assert.equal(crawl.status, "completed"); @@ -321,7 +341,7 @@ test("waitForCrawl treats failed and stopped as terminal", async () => { } }); -test("waitForCrawl times out with CRAWL_WAIT_TIMEOUT, carries crawl_id and does not stop the crawl", async () => { +test("waitForCrawl returns the running crawl when the timeout runs out, and does not stop it", async () => { const methods: string[] = []; const impl = (async (_url: string, init?: RequestInit) => { methods.push(init?.method ?? "GET"); @@ -330,10 +350,8 @@ test("waitForCrawl times out with CRAWL_WAIT_TIMEOUT, carries crawl_id and does headers: { "content-type": "application/json" }, }); }) as unknown as typeof fetch; - await assert.rejects( - () => waitForCrawl("c_1", { apiKey: "k", fetchImpl: impl, timeoutMs: 1, sleepImpl: async () => {} }), - (e: unknown) => - e instanceof ToolkitError && e.code === "CRAWL_WAIT_TIMEOUT" && e.crawl_id === "c_1" && /not stopped/.test(e.likely_cause), - ); + const c = await waitForCrawl("c_1", { apiKey: "k", fetchImpl: impl, timeoutMs: 1, sleepImpl: async () => {} }); + assert.equal(c.status, "running"); + assert.equal(c.crawl_id, "c_1"); assert.ok(methods.every((m) => m === "GET"), "no stop call on timeout"); }); diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index cff1d62..715edfc 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -80,26 +80,53 @@ async function run(args: string[]): Promise<{ code: number; out: Record }; } -test("crawl start rejects flags it does not declare, before any network call", async () => { +test("crawl create rejects flags it does not declare, before any network call", async () => { await withCrawlWorkspace( () => json(crawlBody, 202), async (calls) => { - for (const flag of ["--pagination", "--discovery", "--json-output", "--wait"]) { - const { code, out } = await run(["start", "https://example.com/products/", "--depth", "1", flag]); + for (const flag of ["--pagination", "--discovery", "--json-output", "--wait", "--html", "--include", "--exclude"]) { + const { code, out } = await run(["create", "https://example.com/products/", "--depth", "1", flag, "x"]); assert.equal(code, 1, flag); assert.equal(out.error.code, "UNKNOWN_FLAG", flag); } + for (const args of [["results", "c_1", "--download"], ["results", "c_1", "--cursor", "p2"]]) { + const { code, out } = await run(args); + assert.equal(code, 1, args.join(" ")); + assert.equal(out.error.code, "UNKNOWN_FLAG", args.join(" ")); + } + assert.equal(calls.length, 0); + }, + ); +}); + +test("the old start and status subcommands are gone", async () => { + for (const sub of ["start", "status"]) { + const { code, out } = await run([sub, "c_1"]); + assert.equal(code, 1, sub); + assert.equal(out.error.code, "INVALID_USAGE", sub); + } +}); + +test("crawl create takes --output-format html only and --timeout in positive seconds", async () => { + await withCrawlWorkspace( + () => json(crawlBody, 202), + async (calls) => { + for (const args of [["--output-format", "json"], ["--follow", "--timeout", "0"], ["--follow", "--timeout", "x"]]) { + const { code, out } = await run(["create", "https://example.com/products/", "--depth", "1", ...args]); + assert.equal(code, 1, args.join(" ")); + assert.equal(out.error.code, "INVALID_USAGE", args.join(" ")); + } assert.equal(calls.length, 0); }, ); }); -test("crawl start requires --depth and a whole number in range", async () => { +test("crawl create requires --depth and a whole number in range", async () => { await withCrawlWorkspace( () => json(crawlBody, 202), async (calls) => { for (const args of [[], ["--depth", "0"], ["--depth", "1.5"], ["--depth", "100001"], ["--depth", "x"]]) { - const { code, out } = await run(["start", "https://example.com/products/", ...args]); + const { code, out } = await run(["create", "https://example.com/products/", ...args]); assert.equal(code, 1, args.join(" ")); assert.equal(out.error.code, "INVALID_USAGE", args.join(" ")); } @@ -108,12 +135,12 @@ test("crawl start requires --depth and a whole number in range", async () => { ); }); -test("crawl start sends only the flags the caller set as the create body", async () => { +test("crawl create sends only the flags the caller set as the create body", async () => { await withCrawlWorkspace( () => json(crawlBody, 202, { location: "/v1/crawls/c_1" }), async (calls) => { const { code, out } = await run([ - "start", + "create", "https://example.com/products/", "--depth", "1", @@ -121,13 +148,14 @@ test("crawl start sends only the flags the caller set as the create body", async "3", "--max-pages", "5", - "--include", + "--include-pattern", "/product/", - "--include", + "--include-pattern", "/item/", - "--exclude", + "--exclude-pattern", "?add", - "--html", + "--output-format", + "html", ]); assert.equal(code, 0); assert.equal(out.ok, true); @@ -148,13 +176,13 @@ test("crawl start sends only the flags the caller set as the create body", async ); }); -test("crawl start honors the domain policy and the page cap before any network call", async () => { +test("crawl create honors the domain policy and the page cap before any network call", async () => { await withCrawlWorkspace( () => json(crawlBody, 202), async (calls) => { - const blocked = await run(["start", "https://blocked.example.com/", "--depth", "1"]); + const blocked = await run(["create", "https://blocked.example.com/", "--depth", "1"]); assert.equal(blocked.code, 1); - const overCap = await run(["start", "https://example.com/products/", "--depth", "1", "--max-pages", "50"]); + const overCap = await run(["create", "https://example.com/products/", "--depth", "1", "--max-pages", "50"]); assert.equal(overCap.code, 1); assert.equal(overCap.out.error.code, "POLICY_LIMIT_EXCEEDED"); assert.equal(calls.length, 0); @@ -163,7 +191,7 @@ test("crawl start honors the domain policy and the page cap before any network c ); }); -test("crawl start --follow waits for the end and exits 1 with CRAWL_FAILED on a failed crawl", async () => { +test("crawl create --follow waits for the end and exits 1 with CRAWL_FAILED on a failed crawl", async () => { await withCrawlWorkspace( (url, init) => init?.method === "POST" @@ -176,7 +204,7 @@ test("crawl start --follow waits for the end and exits 1 with CRAWL_FAILED on a next_cursor: null, }), async (calls) => { - const { code, out } = await run(["start", "https://example.com/products/", "--depth", "1", "--follow"]); + const { code, out } = await run(["create", "https://example.com/products/", "--depth", "1", "--follow"]); assert.equal(code, 1); assert.equal(out.ok, false); assert.equal(out.status, "failed"); @@ -190,7 +218,7 @@ test("crawl start --follow waits for the end and exits 1 with CRAWL_FAILED on a ); }); -test("crawl start surfaces 403 REQS008 as CRAWL_NOT_ENABLED", async () => { +test("crawl create surfaces 403 REQS008 as CRAWL_NOT_ENABLED", async () => { await withCrawlWorkspace( () => json( @@ -199,7 +227,7 @@ test("crawl start surfaces 403 REQS008 as CRAWL_NOT_ENABLED", async () => { { "content-type": "application/problem+json" }, ), async () => { - const { code, out } = await run(["start", "https://example.com/products/", "--depth", "1"]); + const { code, out } = await run(["create", "https://example.com/products/", "--depth", "1"]); assert.equal(code, 1); assert.equal(out.error.code, "CRAWL_NOT_ENABLED"); assert.equal(out.error.message, "Crawl is not enabled for this account."); @@ -207,7 +235,7 @@ test("crawl start surfaces 403 REQS008 as CRAWL_NOT_ENABLED", async () => { ); }); -test("crawl status exits 0 for completed and stopped crawls", async () => { +test("crawl get exits 0 for completed and stopped crawls", async () => { for (const [status, stop_reason] of [ ["completed", "max_items"], ["stopped", "user"], @@ -215,20 +243,23 @@ test("crawl status exits 0 for completed and stopped crawls", async () => { await withCrawlWorkspace( () => json({ ...crawlBody, status, stop_reason, results: [{ url: "https://example.com/product/a" }], next_cursor: "c" }), async () => { - const { code, out } = await run(["status", "c_1"]); + const { code, out } = await run(["get", "c_1"]); assert.equal(code, 0, status); assert.equal(out.ok, true); assert.equal(out.stop_reason, stop_reason); + assert.equal(out.results.length, 1); + assert.equal(out.next_cursor, "c"); + assert.equal(out.crawl.results, undefined); }, ); } }); -test("crawl status on an unknown id exits 1 with CRAWL_NOT_FOUND", async () => { +test("crawl get on an unknown id exits 1 with CRAWL_NOT_FOUND", async () => { await withCrawlWorkspace( () => json({ code: "crawl_not_found", title: "Crawl not found", detail: "No crawl c_x.", status: 404 }, 404), async () => { - const { code, out } = await run(["status", "c_x"]); + const { code, out } = await run(["get", "c_x"]); assert.equal(code, 1); assert.equal(out.error.code, "CRAWL_NOT_FOUND"); }, @@ -242,8 +273,9 @@ test("crawl results reads every page of an ended crawl and writes JSONL with --o }; await withCrawlWorkspace( (url) => json(pages[url.searchParams.get("cursor") ?? ""]), - async () => { - const { code, out } = await run(["results", "c_1"]); + async (calls) => { + const { code, out } = await run(["results", "c_1", "--limit", "500"]); + assert.ok(calls.every((c) => c.url.searchParams.get("limit") === "500"), "--limit is the page size"); assert.equal(code, 0); assert.equal(out.count, 2); assert.deepEqual(out.results.map((r: { url: string }) => r.url), ["https://example.com/product/1", "https://example.com/product/2"]); @@ -275,11 +307,11 @@ test("crawl results on a running crawl returns what is kept with partial: true", ); }); -test("crawl results --cursor reads one page and returns next_cursor while running", async () => { +test("crawl get --cursor reads one page and returns next_cursor while running", async () => { await withCrawlWorkspace( () => json({ ...crawlBody, results: [{ url: "https://example.com/product/3" }], next_cursor: "p4" }), async (calls) => { - const { code, out } = await run(["results", "c_1", "--cursor", "p3", "--limit", "500"]); + const { code, out } = await run(["get", "c_1", "--cursor", "p3", "--limit", "500"]); assert.equal(code, 0); assert.equal(out.next_cursor, "p4"); assert.equal(calls[0]!.url.searchParams.get("cursor"), "p3"); @@ -288,7 +320,7 @@ test("crawl results --cursor reads one page and returns next_cursor while runnin ); }); -test("crawl results --download writes the NDJSON export to .jsonl", async () => { +test("crawl download writes the NDJSON export to .jsonl", async () => { await withCrawlWorkspace( () => new Response('{"url":"https://example.com/product/1","content_status":"fetched","content":""}\n', { @@ -296,9 +328,10 @@ test("crawl results --download writes the NDJSON export to .jsonl", async () headers: { "content-type": "application/x-ndjson", "x-crawl-status": "completed" }, }), async (calls) => { - const { code, out } = await run(["results", "c_1", "--download"]); + const { code, out } = await run(["download", "c_1"]); assert.equal(code, 0); assert.equal(out.count, 1); + assert.equal(out.status, "completed"); assert.equal(out.partial, false); assert.equal(calls[0]!.url.pathname, "/v1/crawls/c_1/download"); assert.ok(existsSync(join(process.cwd(), "c_1.jsonl"))); @@ -394,6 +427,28 @@ test("Ctrl-C during crawl wait says the crawl keeps running and exits 130", asyn } }); +test("crawl wait that runs out exits 0 and prints the running crawl with the resume hint", async () => { + await withCrawlWorkspace( + () => json({ ...crawlBody, results: [], next_cursor: "c" }), + async (calls) => { + const { code, out } = await run(["wait", "c_1", "--timeout", "0.001"]); + assert.equal(code, 0); + assert.equal(out.ok, true); + assert.equal(out.status, "running"); + assert.equal(out.crawlId, "c_1"); + assert.ok(calls.every((c) => c.init?.method === "GET"), "the crawl is not stopped"); + + let human = -1; + const err = await captureErr(async () => { + human = await crawl.run(["wait", "c_1", "--timeout", "0.001"], { json: false, yes: false }); + }); + assert.equal(human, 0); + assert.match(err, /still running/); + assert.match(err, /next: zenrows crawl wait c_1/); + }, + ); +}); + test("unknown crawl subcommand is INVALID_USAGE", async () => { const { code, out } = await run(["discover"]); assert.equal(code, 1); diff --git a/tests/e2e/crawl.e2e.test.ts b/tests/e2e/crawl.e2e.test.ts index c5fe6a5..f3e0584 100644 --- a/tests/e2e/crawl.e2e.test.ts +++ b/tests/e2e/crawl.e2e.test.ts @@ -14,7 +14,7 @@ * Run with `npm run test:e2e`, which does NOT load tests/setup.ts (that file * scrubs every ZENROWS_* variable to keep the unit suite hermetic). * - * The test starts one real, small crawl (bills a few pages on the key's account). + * The test creates one real, small crawl (bills a few pages on the key's account). */ import { test } from "node:test"; import assert from "node:assert/strict"; @@ -32,7 +32,7 @@ const skip = enabled : "set ZENROWS_E2E=1, ZENROWS_API_KEY and ZENROWS_E2E_CRAWL_URL to run the Crawl e2e test"; const BIN = fileURLToPath(new URL("../../bin/zenrows.js", import.meta.url)); -/** How long to keep retrying a start that hits the account's active-crawl limit. */ +/** How long to keep retrying a create that hits the account's active-crawl limit. */ const QUOTA_RETRY_MS = 5 * 60_000; interface CliResult { @@ -70,20 +70,20 @@ function log(msg: string): void { test("zenrows crawl end to end against a live API", { skip, timeout: 20 * 60_000 }, async () => { const dir = mkdtempSync(join(tmpdir(), "zr-crawl-e2e-")); try { - // 1. Start a small crawl and wait for it. Other runs may hold the account's + // 1. Create a small crawl and wait for it. Other runs may hold the account's // active-job slots: on CRAWL_TOO_MANY_CRAWLS wait retry_after and try again. const deadline = Date.now() + QUOTA_RETRY_MS; let start: CliResult; for (;;) { start = await cli(dir, [ - "crawl", "start", START_URL!, + "crawl", "create", START_URL!, "--depth", "1", "--max-items", "3", "--max-pages", "5", - ...(INCLUDE ? ["--include", INCLUDE] : []), - "--html", "--follow", "--timeout", "600000", + ...(INCLUDE ? ["--include-pattern", INCLUDE] : []), + "--output-format", "html", "--follow", "--timeout", "600", ]); if (start.out.error?.code !== "CRAWL_TOO_MANY_CRAWLS" || Date.now() > deadline) break; const wait = Number(start.out.error.retry_after ?? 30); - log(`too_many_crawls: retrying start in ${wait}s`); + log(`too_many_crawls: retrying create in ${wait}s`); await new Promise((r) => setTimeout(r, wait * 1000)); } assert.equal(start.code, 0, JSON.stringify(start.out.error ?? start.out)); @@ -97,7 +97,7 @@ test("zenrows crawl end to end against a live API", { skip, timeout: 20 * 60_000 assert.equal(results.code, 0, JSON.stringify(results.out)); const rows = results.out.results as Array<{ url: string; content_status?: string; content_url?: string }>; assert.ok(rows.length >= 1, "at least one result"); - if (INCLUDE) for (const r of rows) assert.ok(r.url.includes(INCLUDE), `${r.url} matches --include`); + if (INCLUDE) for (const r of rows) assert.ok(r.url.includes(INCLUDE), `${r.url} matches --include-pattern`); log(`results: ${rows.length} URL(s)${INCLUDE ? ", all matching the include pattern" : ""}`); // 3. One fetched page's HTML. @@ -111,12 +111,12 @@ test("zenrows crawl end to end against a live API", { skip, timeout: 20 * 60_000 // 4. The NDJSON export has one line per result. const file = join(dir, "export.jsonl"); - const download = await cli(dir, ["crawl", "results", id, "--download", "--out", file]); + const download = await cli(dir, ["crawl", "download", id, "--out", file]); assert.equal(download.code, 0, JSON.stringify(download.out)); const lines = readFileSync(file, "utf8").split("\n").filter((l) => l.trim()); assert.equal(lines.length, rows.length); assert.equal(download.out.count, rows.length); - log(`download: ${lines.length} NDJSON line(s), X-Crawl-Status ${download.out.crawlStatus}`); + log(`download: ${lines.length} NDJSON line(s), X-Crawl-Status ${download.out.status}`); // 5. The new crawl is listed. const list = await cli(dir, ["crawl", "list", "--limit", "100"]); @@ -131,10 +131,10 @@ test("zenrows crawl end to end against a live API", { skip, timeout: 20 * 60_000 log(`stop on ended crawl: status ${stop.out.status}`); // 7. An unknown id is CRAWL_NOT_FOUND. - const missing = await cli(dir, ["crawl", "status", "c_does_not_exist_e2e"]); + const missing = await cli(dir, ["crawl", "get", "c_does_not_exist_e2e"]); assert.equal(missing.code, 1); assert.equal(missing.out.error.code, "CRAWL_NOT_FOUND"); - log("status on unknown id: CRAWL_NOT_FOUND"); + log("get on unknown id: CRAWL_NOT_FOUND"); } finally { rmSync(dir, { recursive: true, force: true }); } From 1f62eda95e6a7d49b49d8e98f13207d0faba6f81 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Fri, 9 Oct 2026 17:49:24 +0200 Subject: [PATCH 05/19] fix(crawl): print snake_case JSON in one shape for every surface Every --json key is now snake_case: crawl_id, content_id and content_type replace crawlId, contentId and contentType. The run artifact keeps crawlId, as Batch keeps jobId. create, get, wait and stop print {ok, crawl_id, status, crawl}. get adds results and next_cursor. stop_reason is no longer at the top level; read it under crawl. A wait that runs out with the crawl still running now adds note under --json, with the command to wait again. download prints status: null when the API sends no X-Crawl-Status. The help, the README and the crawl skill describe the JSON shape. --- README.md | 13 +++++++++---- skills/crawl/SKILL.md | 8 +++++++- src/cli/commands/crawl.ts | 38 ++++++++++++++++++++----------------- tests/crawl-command.test.ts | 28 +++++++++++++++++++++++---- tests/e2e/crawl.e2e.test.ts | 8 ++++---- 5 files changed, 65 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 7902411..62df238 100644 --- a/README.md +++ b/README.md @@ -189,13 +189,18 @@ more fetch, and `--max-pages` counts it. `--include-pattern` / `--exclude-pattern` are repeatable substring filters on the URL. `get` and `list` read one page and take `--cursor` and `--limit`. `results` follows every page (`--limit` is the page size); on a running crawl it returns the URLs kept -so far, with `partial: true`. `download` reports the crawl's `status`, and -`partial: true` while it runs. +so far, with `partial: true`. `download` reports the crawl's `status` (`null` +when the API sends none), and `partial: true` while it runs. + +With `--json`, `create`, `get`, `wait` and `stop` print +`{ok, crawl_id, status, crawl}`. The crawl object (with `coverage`, +`stop_reason` and `error`) is under `crawl`. `get` adds `results` and +`next_cursor`. All keys are snake_case. `wait`, and `create --follow`, poll until the crawl ends or `--timeout` seconds run out (default 600). When the time runs out, the command exits 0 and prints -the crawl with status `running` and the hint `zenrows crawl wait `; -the crawl keeps running. Ctrl-C stops the wait, not the crawl, and exits 130; +the crawl with status `running` and the hint `zenrows crawl wait ` +(under `--json`, in `note`); the crawl keeps running. Ctrl-C stops the wait, not the crawl, and exits 130; resume with `zenrows crawl wait `. A crawl that ends `failed` exits 1 with `CRAWL_FAILED`. diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index 2aba854..c74bdd0 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -39,7 +39,8 @@ zenrows crawl results --out urls.jsonl # same, as JSONL - `create` returns at once. `--follow` (or `zenrows crawl wait `) polls until the crawl ends or `--timeout` seconds run out (default 600). When the time runs out, the command exits 0 and prints the crawl with status - `running` and the hint `zenrows crawl wait `. The crawl keeps running; + `running` and the hint `zenrows crawl wait ` (under `--json`, in + `note`). The crawl keeps running; it is not stopped. Ctrl-C also stops the wait, not the crawl, and exits 130; resume with `zenrows crawl wait `. - `results` on a running crawl returns the URLs kept so far, with @@ -75,6 +76,11 @@ A crawl ends `completed` (nothing left, or a cap hit: `stop_reason` and `error.detail` say why). Only `failed` exits non-zero (`CRAWL_FAILED`). `stop` on a crawl that already ended prints the status it ended with. +With `--json`, `create`, `get`, `wait` and `stop` print +`{ok, crawl_id, status, crawl}`; read `coverage`, `stop_reason` and `error` +under `crawl`. `get` adds `results` and `next_cursor`. `download` prints +`status: null` when the API sends no status. + ## Limits When the account has reached its limit of active jobs (3 by default), shared diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index e3bd52d..c947643 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -71,7 +71,9 @@ export const crawl: Command = { " stop stop a running crawl (an ended crawl answers as it ended)", " wait [--timeout ] poll until the crawl ends or s seconds (default 600) run out;", " a crawl still running then is printed, not stopped", - " --json print structured output", + " --json print structured output: create/get/wait/stop print", + " {ok, crawl_id, status, crawl}; get adds results and next_cursor;", + " a wait that runs out adds note", ].join("\n"), async run(argv: string[], ctx: RunContext): Promise { const [sub, ...rest] = argv; @@ -204,9 +206,10 @@ async function createCmd(rest: string[], ctx: RunContext): Promise { result: { crawlId: created.crawl_id, status: finished.status, coverage: finished.coverage }, ...(failure ? { error: failure.toJSON() } : {}), }); - const code = printCrawl(finished, json, follow ? waitHeadline(finished) : `Started crawl ${created.crawl_id}`); + const code = follow + ? printCrawl(finished, json, waitHeadline(finished), { note: waitNote(finished) }) + : printCrawl(finished, json, `Started crawl ${created.crawl_id}`); if (!json && runDir) log.dim(` artifact: ${runDir}`); - printWaitHint(finished, json); return code; } catch (err) { writeRun({ @@ -256,17 +259,17 @@ async function waitCmd(rest: string[], ctx: RunContext): Promise { assertUsable("crawl"); const apiKey = requireApiKey(); const c = await waitUntilEnd(id, { apiKey, timeoutMs }); - const code = printCrawl(c, json, waitHeadline(c)); - printWaitHint(c, json); - return code; + return printCrawl(c, json, waitHeadline(c), { note: waitNote(c) }); } function waitHeadline(c: Crawl): string { return isTerminal(c.status) ? `Crawl ${c.crawl_id} finished` : `Stopped waiting: crawl ${c.crawl_id} is still running`; } -function printWaitHint(c: Crawl, json: boolean): void { - if (!json && !isTerminal(c.status)) log.dim(` next: zenrows crawl wait ${c.crawl_id}`); +/** Set only when a wait ran out with the crawl still running. */ +function waitNote(c: Crawl): string | undefined { + if (isTerminal(c.status)) return undefined; + return `The wait ran out and the crawl is still running. Run zenrows crawl wait ${c.crawl_id}.`; } async function resultsCmd(rest: string[], ctx: RunContext): Promise { @@ -297,7 +300,7 @@ async function downloadCmd(rest: string[], ctx: RunContext): Promise { const count = ndjson.split("\n").filter((l) => l.trim()).length; const partial = status === "running"; if (json) { - log.out(JSON.stringify({ ok: true, crawlId: id, status, partial, count, file }, null, 2)); + log.out(JSON.stringify({ ok: true, crawl_id: id, status: status ?? null, partial, count, file }, null, 2)); } else { log.success(`Downloaded ${count} result(s) → ${file}`); if (partial) log.warn(" The crawl is still running: this file is partial. Download again once it ends."); @@ -315,7 +318,7 @@ function printResults( if (o.outFile) writeOut(o.outFile, toJsonl(results)); if (o.json) { const rows = o.outFile ? { file: o.outFile } : { results }; - log.out(JSON.stringify({ ok: true, crawlId: id, status, count: results.length, ...rows, partial: o.partial }, null, 2)); + log.out(JSON.stringify({ ok: true, crawl_id: id, status, count: results.length, ...rows, partial: o.partial }, null, 2)); return 0; } if (o.outFile) { @@ -352,12 +355,12 @@ async function contentCmd(rest: string[], ctx: RunContext): Promise { if (outFile) { writeOut(outFile, body); if (json) { - log.out(JSON.stringify({ ok: true, crawlId: id, contentId, contentType, bytes: Buffer.byteLength(body), file: outFile }, null, 2)); + log.out(JSON.stringify({ ok: true, crawl_id: id, content_id: contentId, content_type: contentType, bytes: Buffer.byteLength(body), file: outFile }, null, 2)); } else { log.success(`Wrote ${Buffer.byteLength(body)} bytes (${contentType || "unknown type"}) → ${outFile}`); } } else if (json) { - log.out(JSON.stringify({ ok: true, crawlId: id, contentId, contentType, bytes: Buffer.byteLength(body), content: body }, null, 2)); + log.out(JSON.stringify({ ok: true, crawl_id: id, content_id: contentId, content_type: contentType, bytes: Buffer.byteLength(body), content: body }, null, 2)); } else { log.out(body); } @@ -444,13 +447,14 @@ export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null { /** * Print a crawl (or a stop answer), structured under --json, and return the exit * code: 1 when it ended `failed`, else 0. `stopped` is a deliberate outcome, so - * it exits 0 but prints as a warning. `page` carries the results a `get` read. + * it exits 0 but prints as a warning. `extra` carries the results a `get` read, + * or the note of a wait that ran out. */ function printCrawl( c: Crawl | CrawlStop, json: boolean, headline: string, - page?: { results: CrawlResult[]; next_cursor: string | null }, + extra: { results?: CrawlResult[]; next_cursor?: string | null; note?: string } = {}, ): number { const err = crawlFailure(c); const crawlFields = { ...c } as Record; @@ -462,11 +466,10 @@ function printCrawl( JSON.stringify( { ok: !err, - crawlId: c.crawl_id, + crawl_id: c.crawl_id, status: c.status, - ...(c.stop_reason ? { stop_reason: c.stop_reason } : {}), crawl: crawlFields, - ...page, + ...extra, ...(err ? { error: err.toJSON() } : {}), }, null, @@ -483,6 +486,7 @@ function printCrawl( if (cov) { log.info(` ${cov.items_found} item(s) found · ${cov.pages_fetched} page(s) fetched · ${cov.pages_failed} failed`); } + if (extra.note) log.dim(` next: zenrows crawl wait ${c.crawl_id}`); if (err) printError(err, false); return err ? 1 : 0; } diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index 715edfc..0512620 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -159,7 +159,8 @@ test("crawl create sends only the flags the caller set as the create body", asyn ]); assert.equal(code, 0); assert.equal(out.ok, true); - assert.equal(out.crawlId, "c_1"); + assert.deepEqual(Object.keys(out), ["ok", "crawl_id", "status", "crawl"]); + assert.equal(out.crawl_id, "c_1"); assert.equal(out.status, "running"); assert.equal(calls.length, 1); assert.equal(calls[0]!.url.pathname, "/v1/crawls"); @@ -246,7 +247,8 @@ test("crawl get exits 0 for completed and stopped crawls", async () => { const { code, out } = await run(["get", "c_1"]); assert.equal(code, 0, status); assert.equal(out.ok, true); - assert.equal(out.stop_reason, stop_reason); + assert.deepEqual(Object.keys(out), ["ok", "crawl_id", "status", "crawl", "results", "next_cursor"]); + assert.equal(out.crawl.stop_reason, stop_reason); assert.equal(out.results.length, 1); assert.equal(out.next_cursor, "c"); assert.equal(out.crawl.results, undefined); @@ -284,7 +286,7 @@ test("crawl results reads every page of an ended crawl and writes JSONL with --o const file = join(process.cwd(), "urls.jsonl"); const written = await run(["results", "c_1", "--out", file]); - assert.deepEqual(written.out, { ok: true, crawlId: "c_1", status: "completed", count: 2, file, partial: false }); + assert.deepEqual(written.out, { ok: true, crawl_id: "c_1", status: "completed", count: 2, file, partial: false }); assert.equal(readFileSync(file, "utf8"), '{"url":"https://example.com/product/1"}\n{"url":"https://example.com/product/2"}\n'); }, ); @@ -333,12 +335,25 @@ test("crawl download writes the NDJSON export to .jsonl", async () => { assert.equal(out.count, 1); assert.equal(out.status, "completed"); assert.equal(out.partial, false); + assert.equal(out.crawl_id, "c_1"); assert.equal(calls[0]!.url.pathname, "/v1/crawls/c_1/download"); assert.ok(existsSync(join(process.cwd(), "c_1.jsonl"))); }, ); }); +test("crawl download prints status: null when X-Crawl-Status is absent", async () => { + await withCrawlWorkspace( + () => new Response('{"url":"https://example.com/product/1"}\n', { status: 200 }), + async () => { + const { code, out } = await run(["download", "c_1"]); + assert.equal(code, 0); + assert.equal(out.status, null); + assert.equal(out.partial, false); + }, + ); +}); + test("crawl content accepts a content_url and prints the page", async () => { await withCrawlWorkspace( () => new Response("ok", { status: 200, headers: { "content-type": "text/html" } }), @@ -346,6 +361,9 @@ test("crawl content accepts a content_url and prints the page", async () => { const { code, out } = await run(["content", "c_1", "/v1/crawls/c_1/contents/ct_9"]); assert.equal(code, 0); assert.equal(out.content, "ok"); + assert.equal(out.crawl_id, "c_1"); + assert.equal(out.content_id, "ct_9"); + assert.equal(out.content_type, "text/html"); assert.equal(calls[0]!.url.pathname, "/v1/crawls/c_1/contents/ct_9"); }, ); @@ -367,6 +385,7 @@ test("crawl list and stop call the right endpoints", async () => { const stop = await run(["stop", "c_1"]); assert.equal(stop.code, 0); assert.equal(stop.out.status, "stopped"); + assert.deepEqual(Object.keys(stop.out), ["ok", "crawl_id", "status", "crawl"]); assert.equal(calls[1]!.url.pathname, "/v1/crawls/c_1/stop"); }, ); @@ -435,7 +454,8 @@ test("crawl wait that runs out exits 0 and prints the running crawl with the res assert.equal(code, 0); assert.equal(out.ok, true); assert.equal(out.status, "running"); - assert.equal(out.crawlId, "c_1"); + assert.equal(out.crawl_id, "c_1"); + assert.equal(out.note, "The wait ran out and the crawl is still running. Run zenrows crawl wait c_1."); assert.ok(calls.every((c) => c.init?.method === "GET"), "the crawl is not stopped"); let human = -1; diff --git a/tests/e2e/crawl.e2e.test.ts b/tests/e2e/crawl.e2e.test.ts index f3e0584..f779074 100644 --- a/tests/e2e/crawl.e2e.test.ts +++ b/tests/e2e/crawl.e2e.test.ts @@ -87,10 +87,10 @@ test("zenrows crawl end to end against a live API", { skip, timeout: 20 * 60_000 await new Promise((r) => setTimeout(r, wait * 1000)); } assert.equal(start.code, 0, JSON.stringify(start.out.error ?? start.out)); - const id = start.out.crawlId as string; + const id = start.out.crawl_id as string; assert.match(id, /\S/); assert.equal(start.out.status, "completed"); - log(`crawl: ${start.out.status} (${start.out.stop_reason ?? "nothing left"})`); + log(`crawl: ${start.out.status} (${start.out.crawl.stop_reason ?? "nothing left"})`); // 2. Every kept URL, across all pages. const results = await cli(dir, ["crawl", "results", id]); @@ -105,9 +105,9 @@ test("zenrows crawl end to end against a live API", { skip, timeout: 20 * 60_000 assert.ok(fetched, "at least one result has fetched content"); const content = await cli(dir, ["crawl", "content", id, fetched.content_url!]); assert.equal(content.code, 0, JSON.stringify(content.out)); - assert.match(content.out.contentType, /text\/html/); + assert.match(content.out.content_type, /text\/html/); assert.match(content.out.content, / Date: Fri, 9 Oct 2026 18:01:13 +0200 Subject: [PATCH 06/19] fix(crawl): retry transient Crawl failures as the SDKs do Crawl calls now retry up to 3 times. The wait is the Retry-After seconds, or 250 ms x 2^n with +/-20% jitter, capped at 10 s. - GET requests retry on 429, 502, 503, 504 and network errors, including the per-request timeout. - create retries only with an Idempotency-Key, and only on 502, 503, 504 and network errors. It never retries 429. - stop never retries. Batch and other commands do not change. The README and the crawl skill describe the policy. --- README.md | 6 ++ skills/crawl/SKILL.md | 7 ++ src/core/crawl-api.ts | 166 ++++++++++++++++++++++------------------ tests/crawl-api.test.ts | 96 ++++++++++++++++++++++- 4 files changed, 198 insertions(+), 77 deletions(-) diff --git a/README.md b/README.md index 62df238..cbedcb8 100644 --- a/README.md +++ b/README.md @@ -215,6 +215,12 @@ same claim, top-up and key-cap advice as other commands), `CRAWL_CONTENT_NOT_FOUND` and `CRAWL_FAILED`. Set `ZENROWS_CRAWL_API_BASE` to point the CLI at another deployment. +The CLI retries transient failures, as the Crawl SDKs do: up to 3 retries, +waiting the `Retry-After` seconds or 250 ms x 2^n (+/-20% jitter, at most 10 s). +Reads retry on 429, 502, 503, 504 and network errors (a timeout counts). +`create` retries only with `--idempotency-key`, and only on 502, 503, 504 and +network errors, never on 429. `stop` never retries. + ## 11. Browser Sessions Escalation only — **prefer `fetch`/`extract` for the vast majority of cases**; diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index c74bdd0..bb4472b 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -93,6 +93,13 @@ seconds (about 30), or stop one of your crawls or Batch jobs first. Every Crawl error carries the API's code as `server_code` (when the API sent one), and `crawl_id` on any call about one crawl. +The CLI retries transient failures, as the Crawl SDKs do: up to 3 retries, +waiting the `Retry-After` seconds or 250 ms x 2^n (+/-20% jitter, at most 10 s). +Reads retry on 429, 502, 503, 504 and network errors (a timeout counts). +`create` retries only with `--idempotency-key`, and only on 502, 503, 504 and +network errors, never on 429. `stop` never retries. +The "Retry?" column is for errors that remain after these retries. + | Code | Meaning | Retry? | | --- | --- | --- | | `CRAWL_NOT_ENABLED` | 403 `REQS008`: Crawl is not enabled for this account | no | diff --git a/src/core/crawl-api.ts b/src/core/crawl-api.ts index 6ec4f0e..57067a7 100644 --- a/src/core/crawl-api.ts +++ b/src/core/crawl-api.ts @@ -6,7 +6,8 @@ * param: Crawl answers any query parameter it does not know with 400. Bodies are * JSON; errors come back as `application/problem+json` (RFC 9457) and we branch * on `status` + `code` only. A crawl is an async job: create it, then poll it - * until its status leaves `running`. `fetchImpl` / `sleepImpl` are injectable + * until its status leaves `running`. Calls retry transient failures the way the + * Crawl SDKs do (see `retryStatuses`). `fetchImpl` / `sleepImpl` are injectable * for tests. * * `output_format` takes `html` only (absent = URLs only). Response fields we do @@ -130,14 +131,19 @@ export function isTerminal(status: string | undefined): boolean { return !!status && status !== "running"; } -interface RequestOpts { +interface CallOpts { apiKey: string; + fetchImpl?: typeof fetch; + /** Per HTTP attempt. */ + timeoutMs?: number; + sleepImpl?: (ms: number) => Promise; +} + +interface RequestOpts extends CallOpts { body?: unknown; query?: Record; headers?: Record; accept?: string; - fetchImpl?: typeof fetch; - timeoutMs?: number; /** Set on every call about one crawl, so its errors carry `crawl_id`. */ crawlId?: string; } @@ -148,10 +154,49 @@ interface RawResponse { text: string; } +const MAX_RETRIES = 3; +const READ_RETRY_STATUSES: ReadonlySet = new Set([429, 502, 503, 504]); +// 429 on create is the active-job limit: a retry only waits for a slot that may never free. +const CREATE_RETRY_STATUSES: ReadonlySet = new Set([502, 503, 504]); + +/** + * The statuses to retry, or undefined when the request must run once: a GET is + * safe to repeat, a create only with an Idempotency-Key, and stop never. + */ +function retryStatuses(method: string, headers: Record): ReadonlySet | undefined { + if (method === "GET") return READ_RETRY_STATUSES; + return headers["Idempotency-Key"] ? CREATE_RETRY_STATUSES : undefined; +} + +/** 250 ms x 2^attempt, capped at 10 s, +/-20% jitter. */ +function backoffMs(attempt: number): number { + return Math.min(250 * 2 ** attempt, 10_000) * (0.8 + Math.random() * 0.4); +} + +/** `Retry-After` in whole seconds, as milliseconds. */ +function retryAfterMs(value: string | null): number | undefined { + return value && /^\d+$/.test(value.trim()) ? Number(value) * 1000 : undefined; +} + +const defaultSleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +/** One HTTP attempt with its own timeout; the body is read inside it. */ +async function attempt(url: string, init: RequestInit, opts: RequestOpts): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), opts.timeoutMs ?? 60_000); + try { + const res = await (opts.fetchImpl ?? fetch)(url, { ...init, signal: controller.signal }); + return { status: res.status, headers: res.headers, text: await res.text() }; + } finally { + clearTimeout(timeout); + } +} + /** * Perform a Crawl API request and return the raw body. Sets `X-API-Key`, adds a - * JSON content-type when a body is present, and on a non-2xx parses the - * problem+json body into a normalized ToolkitError. + * JSON content-type when a body is present, retries transient failures as + * `retryStatuses` allows, and on a non-2xx parses the problem+json body into a + * normalized ToolkitError. */ async function crawlFetch(method: string, path: string, opts: RequestOpts): Promise { registerSecret(opts.apiKey); @@ -159,10 +204,6 @@ async function crawlFetch(method: string, path: string, opts: RequestOpts): Prom for (const [k, v] of Object.entries(opts.query ?? {})) { if (v !== undefined && v !== null) url.searchParams.set(k, String(v)); } - const doFetch = opts.fetchImpl ?? fetch; - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), opts.timeoutMs ?? 60_000); - const headers: Record = { "X-API-Key": opts.apiKey, Accept: opts.accept ?? "application/json", @@ -171,35 +212,40 @@ async function crawlFetch(method: string, path: string, opts: RequestOpts): Prom ...(opts.headers ?? {}), }; if (opts.body !== undefined) headers["Content-Type"] = "application/json"; - - let res: Response; - let text: string; - try { - res = await doFetch(url.toString(), { - method, - headers, - body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined, - signal: controller.signal, - }); - text = await res.text(); - } catch (err) { - throw new ToolkitError({ - code: "BACKEND_UNAVAILABLE", - message: "Could not reach the Zenrows Crawl API.", - likely_cause: err instanceof Error ? err.message : String(err), - next_action: - "Check connectivity and retry. Override the host with ZENROWS_CRAWL_API_BASE if you are testing against staging.", - suggested_commands: ["zenrows status"], - crawl_id: opts.crawlId, - }); - } finally { - clearTimeout(timeout); - } - - if (res.status < 200 || res.status >= 300) { - throw problemToError(res.status, text, `${method} ${path}`, res.headers.get("retry-after"), opts.crawlId); + const init: RequestInit = { method, headers, body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined }; + const retryOn = retryStatuses(method, headers); + const sleep = opts.sleepImpl ?? defaultSleep; + + for (let n = 0; ; n++) { + const canRetry = retryOn !== undefined && n < MAX_RETRIES; + let res: RawResponse; + try { + res = await attempt(url.toString(), init, opts); + } catch (err) { + if (canRetry) { + await sleep(backoffMs(n)); + continue; + } + throw new ToolkitError({ + code: "BACKEND_UNAVAILABLE", + message: "Could not reach the Zenrows Crawl API.", + likely_cause: err instanceof Error ? err.message : String(err), + next_action: + "Check connectivity and retry. Override the host with ZENROWS_CRAWL_API_BASE if you are testing against staging.", + suggested_commands: ["zenrows status"], + crawl_id: opts.crawlId, + }); + } + const retryAfter = res.headers.get("retry-after"); + if (canRetry && retryOn.has(res.status)) { + await sleep(retryAfterMs(retryAfter) ?? backoffMs(n)); + continue; + } + if (res.status < 200 || res.status >= 300) { + throw problemToError(res.status, res.text, `${method} ${path}`, retryAfter, opts.crawlId); + } + return res; } - return { status: res.status, headers: res.headers, text }; } /** `crawlFetch` + JSON parse. */ @@ -336,12 +382,6 @@ function classifyProblem( }); } -interface CallOpts { - apiKey: string; - fetchImpl?: typeof fetch; - timeoutMs?: number; -} - /** Start a crawl (202). Sends only the fields set in `params`. */ export function createCrawl(params: CreateCrawlParams, opts: CallOpts & { idempotencyKey?: string }): Promise { const body: Record = { url: params.url, depth: params.depth }; @@ -351,43 +391,29 @@ export function createCrawl(params: CreateCrawlParams, opts: CallOpts & { idempo if (params.exclude_patterns?.length) body.exclude_patterns = params.exclude_patterns; if (params.output_format !== undefined) body.output_format = params.output_format; return crawlJson("POST", "/crawls", { - apiKey: opts.apiKey, + ...opts, body, headers: opts.idempotencyKey ? { "Idempotency-Key": opts.idempotencyKey } : undefined, - fetchImpl: opts.fetchImpl, - timeoutMs: opts.timeoutMs, }); } /** Read a crawl and one page of its results. */ export function getCrawl(id: string, opts: CallOpts & { cursor?: string; limit?: number }): Promise { return crawlJson("GET", `/crawls/${encodeURIComponent(id)}`, { - apiKey: opts.apiKey, + ...opts, crawlId: id, query: { cursor: opts.cursor, limit: opts.limit }, - fetchImpl: opts.fetchImpl, - timeoutMs: opts.timeoutMs, }); } /** One page of the account's crawls, newest first. */ export function listCrawls(opts: CallOpts & { cursor?: string; limit?: number }): Promise { - return crawlJson("GET", "/crawls", { - apiKey: opts.apiKey, - query: { cursor: opts.cursor, limit: opts.limit }, - fetchImpl: opts.fetchImpl, - timeoutMs: opts.timeoutMs, - }); + return crawlJson("GET", "/crawls", { ...opts, query: { cursor: opts.cursor, limit: opts.limit } }); } /** Stop a running crawl. Idempotent: an ended crawl answers as it ended. */ export function stopCrawl(id: string, opts: CallOpts): Promise { - return crawlJson("POST", `/crawls/${encodeURIComponent(id)}/stop`, { - apiKey: opts.apiKey, - crawlId: id, - fetchImpl: opts.fetchImpl, - timeoutMs: opts.timeoutMs, - }); + return crawlJson("POST", `/crawls/${encodeURIComponent(id)}/stop`, { ...opts, crawlId: id }); } /** The page of one kept URL (HTML for `output_format: html`). */ @@ -397,11 +423,9 @@ export async function getCrawlContent( opts: CallOpts, ): Promise<{ contentType: string; body: string }> { const res = await crawlFetch("GET", `/crawls/${encodeURIComponent(id)}/contents/${encodeURIComponent(contentId)}`, { - apiKey: opts.apiKey, + ...opts, crawlId: id, accept: "text/html, application/json;q=0.9, application/problem+json;q=0.8", - fetchImpl: opts.fetchImpl, - timeoutMs: opts.timeoutMs, }); return { contentType: res.headers.get("content-type") ?? "", body: res.text }; } @@ -412,10 +436,9 @@ export async function getCrawlContent( */ export async function downloadCrawl(id: string, opts: CallOpts): Promise<{ status?: string; ndjson: string }> { const res = await crawlFetch("GET", `/crawls/${encodeURIComponent(id)}/download`, { - apiKey: opts.apiKey, + ...opts, crawlId: id, accept: "application/x-ndjson, application/problem+json;q=0.8", - fetchImpl: opts.fetchImpl, timeoutMs: opts.timeoutMs ?? 300_000, }); return { status: res.headers.get("x-crawl-status") ?? undefined, ndjson: res.text }; @@ -458,15 +481,12 @@ export async function listAllResults( * `timeoutMs` (default 600 s) runs out, backing off 2s → ×1.5 → capped at 15s. * A timeout is not an error: it returns the crawl, still `running`. */ -export async function waitForCrawl( - id: string, - opts: CallOpts & { sleepImpl?: (ms: number) => Promise }, -): Promise { - const sleep = opts.sleepImpl ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); +export async function waitForCrawl(id: string, opts: CallOpts): Promise { + const sleep = opts.sleepImpl ?? defaultSleep; const deadline = Date.now() + (opts.timeoutMs ?? 600_000); let delay = 2000; for (;;) { - const crawl = await getCrawl(id, { apiKey: opts.apiKey, fetchImpl: opts.fetchImpl, limit: 1 }); + const crawl = await getCrawl(id, { apiKey: opts.apiKey, fetchImpl: opts.fetchImpl, sleepImpl: opts.sleepImpl, limit: 1 }); if (isTerminal(crawl.status)) return crawl; if (Date.now() + delay > deadline) return crawl; await sleep(delay); diff --git a/tests/crawl-api.test.ts b/tests/crawl-api.test.ts index f325630..d95000c 100644 --- a/tests/crawl-api.test.ts +++ b/tests/crawl-api.test.ts @@ -41,6 +41,27 @@ function stubFetch(status: number, payload: unknown, headers: Record void waits.push(ms) }; +} + +/** Serve `responses` in order (a thrown Error is a transport failure), recording each call. */ +function seqFetch(responses: Array<{ status: number; payload?: unknown; headers?: Record } | Error>) { + const calls: Array<{ url: string; init?: RequestInit }> = []; + const impl = (async (url: string, init?: RequestInit) => { + calls.push({ url, init }); + const r = responses[Math.min(calls.length - 1, responses.length - 1)]!; + if (r instanceof Error) throw r; + return new Response(JSON.stringify(r.payload ?? {}), { + status: r.status, + headers: { "content-type": "application/json", ...r.headers }, + }); + }) as unknown as typeof fetch; + return { impl, calls }; +} + function problem(status: number, code: string, detail = "nope", headers: Record = {}) { return stubFetch(status, { code, title: code, detail, status, instance: "urn:zenrows:request:x" }, { "content-type": "application/problem+json", @@ -220,10 +241,14 @@ test("409 maps to CRAWL_REQUEST_IN_FLIGHT", async () => { ); }); -test("429 too_many_crawls maps to CRAWL_TOO_MANY_CRAWLS with retry_after and no retry", async () => { +test("429 too_many_crawls maps to CRAWL_TOO_MANY_CRAWLS with retry_after, and create never retries it, even with a key", async () => { const { impl, calls } = problem(429, "too_many_crawls", "Too many crawls.", { "retry-after": "30" }); await assert.rejects( - () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: impl, idempotencyKey: "k1" }), + () => + createCrawl( + { url: "https://example.com/products/", depth: 1 }, + { apiKey: "k", fetchImpl: impl, idempotencyKey: "k1", sleepImpl: recordSleep().sleepImpl }, + ), (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_TOO_MANY_CRAWLS" && @@ -256,7 +281,7 @@ test("402 maps to CRAWL_QUOTA_EXCEEDED, and AUTH014 to CRAWL_KEY_CAP_REACHED, wi test("an error body without a code leaves server_code unset", async () => { const { impl } = stubFetch(500, "upstream exploded"); await assert.rejects( - () => getCrawl("c_1", { apiKey: "k", fetchImpl: impl }), + () => getCrawl("c_1", { apiKey: "k", fetchImpl: impl, sleepImpl: recordSleep().sleepImpl }), (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_FAILED" && !("server_code" in e.toJSON()), ); }); @@ -272,10 +297,73 @@ test("a transport failure maps to BACKEND_UNAVAILABLE", async () => { const impl = (async () => { throw new TypeError("fetch failed"); }) as unknown as typeof fetch; + const { waits, sleepImpl } = recordSleep(); await assert.rejects( - () => getCrawl("c", { apiKey: "k", fetchImpl: impl }), + () => getCrawl("c", { apiKey: "k", fetchImpl: impl, sleepImpl }), (e: unknown) => e instanceof ToolkitError && e.code === "BACKEND_UNAVAILABLE" && e.crawl_id === "c", ); + assert.equal(waits.length, 3, "a GET retries a transport failure 3 times before it gives up"); +}); + +const unavailable = { status: 503, payload: { code: "unavailable", status: 503 } }; + +test("a GET is retried on 503 with jittered backoff, then succeeds", async () => { + const { impl, calls } = seqFetch([unavailable, unavailable, { status: 200, payload: { ...crawlBody, results: [], next_cursor: null } }]); + const { waits, sleepImpl } = recordSleep(); + const c = await getCrawl("c_1", { apiKey: "k", fetchImpl: impl, sleepImpl }); + assert.equal(c.crawl_id, "c_1"); + assert.equal(calls.length, 3); + assert.equal(waits.length, 2); + assert.ok(waits[0]! >= 200 && waits[0]! <= 300, `first wait ~250 ms, got ${waits[0]}`); + assert.ok(waits[1]! >= 400 && waits[1]! <= 600, `second wait ~500 ms, got ${waits[1]}`); +}); + +test("a GET gives up after 3 retries and reports the last error", async () => { + const { impl, calls } = seqFetch([unavailable]); + await assert.rejects( + () => listCrawls({ apiKey: "k", fetchImpl: impl, sleepImpl: recordSleep().sleepImpl }), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_FAILED" && e.server_code === "unavailable", + ); + assert.equal(calls.length, 4); +}); + +test("a retry honors Retry-After in seconds", async () => { + const { impl } = seqFetch([ + { status: 429, payload: { code: "rate_limited" }, headers: { "retry-after": "7" } }, + { status: 200, payload: { crawls: [] } }, + ]); + const { waits, sleepImpl } = recordSleep(); + await listCrawls({ apiKey: "k", fetchImpl: impl, sleepImpl }); + assert.deepEqual(waits, [7000]); +}); + +test("create without an Idempotency-Key is not retried on 503", async () => { + const { impl, calls } = seqFetch([unavailable, { status: 202, payload: crawlBody }]); + await assert.rejects( + () => createCrawl({ url: "https://example.com/", depth: 1 }, { apiKey: "k", fetchImpl: impl, sleepImpl: recordSleep().sleepImpl }), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_FAILED", + ); + assert.equal(calls.length, 1); +}); + +test("create with an Idempotency-Key is retried on 503 with the same key", async () => { + const { impl, calls } = seqFetch([unavailable, { status: 202, payload: crawlBody }]); + const c = await createCrawl( + { url: "https://example.com/", depth: 1 }, + { apiKey: "k", fetchImpl: impl, sleepImpl: recordSleep().sleepImpl, idempotencyKey: "k1" }, + ); + assert.equal(c.crawl_id, "c_1"); + assert.equal(calls.length, 2); + assert.ok(calls.every((c) => (c.init?.headers as Record)["Idempotency-Key"] === "k1")); +}); + +test("stop is never retried", async () => { + const { impl, calls } = seqFetch([unavailable, { status: 200, payload: { crawl_id: "c_1", status: "stopped" } }]); + await assert.rejects( + () => stopCrawl("c_1", { apiKey: "k", fetchImpl: impl, sleepImpl: recordSleep().sleepImpl }), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_FAILED" && e.crawl_id === "c_1", + ); + assert.equal(calls.length, 1); }); /** A stub that serves `pages` in order, recording the cursor each call sent. */ From 3ec1457d3eca0f11b6d10fe879753782d071b573 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Fri, 9 Oct 2026 18:02:35 +0200 Subject: [PATCH 07/19] docs(crawl): say New, not Beta Crawl copy now says New, not Beta. The crawl capability and skill have status available. The README Crawl section and the capability notes carry one line that says Crawl is still evolving. Batch and Extract wording does not change. --- README.md | 8 +++++--- docs/capabilities.md | 2 +- registry/capabilities.json | 4 ++-- registry/skills.json | 4 ++-- skills/crawl/SKILL.md | 6 +++--- skills/zenrows/SKILL.md | 2 +- src/cli/commands/crawl.ts | 6 +++--- tests/capabilities.test.ts | 2 +- tests/registry.test.ts | 2 +- 9 files changed, 19 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index cbedcb8..c9375e3 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ that endpoint with extraction parameters, not a separate product. | `zenrows fetch` | Fetch — retrieve a protected page | **available** — `GET https://api.zenrows.com/v1/` | | `zenrows extract` | Extract — `extract=auto` (domain-gated open beta; falls back to Autoparse) / CSS / Markdown | **beta** — same `/v1/` | | `zenrows batch` | Batch — fan out over many URLs | beta — cloud works with beta access; local validate/estimate always | -| `zenrows crawl` | Crawl (beta) — collect the URLs behind one start page, optionally with each page's HTML | beta — `https://api.zenrows.com/v1/crawls` | +| `zenrows crawl` | Crawl (new) — collect the URLs behind one start page, optionally with each page's HTML | **available** — `https://api.zenrows.com/v1/crawls` | | `zenrows browser` | Browser Sessions REST API (same backend as MCP `browser_*`) | **available** — escalation-only; bills by bandwidth + time | | `zenrows mcp` | MCP server config (remote + local) | **available** | | Zenrows CLI | this repo | available | @@ -157,9 +157,11 @@ An Extract task costs the same as a regular one (1 credit at base tier), so keys: `html` (the raw page) and `parsed` (the structured data) — validate a sample with `zenrows extract ` before running the full batch. -## 10. Crawl (beta) +## 10. Crawl (new) -Zenrows **Crawl** is in beta. Crawl (`https://api.zenrows.com/v1/crawls`) +Crawl is still evolving: new features are coming, limits may be tuned, and the changelog announces each change. + +Crawl (`https://api.zenrows.com/v1/crawls`) takes one start URL and returns the URLs it finds behind it, optionally with each page's HTML. Use it when you know the listing or section but not the item URLs. Crawl follows the links on each page up to `--depth` hops and stays on diff --git a/docs/capabilities.md b/docs/capabilities.md index 514b8b9..bc52ad3 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -22,6 +22,6 @@ Classification is based on the public Zenrows documentation: | `protected_fetch` | Fetch `GET https://api.zenrows.com/v1/` with `mode`, `js_render`, `premium_proxy`, `proxy_country`, `wait`/`wait_for`, `js_instructions`, `response_type`, `screenshot`, `original_status`, … | available | | `extract` | Extract `GET https://api.zenrows.com/v1/` via `extract=auto` (domain-gated open beta; CLI falls back to `autoparse`), plus `autoparse`, `css_extractor`, `outputs`, `response_type=markdown\|plaintext` | beta | | `batch` | Batch `https://async.api.zenrows.com/v1` (separate host, `X-API-Key` header) — open beta. Local JSONL spec validation + credit estimation work with no key. | beta | -| `crawl` | Crawl `https://api.zenrows.com/v1/crawls` (`X-API-Key` header; same host as Fetch/Extract). Follows the links on each page from one start URL up to `depth`, on the start URL's registrable domain (subdomains count); returns URLs, and with `output_format: "html"` each page's HTML. Subcommands: `create`, `get`, `list`, `results`, `content`, `download`, `stop`, `wait`. Without Crawl access the API returns 403 `REQS008` → `CRAWL_NOT_ENABLED`. | beta | +| `crawl` | Crawl `https://api.zenrows.com/v1/crawls` (`X-API-Key` header; same host as Fetch/Extract). Follows the links on each page from one start URL up to `depth`, on the start URL's registrable domain (subdomains count); returns URLs, and with `output_format: "html"` each page's HTML. Subcommands: `create`, `get`, `list`, `results`, `content`, `download`, `stop`, `wait`. Without Crawl access the API returns 403 `REQS008` → `CRAWL_NOT_ENABLED`. | available | | `browser` | Browser Sessions REST API `https://mcp.zenrows.com/browser/sessions/*` (Bearer; same backend as `@zenrows/mcp` `browser_*`); CDP via `zenrows browser connect`. Escalation-only (prefer fetch/extract); on by default, opt out via `policy.allow_browser`; bills by bandwidth + session time (15-min max) | available | | `mcp` | Hosted `https://mcp.zenrows.com/mcp` + local `npx -y @zenrows/mcp` | available | diff --git a/registry/capabilities.json b/registry/capabilities.json index 1f0b076..24a1b4f 100644 --- a/registry/capabilities.json +++ b/registry/capabilities.json @@ -31,11 +31,11 @@ "crawl": { "key": "crawl", "label": "Crawl", - "status": "beta", + "status": "available", "command": "zenrows crawl", "backend": "Crawl — https://api.zenrows.com/v1/crawls (X-API-Key header; same host as Fetch/Extract)", "requiresAuth": true, - "notes": "Beta. Follows the links on each page from one start URL up to depth, on the start URL's registrable domain (subdomains count), and returns the URLs it keeps, optionally with each page's HTML (create/get/list/results/content/download/stop/wait). An account without Crawl access gets 403 REQS008 (CRAWL_NOT_ENABLED). When the account has reached its limit of active jobs (3 by default), shared with its Batch jobs, a create gets 429 too_many_crawls (CRAWL_TOO_MANY_CRAWLS); retry after retry_after seconds." + "notes": "Crawl is still evolving: new features are coming, limits may be tuned, and the changelog announces each change. Follows the links on each page from one start URL up to depth, on the start URL's registrable domain (subdomains count), and returns the URLs it keeps, optionally with each page's HTML (create/get/list/results/content/download/stop/wait). An account without Crawl access gets 403 REQS008 (CRAWL_NOT_ENABLED). When the account has reached its limit of active jobs (3 by default), shared with its Batch jobs, a create gets 429 too_many_crawls (CRAWL_TOO_MANY_CRAWLS); retry after retry_after seconds." }, "browser": { "key": "browser", diff --git a/registry/skills.json b/registry/skills.json index 7b6e955..d4fb03f 100644 --- a/registry/skills.json +++ b/registry/skills.json @@ -47,8 +47,8 @@ { "name": "crawl", "type": "skill", - "description": "Collect the URLs behind one start page with Crawl (beta): create, wait, read results and page HTML, stop.", - "status": "beta", + "description": "Collect the URLs behind one start page with Crawl (new): create, wait, read results and page HTML, stop.", + "status": "available", "requires_backend_capabilities": ["crawl"], "requires_auth": true, "version": "0.1.0", diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index bb4472b..71e3617 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -1,13 +1,13 @@ --- name: crawl -description: Collect the URLs behind one start page with Crawl (beta): create a crawl, wait for it, read the kept URLs and optionally each page's HTML, stop it. Use when the user has a listing or section URL but not the item URLs. +description: Collect the URLs behind one start page with Crawl (new): create a crawl, wait for it, read the kept URLs and optionally each page's HTML, stop it. Use when the user has a listing or section URL but not the item URLs. version: 0.1.0 requires_backend_capabilities: [crawl] --- -# Crawl (beta) +# Crawl (new) -Crawl is in beta. Give Crawl one start URL; it follows the links on that page (and, with a +Give Crawl one start URL; it follows the links on that page (and, with a higher depth, on the pages behind it) and returns the URLs it keeps. Use it when you know where the items live (a category page, a blog index, a docs section) but not the item URLs themselves. Once you have the URLs, read pages diff --git a/skills/zenrows/SKILL.md b/skills/zenrows/SKILL.md index 350d6e4..2c6956e 100644 --- a/skills/zenrows/SKILL.md +++ b/skills/zenrows/SKILL.md @@ -74,7 +74,7 @@ Run `zenrows status` for the live capability matrix. As of this toolkit: | Protected Fetch | `zenrows fetch` | available (`GET /v1/`) | | Extract (extract=auto / Autoparse/CSS/Markdown) | `zenrows extract` | beta (same `/v1/`; extract=auto falls back to autoparse) | | Batch | `zenrows batch` | beta — runs cloud jobs (create/status/results); cheapest path at scale. Estimate specs locally with no key | -| Crawl | `zenrows crawl` | beta — collect the URLs behind one start page, optionally with each page's HTML | +| Crawl | `zenrows crawl` | new — collect the URLs behind one start page, optionally with each page's HTML | | Browser | `zenrows browser` | available (Browser Sessions REST API / MCP) | | MCP | `zenrows mcp` | available (remote + local server) | diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index c947643..45cb4eb 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -1,5 +1,5 @@ /** - * `zenrows crawl` — Crawl API (status: beta). + * `zenrows crawl` — Crawl API (new). * * Give Crawl one start URL and read back the URLs it finds behind it, optionally * with each page's HTML. A crawl is an async job: `create` returns at once @@ -40,11 +40,11 @@ const MAX_LIMIT = 100_000; export const crawl: Command = { name: "crawl", - summary: "Crawl a site from one start URL and collect the URLs behind it (beta).", + summary: "Crawl a site from one start URL and collect the URLs behind it (new).", usage: "zenrows crawl --depth N|get |list|results |content |download |stop |wait >", help: [ - "Crawl is in beta. Cloud (needs a key with Crawl access):", + "Crawl is new. Cloud (needs a key with Crawl access):", " create --depth [flags] start a crawl from one URL (returns at once)", " --depth link hops to follow from the start URL (1-100000, required)", " --max-items stop after keeping n URLs (API default 10)", diff --git a/tests/capabilities.test.ts b/tests/capabilities.test.ts index 3485169..73c7da6 100644 --- a/tests/capabilities.test.ts +++ b/tests/capabilities.test.ts @@ -11,7 +11,7 @@ test("capability matrix loads with the honest classifications", () => { assert.equal(caps.mcp?.status, "available"); // Batch is a real product in open beta. assert.equal(caps.batch?.status, "beta"); - assert.equal(caps.crawl?.status, "beta"); + assert.equal(caps.crawl?.status, "available"); }); test("isUsable true for available + open beta, false for absent capabilities", () => { diff --git a/tests/registry.test.ts b/tests/registry.test.ts index 145b479..3ca7fb7 100644 --- a/tests/registry.test.ts +++ b/tests/registry.test.ts @@ -52,5 +52,5 @@ test("assetRunnable reflects backend capability status", () => { const extractSkill = loadRegistry("skill").find((s) => s.name === "extract")!; assert.equal(assetRunnable(extractSkill), true); // needs extract (open beta = usable) const crawlSkill = loadRegistry("skill").find((s) => s.name === "crawl")!; - assert.equal(assetRunnable(crawlSkill), true); // needs crawl (open beta = usable) + assert.equal(assetRunnable(crawlSkill), true); // needs crawl (available) }); From 119c39979567513b438409216d9cca91481cf68e Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 13:24:13 +0200 Subject: [PATCH 08/19] fix(crawl): retry a stop answered 503 crawl_busy, cap Retry-After, report a stopped crawl as complete - stop retries a 503 crawl_busy (the stop was not saved; the crawl still runs) up to 3 times, waiting Retry-After, then fails with CRAWL_BUSY and retry_after. A plain 503 is still not retried. This matches the three SDKs and the MCP server. - Every retry waits at most 30 s, whatever Retry-After asks for. - results on an ended crawl reports partial: false, so a stopped crawl whose pages are still settling is not reported as incomplete. - --depth, --max-items and --max-pages no longer reject 0 locally; the API answers 422 and the CLI reports CRAWL_INVALID_REQUEST. - Error causes no longer end in a doubled period ("account.. The id"). --- README.md | 9 +++-- skills/crawl/SKILL.md | 9 ++--- src/cli/commands/crawl.ts | 11 +++--- src/core/crawl-api.ts | 59 ++++++++++++++++++++++++-------- src/core/errors.ts | 1 + tests/crawl-api.test.ts | 68 ++++++++++++++++++++++++++++++++++++- tests/crawl-command.test.ts | 21 ++++++++++-- 7 files changed, 150 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index c9375e3..eb5e1f3 100644 --- a/README.md +++ b/README.md @@ -214,14 +214,17 @@ Other codes: `CRAWL_QUOTA_EXCEEDED` and `CRAWL_KEY_CAP_REACHED` (402, with the same claim, top-up and key-cap advice as other commands), `CRAWL_INVALID_REQUEST` (400/422, do not retry as is), `CRAWL_REQUEST_IN_FLIGHT` (409, retry once the first request ends), `CRAWL_NOT_FOUND`, -`CRAWL_CONTENT_NOT_FOUND` and `CRAWL_FAILED`. Set `ZENROWS_CRAWL_API_BASE` to +`CRAWL_CONTENT_NOT_FOUND`, `CRAWL_BUSY` (503 on a stop that was not saved yet; +run it again, it is safe) and `CRAWL_FAILED`. Set `ZENROWS_CRAWL_API_BASE` to point the CLI at another deployment. The CLI retries transient failures, as the Crawl SDKs do: up to 3 retries, -waiting the `Retry-After` seconds or 250 ms x 2^n (+/-20% jitter, at most 10 s). +waiting the `Retry-After` seconds (at most 30) or 250 ms x 2^n (+/-20% jitter, +at most 10 s). Reads retry on 429, 502, 503, 504 and network errors (a timeout counts). `create` retries only with `--idempotency-key`, and only on 502, 503, 504 and -network errors, never on 429. `stop` never retries. +network errors, never on 429. `stop` retries only a 503 `crawl_busy`, which +means the stop was not saved and the crawl still runs. ## 11. Browser Sessions diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index 71e3617..e47936a 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -94,10 +94,10 @@ Every Crawl error carries the API's code as `server_code` (when the API sent one), and `crawl_id` on any call about one crawl. The CLI retries transient failures, as the Crawl SDKs do: up to 3 retries, -waiting the `Retry-After` seconds or 250 ms x 2^n (+/-20% jitter, at most 10 s). -Reads retry on 429, 502, 503, 504 and network errors (a timeout counts). -`create` retries only with `--idempotency-key`, and only on 502, 503, 504 and -network errors, never on 429. `stop` never retries. +waiting the `Retry-After` seconds (at most 30) or 250 ms x 2^n (+/-20% jitter, +at most 10 s). Reads retry on 429, 502, 503, 504 and network errors (a timeout +counts). `create` retries only with `--idempotency-key`, and only on 502, 503, +504 and network errors, never on 429. `stop` retries only a 503 `crawl_busy`. The "Retry?" column is for errors that remain after these retries. | Code | Meaning | Retry? | @@ -110,6 +110,7 @@ The "Retry?" column is for errors that remain after these retries. | `CRAWL_REQUEST_IN_FLIGHT` | 409: the same Idempotency-Key is still in flight | yes, once the first request ends | | `CRAWL_NOT_FOUND` / `CRAWL_CONTENT_NOT_FOUND` | 404: wrong id, or the page has no content | no | | `CRAWL_TOO_MANY_CRAWLS` | 429: the account's active-job limit | yes, after `retry_after` | +| `CRAWL_BUSY` | 503 `crawl_busy` on `stop`: the stop was not saved yet and the crawl still runs | yes, run the stop again after `retry_after` | | `CRAWL_FAILED` | the crawl ended `failed`, or another error | 5xx: yes, with backoff | | `BACKEND_UNAVAILABLE` | the API could not be reached | yes | diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index 45cb4eb..fbdd8d4 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -46,7 +46,7 @@ export const crawl: Command = { help: [ "Crawl is new. Cloud (needs a key with Crawl access):", " create --depth [flags] start a crawl from one URL (returns at once)", - " --depth link hops to follow from the start URL (1-100000, required)", + " --depth link hops to follow from the start URL (up to 100000, required)", " --max-items stop after keeping n URLs (API default 10)", " --max-pages stop after fetching n pages (API default 10; bounds cost);", " the local policy max_pages_per_run caps it (default 1000)", @@ -491,15 +491,18 @@ function printCrawl( return err ? 1 : 0; } -/** A positive integer flag up to the API's 100,000 ceiling; undefined when unset. */ +/** + * A whole-number flag up to the API's 100,000 ceiling; undefined when unset. The lower bound is + * the API's to judge: 0 or less comes back as CRAWL_INVALID_REQUEST, naming the parameter. + */ function limitFlag(name: string, v: unknown): number | undefined { if (v === undefined) return undefined; const n = asNumber(v); - if (n === undefined || !Number.isInteger(n) || n < 1 || n > MAX_LIMIT) { + if (n === undefined || !Number.isInteger(n) || n > MAX_LIMIT) { throw new ToolkitError({ code: "INVALID_USAGE", message: `Invalid --${name} value '${String(v)}'.`, - likely_cause: `--${name} takes a whole number from 1 to ${MAX_LIMIT}.`, + likely_cause: `--${name} takes a whole number up to ${MAX_LIMIT}.`, next_action: `Pass e.g. --${name} ${name === "depth" ? 1 : 10}.`, }); } diff --git a/src/core/crawl-api.ts b/src/core/crawl-api.ts index 57067a7..923a98f 100644 --- a/src/core/crawl-api.ts +++ b/src/core/crawl-api.ts @@ -146,6 +146,8 @@ interface RequestOpts extends CallOpts { accept?: string; /** Set on every call about one crawl, so its errors carry `crawl_id`. */ crawlId?: string; + /** Retry a 503 crawl_busy (stop only): the stop was not saved and the crawl still runs. */ + retryBusy?: boolean; } interface RawResponse { @@ -155,13 +157,16 @@ interface RawResponse { } const MAX_RETRIES = 3; +/** The longest a Retry-After may make one retry wait, so a large server value cannot hang a command. */ +const MAX_RETRY_AFTER_SECONDS = 30; const READ_RETRY_STATUSES: ReadonlySet = new Set([429, 502, 503, 504]); // 429 on create is the active-job limit: a retry only waits for a slot that may never free. const CREATE_RETRY_STATUSES: ReadonlySet = new Set([502, 503, 504]); /** * The statuses to retry, or undefined when the request must run once: a GET is - * safe to repeat, a create only with an Idempotency-Key, and stop never. + * safe to repeat, a create only with an Idempotency-Key. Stop retries only a 503 + * crawl_busy (see `crawlFetch`). */ function retryStatuses(method: string, headers: Record): ReadonlySet | undefined { if (method === "GET") return READ_RETRY_STATUSES; @@ -173,9 +178,17 @@ function backoffMs(attempt: number): number { return Math.min(250 * 2 ** attempt, 10_000) * (0.8 + Math.random() * 0.4); } -/** `Retry-After` in whole seconds, as milliseconds. */ +/** `Retry-After` in whole seconds, as milliseconds, at most `MAX_RETRY_AFTER_SECONDS`. */ function retryAfterMs(value: string | null): number | undefined { - return value && /^\d+$/.test(value.trim()) ? Number(value) * 1000 : undefined; + return value && /^\d+$/.test(value.trim()) ? Math.min(Number(value), MAX_RETRY_AFTER_SECONDS) * 1000 : undefined; +} + +function problemCode(body: string): string | undefined { + try { + return (JSON.parse(body) as ProblemJson).code; + } catch { + return undefined; + } } const defaultSleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); @@ -241,6 +254,10 @@ async function crawlFetch(method: string, path: string, opts: RequestOpts): Prom await sleep(retryAfterMs(retryAfter) ?? backoffMs(n)); continue; } + if (opts.retryBusy && n < MAX_RETRIES && res.status === 503 && problemCode(res.text) === "crawl_busy") { + await sleep(retryAfterMs(retryAfter) ?? 1000); + continue; + } if (res.status < 200 || res.status >= 300) { throw problemToError(res.status, res.text, `${method} ${path}`, retryAfter, opts.crawlId); } @@ -282,7 +299,8 @@ function problemToError( // non-JSON error body — fall through with an empty problem } const serverCode = problem.code || undefined; - const seconds = status === 429 && retryAfter && /^\d+$/.test(retryAfter.trim()) ? Number(retryAfter) : undefined; + const throttled = status === 429 || (status === 503 && serverCode === "crawl_busy"); + const seconds = throttled && retryAfter && /^\d+$/.test(retryAfter.trim()) ? Number(retryAfter) : undefined; const err = classifyProblem(status, problem, body, request, serverCode, seconds); return new ToolkitError({ ...err.toJSON(), server_code: serverCode, crawl_id: crawlId, retry_after: seconds }); } @@ -296,13 +314,15 @@ function classifyProblem( retryAfter: number | undefined, ): ToolkitError { const detail = problem.detail || problem.title || body.slice(0, 240) || `HTTP ${status}`; - const cause = `HTTP ${status}${serverCode ? ` (${serverCode})` : ""} for ${request}: ${detail}`; + // The API's detail often ends in a period already; add one only when it does not. + const raw = `HTTP ${status}${serverCode ? ` (${serverCode})` : ""} for ${request}: ${detail}`; + const cause = /[.!?]$/.test(raw) ? raw : `${raw}.`; if (status === 403 && serverCode === CRAWL_NOT_ENABLED_CODE) { return new ToolkitError({ code: "CRAWL_NOT_ENABLED", message: "Crawl is not enabled for this account.", - likely_cause: `${cause}. Crawl is not enabled for this account yet.`, + likely_cause: `${cause} Crawl is not enabled for this account yet.`, next_action: "Ask Zenrows support to enable Crawl for this account. Meanwhile fetch known URLs with `zenrows fetch`.", suggested_commands: ["zenrows fetch "], @@ -321,7 +341,7 @@ function classifyProblem( return new ToolkitError({ code: "CRAWL_CONTENT_NOT_FOUND", message: "Crawl content not found.", - likely_cause: `${cause}. The page was not fetched (yet), its fetch failed, or the crawl ran without --output-format html.`, + likely_cause: `${cause} The page was not fetched (yet), its fetch failed, or the crawl ran without --output-format html.`, next_action: "Read the result's content_status with `zenrows crawl results `; only `fetched` results have content.", suggested_commands: ["zenrows crawl list"], }); @@ -330,7 +350,7 @@ function classifyProblem( return new ToolkitError({ code: "CRAWL_NOT_FOUND", message: "Crawl not found.", - likely_cause: `${cause}. The id may be wrong or the crawl is not owned by this account.`, + likely_cause: `${cause} The id may be wrong or the crawl is not owned by this account.`, next_action: "Check the crawl id, or list your crawls with `zenrows crawl list`.", suggested_commands: ["zenrows crawl list"], }); @@ -339,7 +359,7 @@ function classifyProblem( return new ToolkitError({ code: "CRAWL_TOO_MANY_CRAWLS", message: "The account has reached its limit of active jobs.", - likely_cause: `${cause}. The account has reached its limit of active jobs (3 by default), shared with its Batch jobs.`, + likely_cause: `${cause} The account has reached its limit of active jobs (3 by default), shared with its Batch jobs.`, next_action: `Nothing was created. ${retryAfter !== undefined ? `Retry after ${retryAfter} seconds` : "Retry later"}, or stop one of your crawls (\`zenrows crawl stop \`) or Batch jobs first.`, suggested_commands: ["zenrows crawl list"], }); @@ -358,15 +378,24 @@ function classifyProblem( return new ToolkitError({ code: "CRAWL_REQUEST_IN_FLIGHT", message: "A request with the same Idempotency-Key is still in flight.", - likely_cause: `${cause}.`, + likely_cause: cause, next_action: "Retry the same request once the first one has finished.", }); } + if (status === 503 && serverCode === "crawl_busy") { + return new ToolkitError({ + code: "CRAWL_BUSY", + message: "The stop was not saved yet; the crawl is still running.", + likely_cause: cause, + next_action: `Run the stop again${retryAfter !== undefined ? ` after ${retryAfter} seconds` : ""}. Repeating a stop is safe.`, + suggested_commands: ["zenrows crawl stop "], + }); + } if (status === 400 || status === 422) { return new ToolkitError({ code: "CRAWL_INVALID_REQUEST", message: `Crawl rejected the request (HTTP ${status}).`, - likely_cause: `${cause}.`, + likely_cause: cause, next_action: serverCode === "idempotency_key_reused" ? "This Idempotency-Key was already used for a different request. Send a new key, or no key. Do not retry as is." @@ -376,7 +405,7 @@ function classifyProblem( return new ToolkitError({ code: "CRAWL_FAILED", message: `Crawl request failed (HTTP ${status}).`, - likely_cause: `${cause}.`, + likely_cause: cause, next_action: status >= 500 ? "Retry with a short backoff." : "Fix the reported problem and retry.", }); @@ -413,7 +442,7 @@ export function listCrawls(opts: CallOpts & { cursor?: string; limit?: number }) /** Stop a running crawl. Idempotent: an ended crawl answers as it ended. */ export function stopCrawl(id: string, opts: CallOpts): Promise { - return crawlJson("POST", `/crawls/${encodeURIComponent(id)}/stop`, { ...opts, crawlId: id }); + return crawlJson("POST", `/crawls/${encodeURIComponent(id)}/stop`, { ...opts, crawlId: id, retryBusy: true }); } /** The page of one kept URL (HTML for `output_format: html`). */ @@ -470,7 +499,9 @@ export async function listAllResults( const rows = page.results ?? []; results.push(...rows); if (!rows.length || !page.next_cursor || page.next_cursor === cursor) { - return { crawl: page, results, partial: !isTerminal(page.status) || !!page.next_cursor }; + // Every page was read. An ended crawl keeps no new URLs, even while pages in flight after + // a stop keep next_cursor set, so only a running crawl's results are partial. + return { crawl: page, results, partial: !isTerminal(page.status) }; } cursor = page.next_cursor; } diff --git a/src/core/errors.ts b/src/core/errors.ts index de8ad6c..5b889c3 100644 --- a/src/core/errors.ts +++ b/src/core/errors.ts @@ -43,6 +43,7 @@ export type ErrorCode = | "CRAWL_CONTENT_NOT_FOUND" | "CRAWL_INVALID_REQUEST" | "CRAWL_REQUEST_IN_FLIGHT" + | "CRAWL_BUSY" | "CRAWL_FAILED" | "PLUGIN_UNSUPPORTED" | "MCP_CLIENT_UNSUPPORTED" diff --git a/tests/crawl-api.test.ts b/tests/crawl-api.test.ts index d95000c..5cc733c 100644 --- a/tests/crawl-api.test.ts +++ b/tests/crawl-api.test.ts @@ -357,7 +357,7 @@ test("create with an Idempotency-Key is retried on 503 with the same key", async assert.ok(calls.every((c) => (c.init?.headers as Record)["Idempotency-Key"] === "k1")); }); -test("stop is never retried", async () => { +test("stop is not retried on a plain 503", async () => { const { impl, calls } = seqFetch([unavailable, { status: 200, payload: { crawl_id: "c_1", status: "stopped" } }]); await assert.rejects( () => stopCrawl("c_1", { apiKey: "k", fetchImpl: impl, sleepImpl: recordSleep().sleepImpl }), @@ -366,6 +366,61 @@ test("stop is never retried", async () => { assert.equal(calls.length, 1); }); +const busy = (retryAfter = "1") => ({ + status: 503, + payload: { code: "crawl_busy", title: "Crawl busy", detail: "The stop could not be saved yet.", status: 503 }, + headers: { "retry-after": retryAfter }, +}); + +test("stop retries a 503 crawl_busy after Retry-After and returns the stopped crawl", async () => { + const { impl, calls } = seqFetch([busy(), { status: 200, payload: { crawl_id: "c_1", status: "stopped" } }]); + const { waits, sleepImpl } = recordSleep(); + const s = await stopCrawl("c_1", { apiKey: "k", fetchImpl: impl, sleepImpl }); + assert.equal(s.status, "stopped"); + assert.equal(calls.length, 2); + assert.deepEqual(waits, [1000]); +}); + +test("stop gives up on crawl_busy after 3 retries with CRAWL_BUSY and retry_after", async () => { + const { impl, calls } = seqFetch([busy()]); + await assert.rejects( + () => stopCrawl("c_1", { apiKey: "k", fetchImpl: impl, sleepImpl: recordSleep().sleepImpl }), + (e: unknown) => + e instanceof ToolkitError && + e.code === "CRAWL_BUSY" && + e.retry_after === 1 && + e.crawl_id === "c_1" && + /Repeating a stop is safe/.test(e.next_action ?? ""), + ); + assert.equal(calls.length, 4); +}); + +test("a Retry-After above 30 seconds waits 30 seconds", async () => { + const { impl } = seqFetch([ + { status: 429, payload: { code: "rate_limited" }, headers: { "retry-after": "3600" } }, + { status: 200, payload: { crawls: [] } }, + ]); + const { waits, sleepImpl } = recordSleep(); + await listCrawls({ apiKey: "k", fetchImpl: impl, sleepImpl }); + assert.deepEqual(waits, [30_000]); + + const stop = seqFetch([busy("3600"), { status: 200, payload: { crawl_id: "c_1", status: "stopped" } }]); + const stopSleep = recordSleep(); + await stopCrawl("c_1", { apiKey: "k", fetchImpl: stop.impl, sleepImpl: stopSleep.sleepImpl }); + assert.deepEqual(stopSleep.waits, [30_000]); +}); + +test("an error whose detail ends in a period does not get a second one", async () => { + const { impl } = problem(404, "crawl_not_found", "Crawl not found for this account."); + await assert.rejects( + () => getCrawl("c_nope", { apiKey: "k", fetchImpl: impl }), + (e: unknown) => + e instanceof ToolkitError && + /account\. The id may be wrong/.test(e.likely_cause ?? "") && + !/\.\./.test(e.likely_cause ?? ""), + ); +}); + /** A stub that serves `pages` in order, recording the cursor each call sent. */ function pagedFetch(pages: Array>) { const cursors: Array = []; @@ -406,6 +461,17 @@ test("listAllResults returns what a running crawl kept, ending at the first empt assert.equal(calls(), 2); }); +test("listAllResults on a stopped crawl is complete at the first empty page, even with next_cursor still set", async () => { + const { impl, calls } = pagedFetch([ + { status: "stopped", results: [{ url: "https://example.com/product/1" }], next_cursor: "c1" }, + { status: "stopped", results: [], next_cursor: "c1" }, + ]); + const { results, partial } = await listAllResults("c_1", { apiKey: "k", fetchImpl: impl }); + assert.deepEqual(results.map((r) => r.url), ["https://example.com/product/1"]); + assert.equal(partial, false); + assert.equal(calls(), 2); +}); + test("waitForCrawl polls with limit=1 until the status leaves running, backing off", async () => { const { impl, limits, calls } = pagedFetch([ { status: "running", results: [], next_cursor: "c" }, diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index 0512620..eae9006 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -121,11 +121,11 @@ test("crawl create takes --output-format html only and --timeout in positive sec ); }); -test("crawl create requires --depth and a whole number in range", async () => { +test("crawl create requires --depth as a whole number up to 100000", async () => { await withCrawlWorkspace( () => json(crawlBody, 202), async (calls) => { - for (const args of [[], ["--depth", "0"], ["--depth", "1.5"], ["--depth", "100001"], ["--depth", "x"]]) { + for (const args of [[], ["--depth", "1.5"], ["--depth", "100001"], ["--depth", "x"]]) { const { code, out } = await run(["create", "https://example.com/products/", ...args]); assert.equal(code, 1, args.join(" ")); assert.equal(out.error.code, "INVALID_USAGE", args.join(" ")); @@ -135,6 +135,23 @@ test("crawl create requires --depth and a whole number in range", async () => { ); }); +test("crawl create leaves the lower bound to the API: --depth 0 comes back as CRAWL_INVALID_REQUEST", async () => { + await withCrawlWorkspace( + () => + json( + { type: "about:blank", title: "Invalid parameter", status: 422, code: "invalid_parameter", detail: "'depth' is required, and at least 1." }, + 422, + ), + async (calls) => { + const { code, out } = await run(["create", "https://example.com/products/", "--depth", "0"]); + assert.equal(code, 1); + assert.equal(out.error.code, "CRAWL_INVALID_REQUEST"); + assert.equal(out.error.server_code, "invalid_parameter"); + assert.equal(calls.length, 1); + }, + ); +}); + test("crawl create sends only the flags the caller set as the create body", async () => { await withCrawlWorkspace( () => json(crawlBody, 202, { location: "/v1/crawls/c_1" }), From 0199e422e17b274977e21b1e26cc32824c9ad98a Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 13:54:02 +0200 Subject: [PATCH 09/19] feat(crawl)!: name the pattern flags after the API's fields --include-pattern and --exclude-pattern become --include-patterns and --exclude-patterns, matching include_patterns / exclude_patterns in the Crawl API, the SDKs and the MCP server. Each flag still takes one substring and repeats. The crawl command has not shipped, so no alias keeps the old spelling. --- README.md | 6 +++--- skills/crawl/SKILL.md | 6 +++--- src/cli/commands/crawl.ts | 14 +++++++------- tests/crawl-command.test.ts | 6 +++--- tests/e2e/crawl.e2e.test.ts | 7 ++++--- 5 files changed, 20 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index eb5e1f3..e0221c3 100644 --- a/README.md +++ b/README.md @@ -172,7 +172,7 @@ the start URL's registrable domain (subdomains count). Without ```bash zenrows crawl create https://example.com/products/ --depth 1 \ - --max-items 20 --include-pattern /product/ --follow # create and wait for the end + --max-items 20 --include-patterns /product/ --follow # create and wait for the end zenrows crawl results --out urls.jsonl # every kept URL (paginated) zenrows crawl create --depth 2 --output-format html # also fetch each kept page's HTML zenrows crawl content # one page's HTML @@ -187,8 +187,8 @@ zenrows crawl stop # stop a running crawl crawl (API default 10 each), and `--max-pages` bounds the cost, since each page is one fetch. The local policy `max_pages_per_run` caps `--max-pages` (default 1000). `--output-format html` uses up the page budget: each kept page is one -more fetch, and `--max-pages` counts it. `--include-pattern` / -`--exclude-pattern` are repeatable substring filters on the URL. `get` and +more fetch, and `--max-pages` counts it. `--include-patterns` / +`--exclude-patterns` are repeatable substring filters on the URL. `get` and `list` read one page and take `--cursor` and `--limit`. `results` follows every page (`--limit` is the page size); on a running crawl it returns the URLs kept so far, with `partial: true`. `download` reports the crawl's `status` (`null` diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index e47936a..e573ab9 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -22,7 +22,7 @@ do not retry it. ``` zenrows crawl create https://example.com/products/ --depth 1 \ - --max-items 20 --max-pages 25 --include-pattern /product/ --follow --json + --max-items 20 --max-pages 25 --include-patterns /product/ --follow --json zenrows crawl results --json # every kept URL zenrows crawl results --out urls.jsonl # same, as JSONL ``` @@ -33,9 +33,9 @@ zenrows crawl results --out urls.jsonl # same, as JSONL both default to 10. `--max-pages` bounds the cost: each page is one fetch. The local policy `max_pages_per_run` caps `--max-pages` (default 1000); above it the CLI refuses with `POLICY_LIMIT_EXCEEDED` before any request. -- `--include-pattern` / `--exclude-pattern` are substring matches on the +- `--include-patterns` / `--exclude-patterns` are substring matches on the normalized URL and repeat: - `--include-pattern /product/ --exclude-pattern ?add-to-cart`. + `--include-patterns /product/ --exclude-patterns ?add-to-cart`. - `create` returns at once. `--follow` (or `zenrows crawl wait `) polls until the crawl ends or `--timeout` seconds run out (default 600). When the time runs out, the command exits 0 and prints the crawl with status diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index fbdd8d4..d01e366 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -50,8 +50,8 @@ export const crawl: Command = { " --max-items stop after keeping n URLs (API default 10)", " --max-pages stop after fetching n pages (API default 10; bounds cost);", " the local policy max_pages_per_run caps it (default 1000)", - " --include-pattern

keep only URLs containing this substring (repeatable)", - " --exclude-pattern

drop URLs containing this substring (repeatable)", + " --include-patterns

keep only URLs containing this substring (repeatable)", + " --exclude-patterns

drop URLs containing this substring (repeatable)", " --output-format html also fetch each kept URL's page HTML (read with content/download);", " each kept page is one more fetch counted by --max-pages", " --follow wait until the crawl ends (Ctrl-C stops the wait, not the crawl)", @@ -117,8 +117,8 @@ async function createCmd(rest: string[], ctx: RunContext): Promise { depth: { type: "string" }, "max-items": { type: "string" }, "max-pages": { type: "string" }, - "include-pattern": { type: "string", multiple: true }, - "exclude-pattern": { type: "string", multiple: true }, + "include-patterns": { type: "string", multiple: true }, + "exclude-patterns": { type: "string", multiple: true }, "output-format": { type: "string" }, follow: { type: "boolean" }, timeout: { type: "string" }, @@ -152,9 +152,9 @@ async function createCmd(rest: string[], ctx: RunContext): Promise { if (maxItems !== undefined) params.max_items = maxItems; const maxPages = limitFlag("max-pages", values["max-pages"]); if (maxPages !== undefined) params.max_pages = maxPages; - const include = patterns(values["include-pattern"]); + const include = patterns(values["include-patterns"]); if (include.length) params.include_patterns = include; - const exclude = patterns(values["exclude-pattern"]); + const exclude = patterns(values["exclude-patterns"]); if (exclude.length) params.exclude_patterns = exclude; const outputFormat = asString(values["output-format"]); if (outputFormat !== undefined) { @@ -431,7 +431,7 @@ export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null { insufficient_credits: "The account ran out of credits. Check `zenrows usage`, add credits, then start a new crawl.", seed_unreachable: "The start URL could not be fetched (the site may be blocking it). Check it with `zenrows fetch `.", no_items_found: - "Nothing matched. Loosen --include-pattern / --exclude-pattern, raise --depth, or check the start page with `zenrows fetch `.", + "Nothing matched. Loosen --include-patterns / --exclude-patterns, raise --depth, or check the start page with `zenrows fetch `.", }; return new ToolkitError({ code: "CRAWL_FAILED", diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index eae9006..178ba1a 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -165,11 +165,11 @@ test("crawl create sends only the flags the caller set as the create body", asyn "3", "--max-pages", "5", - "--include-pattern", + "--include-patterns", "/product/", - "--include-pattern", + "--include-patterns", "/item/", - "--exclude-pattern", + "--exclude-patterns", "?add", "--output-format", "html", diff --git a/tests/e2e/crawl.e2e.test.ts b/tests/e2e/crawl.e2e.test.ts index f779074..8597df6 100644 --- a/tests/e2e/crawl.e2e.test.ts +++ b/tests/e2e/crawl.e2e.test.ts @@ -78,7 +78,7 @@ test("zenrows crawl end to end against a live API", { skip, timeout: 20 * 60_000 start = await cli(dir, [ "crawl", "create", START_URL!, "--depth", "1", "--max-items", "3", "--max-pages", "5", - ...(INCLUDE ? ["--include-pattern", INCLUDE] : []), + ...(INCLUDE ? ["--include-patterns", INCLUDE] : []), "--output-format", "html", "--follow", "--timeout", "600", ]); if (start.out.error?.code !== "CRAWL_TOO_MANY_CRAWLS" || Date.now() > deadline) break; @@ -97,7 +97,7 @@ test("zenrows crawl end to end against a live API", { skip, timeout: 20 * 60_000 assert.equal(results.code, 0, JSON.stringify(results.out)); const rows = results.out.results as Array<{ url: string; content_status?: string; content_url?: string }>; assert.ok(rows.length >= 1, "at least one result"); - if (INCLUDE) for (const r of rows) assert.ok(r.url.includes(INCLUDE), `${r.url} matches --include-pattern`); + if (INCLUDE) for (const r of rows) assert.ok(r.url.includes(INCLUDE), `${r.url} matches --include-patterns`); log(`results: ${rows.length} URL(s)${INCLUDE ? ", all matching the include pattern" : ""}`); // 3. One fetched page's HTML. @@ -138,4 +138,5 @@ test("zenrows crawl end to end against a live API", { skip, timeout: 20 * 60_000 } finally { rmSync(dir, { recursive: true, force: true }); } -}); +} +); From fb4ca72f169efdb7ffbc15187465642781b883f8 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 13:54:31 +0200 Subject: [PATCH 10/19] fix(crawl): correct file contents --- tests/e2e/crawl.e2e.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/e2e/crawl.e2e.test.ts b/tests/e2e/crawl.e2e.test.ts index 8597df6..4561952 100644 --- a/tests/e2e/crawl.e2e.test.ts +++ b/tests/e2e/crawl.e2e.test.ts @@ -138,5 +138,4 @@ test("zenrows crawl end to end against a live API", { skip, timeout: 20 * 60_000 } finally { rmSync(dir, { recursive: true, force: true }); } -} -); +}); From 15cda24ab09cd64302efec255523ee5f7b0fc8b3 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 14:09:18 +0200 Subject: [PATCH 11/19] test(crawl): add a first-install acceptance test for the Crawl client acceptance/crawl/ builds the release artifact from this checkout, installs it in a clean container the way a new user would, and runs every Crawl feature against the live API: README example, create, wait, results, content, download, stop, errors, naming and docs. One JSON result per check and a markdown report land in acceptance/crawl/out/ (git-ignored). It is kept out of the published package. --- acceptance/crawl/.gitignore | 1 + acceptance/crawl/README.md | 110 +++ acceptance/crawl/builders/build-npm.sh | 16 + acceptance/crawl/builders/fetch-source.sh | 11 + acceptance/crawl/run.sh | 148 +++ acceptance/crawl/surface/harness.mjs | 223 +++++ acceptance/crawl/surface/run-in-container.sh | 6 + acceptance/crawl/surface/test.mjs | 890 +++++++++++++++++++ acceptance/crawl/tools/build_failed.py | 17 + acceptance/crawl/tools/cleanup.py | 64 ++ acceptance/crawl/tools/report.py | 190 ++++ acceptance/crawl/tools/scan_key.py | 21 + acceptance/crawl/versions.env | 23 + 13 files changed, 1720 insertions(+) create mode 100644 acceptance/crawl/.gitignore create mode 100644 acceptance/crawl/README.md create mode 100644 acceptance/crawl/builders/build-npm.sh create mode 100644 acceptance/crawl/builders/fetch-source.sh create mode 100644 acceptance/crawl/run.sh create mode 100644 acceptance/crawl/surface/harness.mjs create mode 100644 acceptance/crawl/surface/run-in-container.sh create mode 100644 acceptance/crawl/surface/test.mjs create mode 100644 acceptance/crawl/tools/build_failed.py create mode 100644 acceptance/crawl/tools/cleanup.py create mode 100644 acceptance/crawl/tools/report.py create mode 100644 acceptance/crawl/tools/scan_key.py create mode 100644 acceptance/crawl/versions.env diff --git a/acceptance/crawl/.gitignore b/acceptance/crawl/.gitignore new file mode 100644 index 0000000..89f9ac0 --- /dev/null +++ b/acceptance/crawl/.gitignore @@ -0,0 +1 @@ +out/ diff --git a/acceptance/crawl/README.md b/acceptance/crawl/README.md new file mode 100644 index 0000000..86de5a0 --- /dev/null +++ b/acceptance/crawl/README.md @@ -0,0 +1,110 @@ +# Crawl first-install acceptance test: CLI + +This folder tests the CLI (`@zenrows/cli`) the way a new customer meets it. It builds the release +artifact, installs it in a clean container with nothing else, follows the README's Crawl +quickstart, and checks every Crawl feature against the live API. The result is one report. + +It tests this repo only, so it runs on its own. + +## What it needs + +- bash, git and Docker. Nothing else on the host. +- Network access to GitHub, the package registries and the Crawl API. +- A Zenrows API key with Crawl access, in `ZENROWS_API_KEY`. + +## How to run it + +From the repo root: + +```bash +export ZENROWS_API_KEY="$(cat path/to/key-file)" # keep it out of your shell history +bash acceptance/crawl/run.sh # build, run every check, write the report +bash acceptance/crawl/run.sh --no-key # only the checks that need no key; the live checks are SKIP +bash acceptance/crawl/run.sh --report # rebuild the report from out/ +REF=v1.2.3 bash acceptance/crawl/run.sh # test a tag, branch or SHA from GitHub instead of this checkout +``` + +Without `ZENROWS_API_KEY`, `run.sh` stops and asks for it. The report is +`acceptance/crawl/out/crawl-first-install-report.md`; `out/` is not committed. + +Every input is an environment variable with a default in `versions.env`: `CRAWL_URL` +(default `https://www.scrapingcourse.com/ecommerce/`), `CRAWL_INCLUDE` (default `/product/`), +`REF`, `CRAWL_API_BASE`, the images (`node:20`, `python:3.12-slim`, `golang:1.25`, pinned by +digest) and the tool versions. `OUT_DIR` moves the output. + +## What a run does + +1. **Source.** By default, this checkout, including changes not yet committed. With `REF`, the + repo at that ref from GitHub. +2. **Build.** In its own container, the release artifact: `npm install -g` of the `npm pack` tarball. +3. **Clean test container.** It gets only the artifact (read-only), the checks in `surface/` + (read-only), the inputs, and the key by name (`-e ZENROWS_API_KEY`). It has no source, no + caches from the build, and no tokens. +4. **README first.** It runs the README's Crawl quickstart as written. The only changes are the + ones the cost rules need: the key, `CRAWL_URL`, `CRAWL_INCLUDE`, `max_pages` 10 or less, and + no html. The report lists each change. +5. **Checks 1 to 21 and the extras.** Each writes one JSON result: + `{surface, check, title, status, expected, actual, evidence}`, with `status` PASS, FAIL or + SKIP. The evidence is the command and its trimmed output, with the key redacted. +6. **Clean-up.** It stops every crawl the run started and reads each one's final status and + `coverage.pages_fetched`. It runs even when a check fails, or on Ctrl-C. +7. **Report.** Verdict, environment, install steps, feature matrix, findings, cost and + clean-up. Last, the run checks that the key appears in no file under `out/`. + +Some checks don't use the live API, so they behave the same on every run. They point the +client at a small local stand-in server that records what it sends. These are: "only the +fields you set are sent", "the error code is empty when the body has none", and the timing +and signal checks. + +## The checks + +| # | Feature | Expected | +| --- | --- | --- | +| 1 | Create | returns at once: `running`, a `crawl_id`, the echoed parameters; only the fields set are sent | +| 2 | Create with html | results get `content_status` and, once fetched, `content_url` | +| 3 | Idempotency | same key and body → same `crawl_id`; different body → 422 `idempotency_key_reused` (MCP, CLI: `CRAWL_INVALID_REQUEST` with the advice to send a new key) | +| 4 | Wait | returns the ended crawl; `failed` is returned, not raised (CLI: exit 1) | +| 5 | Wait runs out | a 1 s timeout on a running crawl returns it `running`, no error (CLI: exit 0; MCP, CLI: a `note`) | +| 6 | Get | `limit` 2, then the cursor: one page each; `next_cursor` moves on and is null on the last page | +| 7 | List | `limit` 2, then the next page: newest first; `next_cursor` absent or null on the last page | +| 8 | Results | every kept URL once, each matching `CRAWL_INCLUDE`; on a running crawl, what is kept so far (MCP, CLI: `partial: true`) | +| 9 | Content | by content id and by `content_url`: the same HTML | +| 10 | Download | a status, and one line per result with `url`, `content_status`, `content` (MCP has no download) | +| 11 | Stop | running → `stopped`, `stop_reason: user`; ended → answers as it ended | +| 12 | Not found | 404 `crawl_not_found` (MCP, CLI: `CRAWL_NOT_FOUND` with `crawl_id` and `server_code`) | +| 13 | Invalid input | `depth` 0 → 422 `invalid_parameter` (MCP, CLI: `CRAWL_INVALID_REQUEST`) | +| 14 | Gated option | `output_format` json → 422 `invalid_parameter`; if it is accepted, the key is in a beta segment, not a bug | +| 15 | Bad key | the key `invalid` → 401 (MCP, CLI: `AUTH_INVALID`) | +| 16 | Errors carry the server code | SDKs: `code` holds it, empty when the body has none; MCP, CLI: `server_code` | +| 17 | Naming | the API's operation and parameter names, in this surface's casing | +| 18 | Copy | Crawl is "new", never "beta"; one "still evolving" line; the 429 limit reads as the account's active jobs, shared with Batch | +| 19 | Failed crawl | an include pattern that matches nothing ends `failed` with `no_items_found`; SDKs return it, not raise it; MCP and CLI say `ok: false` (CLI: exit 1, `CRAWL_FAILED`) | +| 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results ends at once (MCP and CLI: `partial: false`, no note to call again), and the same URLs come back once the pages in flight finish | +| 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (10) | + +Extra checks for this surface: every subcommand prints JSON with `--json` (`create`, `get`, `wait` and `stop` print `{ok, crawl_id, status, crawl, note?}`), Ctrl-C during `crawl wait` prints the resume hint and exits 130, and `--wait` and `--html` are rejected as unknown flags. + +## What it costs + +Every crawl uses `depth` 1 and `max_pages` 10 or less. A run starts five crawls: the README run, +one URLs-only crawl, one idempotency crawl that is stopped at once, one html crawl that is +stopped with its pages in flight (check 20), and one that keeps nothing (check 19). On the default `CRAWL_URL`, a URLs-only crawl fetches 1 page and the +html crawl 6, so a run costs about **10 pages**, and at most 50 if every crawl reaches its +`max_pages`. The report lists each crawl id and its `pages_fetched`. + +An account can run 3 active jobs at once, shared with Batch jobs. A run has at most 2 crawls +running at a time. If a create still gets a 429, the run waits `Retry-After` and tries again. +It never sends requests to trigger the limit. Don't run two copies against one account at the +same time. + +## How to read the report + +- **Verdict:** ready, unless there is a blocker or major finding. +- **Feature matrix:** PASS, FAIL or SKIP, with a short reason. SKIP means the check does not + apply to this surface, the client rejects the input before the API, a crawl ended before + the check could act on it, or there was no key. +- **Findings:** one entry per FAIL, most severe first, with the command and output. An install + or README failure is a blocker; a wrong answer from a feature is major; naming, copy, the + defaults and the extras are minor. +- `out/results/checks/*.json` holds every result; `out/results/container.log` is the full + output. diff --git a/acceptance/crawl/builders/build-npm.sh b/acceptance/crawl/builders/build-npm.sh new file mode 100644 index 0000000..48e2dee --- /dev/null +++ b/acceptance/crawl/builders/build-npm.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Runs in the Node image. Builds the npm tarball a release would publish: the repo's own +# install and build, then npm pack. Only artifact/*.tgz leaves this container. +set -euo pipefail +cp -a /out/source /build && cd /build +if [ -f pnpm-lock.yaml ]; then + npm install -g --no-audit --no-fund "pnpm@$PNPM_VERSION" >/dev/null + pnpm install --frozen-lockfile --store-dir /tmp/pnpm-store + pnpm run build +else + npm ci --no-audit --no-fund + npm run build +fi +rm -rf /out/artifact && mkdir -p /out/artifact +npm pack --pack-destination /out/artifact 2>&1 | tail -n 60 +ls -l /out/artifact diff --git a/acceptance/crawl/builders/fetch-source.sh b/acceptance/crawl/builders/fetch-source.sh new file mode 100644 index 0000000..51f1209 --- /dev/null +++ b/acceptance/crawl/builders/fetch-source.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Runs in the Node image when REF is set: downloads ZenRows/$REPO at $REF as GitHub's source +# tarball and reads the full commit SHA from it. +set -euo pipefail +cd /out +curl -fsSL --retry 4 -o src.tar.gz "https://codeload.github.com/ZenRows/$REPO/tar.gz/$REF" +# git get-tar-commit-id stops reading after the header, so the pipe breaks: ignore that. +(gunzip -c src.tar.gz | git get-tar-commit-id > sha) || true +[ -s sha ] || { echo "no commit id in the tarball of $REPO@$REF" >&2; exit 1; } +rm -rf source && mkdir -p source && tar -xzf src.tar.gz -C source --strip-components 1 && rm src.tar.gz +echo "fetched $REPO@$REF = $(cat sha)" diff --git a/acceptance/crawl/run.sh b/acceptance/crawl/run.sh new file mode 100644 index 0000000..03d63bb --- /dev/null +++ b/acceptance/crawl/run.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# Crawl first-install acceptance test for this repo. See README.md in this folder. +# +# ZENROWS_API_KEY=... bash run.sh build the release artifact, run every check, write the report +# bash run.sh --no-key only the checks that need no key; the live checks are SKIP +# bash run.sh --report rebuild the report from the results already in out/ +# +# Needs bash, git and Docker. The key reaches containers by name only (-e ZENROWS_API_KEY). +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=versions.env +source "$HERE/versions.env" +OUT_DIR="${OUT_DIR:-$HERE/out}" + +no_key=0; report_only=0 +for arg in "$@"; do + case "$arg" in + --no-key) no_key=1 ;; + --report) report_only=1 ;; + -h|--help) sed -n '2,8p' "$0"; exit 0 ;; + *) echo "unknown argument: $arg (expected --no-key or --report)" >&2; exit 2 ;; + esac +done + +if [ "$report_only" = 0 ] && [ "$no_key" = 0 ] && [ -z "${ZENROWS_API_KEY:-}" ]; then + echo "ZENROWS_API_KEY is not set. Set it (a key with Crawl access), or run 'bash run.sh --no-key' for the offline checks." >&2 + exit 2 +fi +if [ "$no_key" = 1 ]; then unset ZENROWS_API_KEY; fi +export ZENROWS_API_KEY="${ZENROWS_API_KEY:-}" CRAWL_URL CRAWL_INCLUDE CRAWL_API_BASE + +log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*" >&2; } +RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" +CMDLINE="bash run.sh $*" +# Unique per run, so clean-up never touches the containers of another run. +LABEL="crawl-acceptance.run=$RUN_ID-$$-$RANDOM" +docker_run() { docker run --rm --label "$LABEL" "$@"; } + +# ---- 1. Source: this checkout (default), or the repo on GitHub at $REF ---- +prepare_source() { + rm -rf "$OUT_DIR/build"; mkdir -p "$OUT_DIR/build/source" + if [ -n "${REF:-}" ]; then + log "source: ZenRows/$REPO@$REF from GitHub" + docker_run -v "$OUT_DIR/build:/out" -v "$HERE/builders:/builders:ro" \ + -e REPO="$REPO" -e REF="$REF" "$NODE_IMAGE" bash /builders/fetch-source.sh >"$OUT_DIR/build/fetch.log" 2>&1 + echo "github:$REF" >"$OUT_DIR/build/origin" + else + local root; root="$(git -C "$HERE" rev-parse --show-toplevel)" + log "source: this checkout ($root)" + # Tracked and untracked-but-not-ignored files, as a release build would see them. + (cd "$root" && git ls-files -co --exclude-standard -z | tar --null -T - -cf -) | tar -C "$OUT_DIR/build/source" -xf - + local sha; sha="$(git -C "$root" rev-parse HEAD)" + if [ -n "$(git -C "$root" status --porcelain -- . ":(exclude)$(realpath --relative-to="$root" "$HERE")")" ]; then + sha="$sha+local-changes" + fi + echo "$sha" >"$OUT_DIR/build/sha" + echo "checkout" >"$OUT_DIR/build/origin" + fi + # The test never reads the harness from the source; only the artifact goes in. + rm -rf "$OUT_DIR/build/source/acceptance" +} + +# ---- 2. Builder: the release artifact, in its own container ---- +build_artifact() { + mkdir -p "$OUT_DIR/build/artifact" + case "$SURFACE" in + node|mcp|cli) + log "build: npm pack" + docker_run -v "$OUT_DIR/build:/out" -v "$HERE/builders:/builders:ro" \ + -e SURFACE -e PNPM_VERSION "$NODE_IMAGE" bash /builders/build-npm.sh >"$OUT_DIR/build/build.log" 2>&1 ;; + python) + log "build: the wheel" + docker_run -v "$OUT_DIR/build:/out" -v "$HERE/builders:/builders:ro" \ + -e PY_BUILD_VERSION "$PYTHON_IMAGE" bash /builders/build-python.sh >"$OUT_DIR/build/build.log" 2>&1 ;; + go) + # Go has no artifact: the clean container runs `go get` at a pushed commit, as a user would. + local sha; sha="$(cat "$OUT_DIR/build/sha")" + case "$sha" in *+local-changes) + log "go: local changes are not tested; go get installs the commit ${sha%+local-changes} from GitHub" ;; + esac + echo "${sha%+local-changes}" >"$OUT_DIR/build/go_ref" ;; + esac +} + +# ---- 3. Clean test container: only the artifact, the checks and the inputs go in ---- +run_checks() { + rm -rf "$OUT_DIR/results"; mkdir -p "$OUT_DIR/results" + local image + case "$SURFACE" in node|mcp|cli) image="$NODE_IMAGE" ;; python) image="$PYTHON_IMAGE" ;; go) image="$GO_IMAGE" ;; esac + log "checks: a clean $image" + local rc=0 + docker_run --init \ + -v "$OUT_DIR/build/artifact:/artifact:ro" \ + -v "$HERE/surface:/harness:ro" \ + -v "$OUT_DIR/results:/results" \ + -e ZENROWS_API_KEY -e CRAWL_URL -e CRAWL_INCLUDE -e CRAWL_API_BASE \ + -e SURFACE -e GO_REF="$(cat "$OUT_DIR/build/go_ref" 2>/dev/null || true)" \ + -e MCP_SDK_VERSION -e TYPESCRIPT_VERSION -e MYPY_VERSION \ + -w /work --tmpfs /work:exec,size=2g \ + "$image" bash /harness/run-in-container.sh >"$OUT_DIR/results/container.log" 2>&1 || rc=$? + log "checks: the container exited $rc (out/results/container.log)" +} + +# ---- 4. Clean-up: stop every crawl this run started. Runs even when a check fails. ---- +cleanup() { + [ "$report_only" = 1 ] && return 0 + docker ps -q --filter "label=$LABEL" | xargs -r docker kill >/dev/null 2>&1 || true + if [ -n "${ZENROWS_API_KEY:-}" ] && [ -d "$OUT_DIR/results" ]; then + log "clean-up: stopping every crawl this run started" + docker_run -v "$OUT_DIR:/out" -v "$HERE/tools:/tools:ro" \ + -e ZENROWS_API_KEY -e CRAWL_API_BASE "$PYTHON_IMAGE" python -I /tools/cleanup.py /out || log "clean-up failed" + fi +} + +report() { + log "report: out/crawl-first-install-report.md" + docker_run -v "$OUT_DIR:/out" -v "$HERE/tools:/tools:ro" \ + -e CMDLINE="$CMDLINE" -e SURFACE -e REPO -e CRAWL_URL -e CRAWL_INCLUDE -e CRAWL_API_BASE \ + -e NODE_IMAGE -e PYTHON_IMAGE -e GO_IMAGE -e MCP_SDK_VERSION -e TYPESCRIPT_VERSION \ + -e MYPY_VERSION -e PY_BUILD_VERSION -e PNPM_VERSION \ + "$PYTHON_IMAGE" python -I /tools/report.py /out + # The key must appear in nothing the run wrote. Checked by value, inside a container. + if [ -n "${ZENROWS_API_KEY:-}" ]; then + docker_run -v "$OUT_DIR:/out" -v "$HERE/tools:/tools:ro" -e ZENROWS_API_KEY "$PYTHON_IMAGE" \ + python -I /tools/scan_key.py /out + fi +} + +export SURFACE REPO NODE_IMAGE PYTHON_IMAGE GO_IMAGE PNPM_VERSION PY_BUILD_VERSION \ + MCP_SDK_VERSION TYPESCRIPT_VERSION MYPY_VERSION +if [ "$report_only" = 1 ]; then report; exit 0; fi + +mkdir -p "$OUT_DIR" +echo "$RUN_ID" >"$OUT_DIR/run_id" +trap cleanup EXIT +trap 'exit 130' INT TERM +if prepare_source && build_artifact; then + run_checks +else + log "the source or build step failed (out/build/*.log)" + mkdir -p "$OUT_DIR/results" + docker_run -v "$OUT_DIR:/out" -v "$HERE/tools:/tools:ro" -e SURFACE "$PYTHON_IMAGE" python -I /tools/build_failed.py /out +fi +cleanup +trap - EXIT +report +log "done: $OUT_DIR/crawl-first-install-report.md" diff --git a/acceptance/crawl/surface/harness.mjs b/acceptance/crawl/surface/harness.mjs new file mode 100644 index 0000000..ff958b9 --- /dev/null +++ b/acceptance/crawl/surface/harness.mjs @@ -0,0 +1,223 @@ +// Helpers for the JavaScript checks (node, mcp, cli), mounted read-only at /harness. +// Node built-ins only, so they add nothing to the clean container. +import { spawnSync } from "node:child_process"; +import { appendFileSync, mkdirSync, writeFileSync } from "node:fs"; +import http from "node:http"; + +export const SURFACE = process.env.SURFACE; +export const KEY = process.env.ZENROWS_API_KEY || ""; +export const HAS_KEY = KEY.length > 0; +export const CRAWL_URL = process.env.CRAWL_URL; +export const INCLUDE = process.env.CRAWL_INCLUDE; +export const API_BASE = process.env.CRAWL_API_BASE || "https://api.zenrows.com/v1"; +/** An include pattern no URL contains: the crawl keeps nothing and ends failed (check 19). */ +export const NO_MATCH = "/crawl-acceptance-matches-nothing/"; +/** The default every client documents for max_pages and max_items (check 21). */ +export const DOCUMENTED_DEFAULT = 10; +export const PASS = "PASS"; +export const FAIL = "FAIL"; +export const SKIP = "SKIP"; +export const TERMINAL = new Set(["completed", "stopped", "failed"]); +const RESULTS = "/results"; +mkdirSync(`${RESULTS}/checks`, { recursive: true }); + +export const passIf = (ok) => (ok ? PASS : FAIL); +export const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** Replace the key, wherever it appears, before anything is written or printed. */ +export function redact(value) { + const s = typeof value === "string" ? value : JSON.stringify(value, null, 1) ?? String(value); + return KEY ? s.split(KEY).join("") : s; +} + +/** Redact, then cut long output for the evidence field. */ +export function trim(value, max = 1500) { + const s = redact(value); + return s.length > max ? `${s.slice(0, max)}… [${s.length - max} more chars]` : s; +} + +let seq = 0; +/** Write one result: {surface, check, title, status, expected, actual, evidence}. */ +export function record(check, title, status, expected, actual, evidence = "") { + const r = { + surface: SURFACE, + check: String(check), + title, + status, + expected, + actual: trim(actual, 800), + evidence: trim(evidence, 2500), + }; + seq += 1; + const name = `${String(seq).padStart(3, "0")}-${r.check.replace(/[^A-Za-z0-9_.-]+/g, "_")}.json`; + writeFileSync(`${RESULTS}/checks/${name}`, `${JSON.stringify(r, null, 2)}\n`); + console.log( + `[${status}] ${SURFACE} ${r.check} ${title}: ${r.actual.split("\n")[0].slice(0, 200)}`, + ); + return r; +} + +/** Run one check; an unexpected throw is a FAIL with the error as evidence. */ +export async function check(id, title, expected, fn) { + try { + const out = await fn(); + if (out?.status) + return record(id, title, out.status, expected, out.actual ?? "", out.evidence ?? ""); + return record(id, title, PASS, expected, out?.actual ?? "ok", out?.evidence ?? ""); + } catch (e) { + return record( + id, + title, + FAIL, + expected, + `threw: ${e?.name ?? ""} ${e?.message ?? e}`, + e?.stack ?? String(e), + ); + } +} + +/** A live check without a key is SKIP, not a failure. */ +export async function liveCheck(id, title, expected, fn) { + if (!HAS_KEY) + return record(id, title, SKIP, expected, "ZENROWS_API_KEY not set (run without --no-key)"); + return check(id, title, expected, fn); +} + +/** Resolve within `ms`, or reject: a check that pages forever fails instead of hanging the run. */ +export function within(ms, promise) { + let timer; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`did not finish within ${ms / 1000} s`)), ms); + }); + return Promise.race([promise, timeout]).finally(() => clearTimeout(timer)); +} + +/** Every crawl the checks create goes in the ledger, so clean-up can stop it and the report can count it. */ +export function ledger(crawlId, purpose) { + if (crawlId) + appendFileSync( + `${RESULTS}/crawls.jsonl`, + `${JSON.stringify({ crawl_id: crawlId, purpose })}\n`, + ); +} + +/** The names this surface uses, for check 17. */ +export function naming(map) { + writeFileSync(`${RESULTS}/naming.json`, `${JSON.stringify(map, null, 2)}\n`); +} + +/** Run a command and wait. The env carries the key, never argv. */ +export function sh(cmd, args, opts = {}) { + const r = spawnSync(cmd, args, { encoding: "utf8", maxBuffer: 64 * 1024 * 1024, ...opts }); + return { + code: r.status ?? (r.signal ? 128 : 1), + stdout: r.stdout ?? "", + stderr: r.stderr ?? (r.error ? String(r.error) : ""), + cmd: [cmd, ...args].join(" "), + }; +} + +/** Plain call to the Crawl API, for setup only (finding the README run's crawls). */ +export async function api(method, path) { + const res = await fetch(`${API_BASE}${path}`, { method, headers: { "X-API-Key": KEY } }); + const text = await res.text(); + let json; + try { + json = JSON.parse(text); + } catch { + json = undefined; + } + return { status: res.status, json }; +} + +/** The crawls created at or after `since` (ms), newest first. */ +export async function crawlsSince(since) { + if (!HAS_KEY) return []; + const r = await api("GET", "/crawls?limit=100"); + if (r.status !== 200) return []; + return r.json.crawls.filter((c) => Date.parse(c.created_at) >= since).map((c) => c.crawl_id); +} + +const runningCrawl = () => ({ + crawl_id: "c_running", + status: "running", + url: "https://example.com/", + depth: 1, + max_items: 10, + max_pages: 10, + coverage: { pages_fetched: 0, pages_failed: 0, items_found: 0 }, + created_at: new Date().toISOString(), +}); + +/** + * A local stand-in for the Crawl API that records what the client sends. It answers a create + * with a running crawl, `GET /crawls/c_running` with a crawl that never ends, `GET + * /crawls/c_nobody` with a 502 that has no body, and anything else with a 404 problem. + * Call the client asynchronously while it runs: a blocking call starves it. + */ +export async function captureServer() { + const requests = []; + const server = http.createServer((req, res) => { + let raw = ""; + req.on("data", (c) => { + raw += c; + }); + req.on("end", () => { + let body; + try { + body = raw ? JSON.parse(raw) : undefined; + } catch { + body = raw; + } + requests.push({ method: req.method, url: req.url, headers: req.headers, body }); + if (req.method === "POST" && req.url.replace(/\?.*$/, "").endsWith("/crawls")) { + res.writeHead(201, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ...runningCrawl(), crawl_id: "c_capture", url: body?.url ?? "" })); + } else if (req.url.includes("c_running")) { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ...runningCrawl(), results: [], next_cursor: "n" })); + } else if (req.url.includes("c_nobody")) { + res.writeHead(502); + res.end(); + } else { + res.writeHead(404, { "Content-Type": "application/problem+json" }); + res.end( + JSON.stringify({ + status: 404, + title: "Not Found", + code: "crawl_not_found", + detail: "stand-in", + }), + ); + } + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address(); + return { + base: `http://127.0.0.1:${port}/v1`, + requests, + close: () => new Promise((r) => server.close(r)), + }; +} + +/** True when `obj` has exactly these keys, in any order. */ +export function sameKeys(obj, keys) { + const got = Object.keys(obj ?? {}).sort(); + const want = [...keys].sort(); + return got.length === want.length && got.every((k, i) => k === want[i]); +} + +/** Check 18 on a text: "new" not "beta" on Crawl lines, one "still evolving" line, the 429 wording. */ +export function copyFindings(text) { + const lines = text.split("\n"); + const beta = lines.filter((l) => /crawl/i.test(l) && /\bbeta\b/i.test(l)); + const evolving = lines.filter((l) => /still evolving/i.test(l)); + const limit = lines.filter((l) => /active jobs/i.test(l)); + const problems = []; + if (beta.length) problems.push(`"beta" on a Crawl line: ${beta[0].trim().slice(0, 160)}`); + if (evolving.length !== 1) problems.push(`${evolving.length} "still evolving" lines (want 1)`); + if (!limit.some((l) => /batch/i.test(l))) + problems.push("no line reads the 429 limit as the account's active jobs, shared with Batch"); + return { problems, evolving, limit }; +} diff --git a/acceptance/crawl/surface/run-in-container.sh b/acceptance/crawl/surface/run-in-container.sh new file mode 100644 index 0000000..726e44d --- /dev/null +++ b/acceptance/crawl/surface/run-in-container.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Entry point in the clean node container: record the tool versions, then run the checks. +set -uo pipefail +printf '{"node":"%s","npm":"%s","os":"%s"}\n' "$(node -v)" "$(npm -v)" "$(. /etc/os-release; echo "$PRETTY_NAME")" > /results/env.json +export npm_config_update_notifier=false npm_config_fund=false npm_config_audit=false +node /harness/test.mjs diff --git a/acceptance/crawl/surface/test.mjs b/acceptance/crawl/surface/test.mjs new file mode 100644 index 0000000..33cb3f5 --- /dev/null +++ b/acceptance/crawl/surface/test.mjs @@ -0,0 +1,890 @@ +// CLI (`@zenrows/cli`): the README walk-through, checks 1-21 and the CLI extra checks. +// Installs the release tarball globally, as the README's `npm install -g @zenrows/cli` does, +// and drives `zenrows crawl …` as a user would. The key reaches the CLI by environment only. +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, readdirSync } from "node:fs"; +import * as H from "/harness/harness.mjs"; + +const { + PASS, + FAIL, + SKIP, + passIf, + record, + check, + liveCheck, + ledger, + sh, + KEY, + HAS_KEY, + CRAWL_URL, + INCLUDE, +} = H; +const APP = "/work/app"; +const HOME = "/work/home"; +const tarball = `/artifact/${readdirSync("/artifact").find((f) => f.endsWith(".tgz"))}`; +mkdirSync(APP, { recursive: true }); +mkdirSync(HOME, { recursive: true }); + +// Without a key, "invalid" keeps the CLI from creating an account. +const baseEnv = { ...process.env, HOME, NO_COLOR: "1", ZENROWS_API_KEY: HAS_KEY ? KEY : "invalid" }; +const parse = (s) => { + try { + return JSON.parse(s); + } catch { + return undefined; + } +}; +/** Run `zenrows ` and wait. */ +function z(args, env = {}) { + const r = sh("zenrows", args, { cwd: APP, env: { ...baseEnv, ...env }, timeout: 15 * 60_000 }); + return { ...r, json: parse(r.stdout), cmd: `zenrows ${args.join(" ")}` }; +} +/** Run `zenrows ` without blocking, for calls the in-process stand-in server answers. */ +function zAsync(args, env = {}) { + return new Promise((resolve) => { + const child = spawn("zenrows", args, { cwd: APP, env: { ...baseEnv, ...env } }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (d) => { + stdout += d; + }); + child.stderr.on("data", (d) => { + stderr += d; + }); + child.on("exit", (code) => + resolve({ code, stdout, stderr, json: parse(stdout), cmd: `zenrows ${args.join(" ")}` }), + ); + }); +} +const show = (r) => + H.trim( + `$ ${r.cmd}\nexit ${r.code}\n${r.stdout}${r.stderr ? `\n[stderr] ${r.stderr}` : ""}`, + 1500, + ); +const idOf = (text) => text.match(/\bc_[0-9A-Z]{20,}\b/)?.[0]; + +// ---------- README: install and init ---------- +const install = sh("npm", ["install", "-g", tarball], { cwd: APP, env: baseEnv }); +record( + "readme-1", + "Install (README: `npm install -g @zenrows/cli`; here the release tarball)", + passIf(install.code === 0), + "zenrows is on PATH", + `exit ${install.code}; zenrows ${z(["--version"]).stdout.trim()}`, + `${install.cmd}\n${install.stdout}${install.stderr}`, +); +if (install.code !== 0) process.exit(1); +const pkgReadme = readFileSync( + `${sh("npm", ["root", "-g"]).stdout.trim()}/@zenrows/cli/README.md`, + "utf8", +); +const init = z(["init"]); +record( + "readme-1b", + "`zenrows init` (README quickstart)", + passIf(init.code === 0), + "the workspace is created", + `exit ${init.code}`, + show(init), +); + +// ---------- README: the Crawl section, line by line ---------- +const section = pkgReadme.split(/^## \d+\. Crawl \(new\)\s*$/m)[1]?.split(/^## /m)[0] ?? ""; +const block = section.match(/```bash\n([\s\S]*?)```/)?.[1] ?? ""; +record( + "readme-2", + "README has a Crawl section with commands", + passIf(block.length > 0), + "a ```bash block under the Crawl (new) section", + `${block.split("\n").filter(Boolean).length} command lines`, + block || section.slice(0, 600), +); + +await liveCheck( + "readme-3", + "Run the README's Crawl commands", + "each command runs; skipped lines are named", + async () => { + // Join the backslash-continued line, then take each command without its comment. + const lines = block + .replace(/\\\n\s*/g, "") + .split("\n") + .map((l) => l.replace(/\s+#.*$/, "").trim()) + .filter(Boolean); + const log = []; + let id; + let failed = 0; + for (const line of lines) { + let args = line.split(/\s+/).slice(1); + if (args[1] === "create" && args.includes("--output-format")) { + log.push( + `SKIP ${line}\n (depth 2 and html break the cost rules; checks 2, 9 and 10 cover html)`, + ); + continue; + } + if (args[1] === "content") { + log.push(`SKIP ${line}\n (needs an html crawl; check 9 covers it)`); + continue; + } + let note = ""; + if (args[1] === "create") { + args = args.map((a) => + a === "https://example.com/products/" ? CRAWL_URL : a === "/product/" ? INCLUDE : a, + ); + const mi = args.indexOf("--max-items"); + if (mi >= 0 && Number(args[mi + 1]) > 5) { + args[mi + 1] = "5"; + note = " (start URL → CRAWL_URL, --max-items → 5)"; + } + } + if (args.includes("")) { + if (!id) { + log.push(`SKIP ${line}\n (no crawl id from the create line)`); + continue; + } + args = args.map((a) => (a === "" ? id : a)); + } + const r = z(args); + if (args[1] === "create") { + id = idOf(r.stdout + r.stderr); + if (id) ledger(id, "README quickstart"); + } + if (r.code !== 0) failed++; + const outLine = H.trim((r.stdout + r.stderr).trim().split("\n").slice(0, 4).join(" | "), 300); + log.push( + `${r.code === 0 ? "OK " : "FAIL"} zenrows ${args.join(" ")}${note}\n exit ${r.code}: ${outLine}`, + ); + } + return { + status: passIf(failed === 0 && Boolean(id)), + actual: `${lines.length} lines, ${failed} failed; crawl ${id ?? "none"}`, + evidence: log.join("\n"), + }; + }, +); + +// ---------- extras against the stand-in: unknown flags, Ctrl-C during wait ---------- +const cap = await H.captureServer(); +const capEnv = { ZENROWS_CRAWL_API_BASE: cap.base, ZENROWS_API_KEY: "capture-key" }; +await check( + "x-cli-unknown-flags", + "--wait and --html are rejected as unknown flags", + "both exit non-zero with UNKNOWN_FLAG; nothing is sent", + async () => { + const before = cap.requests.length; + const w = await zAsync( + ["crawl", "create", "https://example.com/", "--depth", "1", "--wait", "--json"], + capEnv, + ); + const h = await zAsync( + ["crawl", "create", "https://example.com/", "--depth", "1", "--html", "--json"], + capEnv, + ); + const sentNow = cap.requests.length - before; + const ok = + w.code !== 0 && + h.code !== 0 && + w.json?.error?.code === "UNKNOWN_FLAG" && + h.json?.error?.code === "UNKNOWN_FLAG" && + sentNow === 0; + return { + status: passIf(ok), + actual: `--wait: exit ${w.code} ${w.json?.error?.code}; --html: exit ${h.code} ${h.json?.error?.code}; requests sent ${sentNow}`, + evidence: `${show(w)}\n${show(h)}`, + }; + }, +); +await check( + "x-cli-ctrl-c", + "Ctrl-C during `crawl wait` prints the resume hint and exits 130", + "exit 130 and `zenrows crawl wait ` in the output", + async () => { + const child = spawn("zenrows", ["crawl", "wait", "c_running"], { + cwd: APP, + env: { ...baseEnv, ...capEnv }, + }); + let out = ""; + child.stdout.on("data", (d) => { + out += d; + }); + child.stderr.on("data", (d) => { + out += d; + }); + await H.sleep(3000); + child.kill("SIGINT"); + const code = await new Promise((resolve) => + child.on("exit", (c, s) => resolve(c ?? (s ? 128 : 1))), + ); + const hint = /zenrows crawl wait c_running/.test(out); + return { + status: passIf(code === 130 && hint), + actual: `exit ${code}; resume hint ${hint}`, + evidence: `zenrows crawl wait c_running against a stand-in that always answers running; SIGINT after 3 s\n${out}`, + }; + }, +); + +// ---------- check 1, offline half: what the CLI sends ---------- +const before = cap.requests.length; +await zAsync(["crawl", "create", "https://example.com/", "--depth", "1", "--json"], capEnv); +await zAsync( + [ + "crawl", + "create", + "https://example.com/", + "--depth", + "1", + "--max-items", + "5", + "--max-pages", + "10", + "--include-patterns", + "/p/", + "--idempotency-key", + "idem-1", + "--json", + ], + capEnv, +); +const posts = cap.requests.slice(before).filter((r) => r.method === "POST"); +const sent = { minimal: posts[0]?.body, full: posts[1]?.body }; +const sentOk = + H.sameKeys(sent.minimal, ["url", "depth"]) && + H.sameKeys(sent.full, ["url", "depth", "max_items", "max_pages", "include_patterns"]); +const sentLine = `sent (stand-in server): minimal=${JSON.stringify(sent.minimal)} full=${JSON.stringify(sent.full)}`; +await cap.close(); + +// ---------- live ---------- +const baseArgs = [ + CRAWL_URL, + "--depth", + "1", + "--max-items", + "5", + "--max-pages", + "10", + "--include-patterns", + INCLUDE, +]; +function createWith(args) { + for (let i = 0; i < 10; i++) { + const r = z(["crawl", "create", ...args, "--json"]); + if (r.json?.error?.code === "CRAWL_TOO_MANY_CRAWLS") { + sh("sleep", [String(r.json.error.retry_after ?? 30)]); // the account's active jobs are shared: wait as told + continue; + } + if (r.json?.crawl_id) ledger(r.json.crawl_id, "check"); + return r; + } + throw new Error("still 429 after 10 tries"); +} +const create = (extra = []) => createWith([...baseArgs, ...extra]); +const isEnvelope = (j) => + j?.ok === true && + typeof j.crawl_id === "string" && + typeof j.status === "string" && + typeof j.crawl === "object"; +const ENVELOPE_KEYS = new Set([ + "ok", + "crawl_id", + "status", + "crawl", + "note", + "results", + "next_cursor", + "error", +]); +const shapes = {}; +const noteShape = (sub, r) => { + shapes[sub] = { exit: r.code, keys: r.json ? Object.keys(r.json) : null }; +}; + +let A; +let B; +let C; +let F; +let ended; +await check( + "1", + "Create", + "returns at once: {ok, crawl_id, status: running, crawl}; only the fields set are sent", + async () => { + if (!HAS_KEY) { + return { + status: sentOk ? SKIP : FAIL, + actual: `live part skipped (no key); only set fields sent: ${sentOk}`, + evidence: sentLine, + }; + } + const r = create(); + noteShape("create", r); + A = r.json; + const c = A?.crawl ?? {}; + const echo = JSON.stringify([c.url, c.depth, c.max_items, c.max_pages, c.include_patterns]); + const ok = + sentOk && + r.code === 0 && + isEnvelope(A) && + A.status === "running" && + echo === JSON.stringify([CRAWL_URL, 1, 5, 10, [INCLUDE]]); + return { + status: passIf(ok), + actual: `exit ${r.code}; ${A?.crawl_id} ${A?.status}; only set fields sent: ${sentOk}`, + evidence: `${show(r)}\n${sentLine}`, + }; + }, +); +await liveCheck( + "5", + "Wait runs out", + "crawl wait --timeout 1 on a running crawl: exit 0, running, a note", + async () => { + if (!A) return { status: SKIP, actual: "no crawl from check 1" }; + const r = z(["crawl", "wait", A.crawl_id, "--timeout", "1", "--json"]); + noteShape("wait (ran out)", r); + if (r.json?.status && r.json.status !== "running") + return { status: SKIP, actual: `the crawl had already ended (${r.json.status})` }; + return { + status: passIf( + r.code === 0 && r.json?.status === "running" && typeof r.json?.note === "string", + ), + actual: `exit ${r.code}; ${r.json?.status}; note: ${r.json?.note ?? "none"}`, + evidence: show(r), + }; + }, +); +const partial = A ? z(["crawl", "results", A.crawl_id, "--json"]) : undefined; + +await liveCheck( + "3", + "Idempotency", + "same key + body → same crawl_id; different body → CRAWL_INVALID_REQUEST and the advice to send a new key", + async () => { + const small = [ + CRAWL_URL, + "--depth", + "1", + "--max-items", + "3", + "--max-pages", + "3", + "--include-patterns", + INCLUDE, + "--idempotency-key", + `acc-${randomUUID()}`, + "--json", + ]; + const r1 = z(["crawl", "create", ...small]); + C = r1.json; + if (C?.crawl_id) { + ledger(C.crawl_id, "check"); + // Check 11's first stop, right away: a CLI call takes about a second, and a small crawl can end on its own in two. + C.firstStop = z(["crawl", "stop", C.crawl_id, "--json"]); + } + const r2 = z(["crawl", "create", ...small]); + const r3 = z(["crawl", "create", ...small.map((a) => (a === "3" ? "4" : a))]); + if (r3.json?.crawl_id) + ledger(r3.json.crawl_id, "idempotency, different body (unexpected create)"); + const e = r3.json?.error; + const ok = + Boolean(C?.crawl_id) && + C.crawl_id === r2.json?.crawl_id && + e?.code === "CRAWL_INVALID_REQUEST" && + e?.server_code === "idempotency_key_reused" && + /new key/i.test(e?.next_action ?? ""); + return { + status: passIf(ok), + actual: `first ${C?.crawl_id}, second ${r2.json?.crawl_id}; different body → exit ${r3.code} ${e?.code} (${e?.server_code}): ${e?.next_action}`, + evidence: show(r3), + }; + }, +); +await liveCheck( + "11", + "Stop", + "running → stopped, stop_reason user; ended → answers as it ended, no error", + async () => { + if (!C?.firstStop) return { status: SKIP, actual: "no crawl from check 3" }; + const s1 = C.firstStop; + noteShape("stop", s1); + if (s1.json?.status === "completed" && s1.json?.crawl?.stop_reason !== "user") { + return { + status: SKIP, + actual: `the crawl had already ended on its own (${s1.json.status}/${s1.json.crawl?.stop_reason})`, + }; + } + const s2 = z(["crawl", "stop", C.crawl_id, "--json"]); + const ok = + s1.code === 0 && + s1.json?.status === "stopped" && + s1.json?.crawl?.stop_reason === "user" && + s2.code === 0 && + s2.json?.status === "stopped"; + return { + status: passIf(ok), + actual: `first: exit ${s1.code} ${s1.json?.status}/${s1.json?.crawl?.stop_reason}; second: exit ${s2.code} ${s2.json?.status}`, + evidence: `${show(s1)}\n${show(s2)}`, + }; + }, +); +await liveCheck("4", "Wait", "crawl wait returns the ended crawl; failed exits 1", async () => { + if (!A) return { status: SKIP, actual: "no crawl from check 1" }; + const r = z(["crawl", "wait", A.crawl_id, "--json"]); + noteShape("wait", r); + ended = r.json; + const st = r.json?.status; + return { + status: passIf(H.TERMINAL.has(st) && r.code === (st === "failed" ? 1 : 0)), + actual: `exit ${r.code}; ${st}/${r.json?.crawl?.stop_reason ?? "-"}; coverage ${JSON.stringify(r.json?.crawl?.coverage)}`, + evidence: show(r), + }; +}); +await liveCheck( + "6", + "Get", + "crawl get --limit 2 then --cursor: one page each; next_cursor moves on and is null on the last page", + async () => { + if (!ended) return { status: SKIP, actual: "no ended crawl" }; + const pages = []; + let cursor; + for (let i = 0; i < 20; i++) { + const r = z([ + "crawl", + "get", + A.crawl_id, + "--limit", + "2", + ...(cursor ? ["--cursor", cursor] : []), + "--json", + ]); + if (i === 0) noteShape("get", r); + if (r.code !== 0) return { status: FAIL, actual: show(r) }; + pages.push({ + n: r.json.results.length, + next: r.json.next_cursor, + urls: r.json.results.map((x) => x.url), + }); + if (r.json.next_cursor === null) break; + cursor = r.json.next_cursor; + } + const urls = pages.flatMap((p) => p.urls); + const curs = pages.map((p) => p.next).filter((c) => c !== null); + const ok = + pages.every((p) => p.n <= 2) && + pages.at(-1).next === null && + new Set(curs).size === curs.length && + new Set(urls).size === urls.length && + (pages.length >= 2 || ended.crawl.coverage.items_found <= 2); + return { + status: passIf(ok), + actual: `${pages.length} pages of ${pages.map((p) => p.n).join(",")}; last next_cursor ${pages.at(-1).next}`, + }; + }, +); +await liveCheck( + "7", + "List", + "crawl list --limit 2 then --cursor: newest first; next_cursor null on the last page", + async () => { + const p1 = z(["crawl", "list", "--limit", "2", "--json"]); + noteShape("list", p1); + const p2 = p1.json?.next_cursor + ? z(["crawl", "list", "--limit", "2", "--cursor", p1.json.next_cursor, "--json"]) + : { json: { crawls: [] } }; + const all = [...(p1.json?.crawls ?? []), ...(p2.json?.crawls ?? [])]; + const t = all.map((c) => Date.parse(c.created_at)); + const newest = t.every((x, i) => i === 0 || t[i - 1] >= x); + let cursor = p2.json?.next_cursor; + let pages = 2; + while (cursor && pages < 60) { + cursor = z(["crawl", "list", "--limit", "100", "--cursor", cursor, "--json"]).json + ?.next_cursor; + pages++; + } + const ok = + p1.json?.crawls?.length === 2 && + newest && + new Set(all.map((c) => c.crawl_id)).size === all.length && + !cursor; + return { + status: passIf(ok), + actual: `newest first ${newest}; last page after ${pages} pages: next_cursor ${JSON.stringify(cursor ?? null)}`, + }; + }, +); +await liveCheck( + "8", + "Results", + "every kept URL once, each matching CRAWL_INCLUDE; partial: true on a running crawl", + async () => { + if (!ended) return { status: SKIP, actual: "no ended crawl" }; + const r = z(["crawl", "results", A.crawl_id, "--limit", "2", "--json"]); + noteShape("results", r); + const urls = (r.json?.results ?? []).map((x) => x.url); + const ok = + r.code === 0 && + urls.length === ended.crawl.coverage.items_found && + new Set(urls).size === urls.length && + urls.every((u) => u.includes(INCLUDE)) && + r.json.partial === false; + const run = partial + ? `while running: ${partial.json?.status}, partial ${partial.json?.partial}, ${partial.json?.count} URLs` + : "not read"; + const partialOk = + !partial || partial.json?.status !== "running" || partial.json?.partial === true; + return { + status: passIf(ok && partialOk), + actual: `${urls.length} URLs (items_found ${ended.crawl.coverage.items_found}), partial ${r.json?.partial}; ${run}`, + evidence: urls.join("\n"), + }; + }, +); +// ---------- the html crawl (checks 20, 2, 9, 10): stopped with pages in flight, read, then settled ---------- +let stopInfo; +/** Poll until no result is still `pending` (the pages in flight finished), at most 90 s. */ +function settle(id) { + for (let i = 0; i < 45; i++) { + const g = z(["crawl", "get", id, "--limit", "100", "--json"]); + if (!(g.json?.results ?? []).some((r) => r.content_status === "pending")) return "settled"; + sh("sleep", ["2"]); + } + return "still pending after 90 s"; +} +if (HAS_KEY) { + const r = create(["--output-format", "html"]); + B = r.json; + if (r.code !== 0 || !B?.crawl_id) stopInfo = { error: show(r) }; + else { + // Stop as soon as the crawl has kept URLs: their pages are then still being fetched. + let g; + for (let i = 0; i < 40; i++) { + g = z(["crawl", "get", B.crawl_id, "--limit", "1", "--json"]); + if (g.json?.status !== "running" || g.json?.crawl?.coverage?.items_found > 0) break; + } + const st = z(["crawl", "stop", B.crawl_id, "--json"]); + stopInfo = { items: g.json?.crawl?.coverage?.items_found, status: st.json?.status }; + } +} +await liveCheck( + "20", + "Results after stop", + "crawl results on a crawl stopped with pages in flight: partial false, and no URL lost", + async () => { + if (!B || stopInfo?.error) return { status: FAIL, actual: `setup failed: ${stopInfo?.error}` }; + if (stopInfo.status !== "stopped") { + return { + status: SKIP, + actual: `the crawl had already ended (${stopInfo.status}) before the stop`, + }; + } + const now = z(["crawl", "results", B.crawl_id, "--json"]); + const j = now.json ?? {}; + const settled = settle(B.crawl_id); + const settledNote = + settled === "settled" + ? settled + : `${settled} (an API issue: the stop left these pages pending on an ended crawl)`; + const later = z(["crawl", "results", B.crawl_id, "--json"]).json?.results ?? []; + const urls = (rows) => JSON.stringify((rows ?? []).map((x) => x.url).sort()); + const same = urls(j.results) === urls(later); + const pending = (j.results ?? []).filter((x) => x.content_status === "pending").length; + return { + status: passIf(now.code === 0 && j.partial === false && same), + actual: `stopped with ${stopInfo.items} kept; first read: exit ${now.code}, ${j.count} URLs (${pending} pending), partial ${j.partial}; then ${settledNote}: ${later.length} URLs, same set ${same}`, + evidence: show(now), + }; + }, +); +await liveCheck( + "2", + "Create with html", + "results get content_status and, once fetched, content_url", + async () => { + if (!B?.crawl_id) return { status: SKIP, actual: "no html crawl" }; + if (stopInfo?.status === "stopped") settle(B.crawl_id); + const rs = z(["crawl", "results", B.crawl_id, "--json"]).json?.results ?? []; + const ok = + B.crawl?.output_format === "html" && + rs.length > 0 && + rs.every((x) => x.content_status && (x.content_status !== "fetched" || x.content_url)); + return { + status: passIf(ok), + actual: `${B.crawl_id} (${stopInfo?.status ?? "-"}); ${rs.length} results; with content_url ${rs.filter((x) => x.content_url).length}`, + }; + }, +); +await liveCheck( + "9", + "Content", + "crawl content by content id, then by content_url: the same HTML", + async () => { + if (!B) return { status: SKIP, actual: "no html crawl" }; + const row = (z(["crawl", "results", B.crawl_id, "--json"]).json?.results ?? []).find( + (x) => x.content_url, + ); + if (!row) + return stopInfo?.status === "stopped" + ? { + status: SKIP, + actual: + "the stop cancelled every page fetch, so no page has content to read (see check 20)", + } + : { status: FAIL, actual: "no result with content_url" }; + const id = row.content_url.split("/").pop(); + const a = z(["crawl", "content", B.crawl_id, id, "--out", "/work/a.html"]); + const b = z(["crawl", "content", B.crawl_id, row.content_url, "--out", "/work/b.html"]); + noteShape("content", z(["crawl", "content", B.crawl_id, id, "--json"])); + const ha = existsSync("/work/a.html") ? readFileSync("/work/a.html", "utf8") : ""; + const hb = existsSync("/work/b.html") ? readFileSync("/work/b.html", "utf8") : ""; + const ok = a.code === 0 && b.code === 0 && ha === hb && / { + if (!B) return { status: SKIP, actual: "no html crawl" }; + const r = z(["crawl", "download", B.crawl_id, "--out", "/work/dl.jsonl", "--json"]); + noteShape("download", r); + const lines = existsSync("/work/dl.jsonl") + ? readFileSync("/work/dl.jsonl", "utf8") + .split("\n") + .filter(Boolean) + .map((l) => JSON.parse(l)) + : []; + const fetched = lines.filter((l) => l.content_status === "fetched"); + const ok = + r.code === 0 && + typeof r.json?.status === "string" && + lines.length > 0 && + lines.every((l) => l.url && l.content_status) && + fetched.every((l) => typeof l.content === "string" && l.content); + return { + status: passIf(ok), + actual: `exit ${r.code}; status ${r.json?.status}; ${lines.length} lines; ${fetched.length} with content`, + evidence: show(r), + }; + }, +); + +const errs = {}; +await liveCheck("12", "Not found", "CRAWL_NOT_FOUND, with crawl_id and server_code", async () => { + const r = z(["crawl", "get", "c_does_not_exist", "--json"]); + errs.notFound = r.json?.error; + const e = errs.notFound; + return { + status: passIf( + r.code !== 0 && + e?.code === "CRAWL_NOT_FOUND" && + e?.crawl_id === "c_does_not_exist" && + e?.server_code === "crawl_not_found", + ), + actual: `exit ${r.code} ${JSON.stringify(e)}`, + }; +}); +await liveCheck( + "13", + "Invalid input", + "crawl create --depth 0 → CRAWL_INVALID_REQUEST", + async () => { + const r = z(["crawl", "create", CRAWL_URL, "--depth", "0", "--json"]); + if (r.json?.crawl_id) { + ledger(r.json.crawl_id, "depth 0 (unexpected create)"); + z(["crawl", "stop", r.json.crawl_id]); + } + errs.invalid = r.json?.error; + const ok = errs.invalid?.code === "CRAWL_INVALID_REQUEST"; + return { + status: passIf(ok), + actual: ok + ? `exit ${r.code} ${JSON.stringify(errs.invalid)}` + : `rejected before the API: exit ${r.code} ${errs.invalid?.code}: ${errs.invalid?.message}`, + evidence: show(r), + }; + }, +); +await liveCheck("14", "Gated option", "--output-format json → 422 invalid_parameter", async () => { + const r = z(["crawl", "create", CRAWL_URL, "--depth", "1", "--output-format", "json", "--json"]); + if (r.json?.crawl_id) { + ledger(r.json.crawl_id, "output_format json (accepted)"); + z(["crawl", "stop", r.json.crawl_id]); + return { + status: PASS, + actual: `accepted (${r.json.crawl_id}): the key is in a beta segment, not a bug; stopped it`, + }; + } + return { + status: SKIP, + actual: `the client does not allow it: ${r.json?.error?.code}: ${r.json?.error?.message}`, + }; +}); +await check("15", "Bad key", "AUTH_INVALID", async () => { + const r = z(["crawl", "get", "c_does_not_exist", "--json"], { ZENROWS_API_KEY: "invalid" }); + errs.badKey = r.json?.error; + return { + status: passIf(r.code !== 0 && errs.badKey?.code === "AUTH_INVALID"), + actual: `exit ${r.code} ${JSON.stringify(errs.badKey)}`, + }; +}); +await check( + "16", + "Errors carry the server code", + "server_code on CRAWL_NOT_FOUND, CRAWL_INVALID_REQUEST and AUTH_INVALID", + async () => { + const fromApi = errs.invalid?.code === "CRAWL_INVALID_REQUEST"; + const rows = { + notFound: errs.notFound?.server_code, + invalid: fromApi ? errs.invalid.server_code : "n/a: rejected before the API (see 13)", + badKey: errs.badKey?.server_code, + }; + const liveOk = + !HAS_KEY || + (rows.notFound === "crawl_not_found" && (!fromApi || rows.invalid === "invalid_parameter")); + const ok = liveOk && Boolean(rows.badKey); + return { status: ok ? (HAS_KEY ? PASS : SKIP) : FAIL, actual: JSON.stringify(rows) }; + }, +); + +// ---------- 19-21: a failed crawl, results after a stop, the documented defaults ---------- +await liveCheck( + "19", + "Failed crawl", + "a crawl that keeps nothing ends failed (no_items_found); crawl wait exits 1 with ok: false and CRAWL_FAILED", + async () => { + const r = createWith([CRAWL_URL, "--depth", "1", "--include-patterns", H.NO_MATCH]); + F = r.json; + if (!F?.crawl_id) return { status: FAIL, actual: show(r) }; + const w = z(["crawl", "wait", F.crawl_id, "--json"]); + const j = w.json ?? {}; + const ok = + w.code === 1 && + j.ok === false && + j.status === "failed" && + j.crawl?.error?.code === "no_items_found" && + j.error?.code === "CRAWL_FAILED" && + j.error?.server_code === "no_items_found"; + return { + status: passIf(ok), + actual: `${F.crawl_id}: exit ${w.code}, status ${j.status}, ok ${j.ok}, error ${j.error?.code}/${j.error?.server_code}`, + evidence: show(w), + }; + }, +); +await liveCheck( + "21", + "Documented defaults", + `a create without --max-pages and --max-items echoes the documented default (${H.DOCUMENTED_DEFAULT})`, + async () => { + if (!F?.crawl) return { status: SKIP, actual: "no crawl from check 19" }; + const ok = + F.crawl.max_pages === H.DOCUMENTED_DEFAULT && F.crawl.max_items === H.DOCUMENTED_DEFAULT; + const help = z(["crawl", "--help"]); + const doc = `${help.stdout}${help.stderr}` + .split("\n") + .filter((l) => /max-(pages|items)/.test(l)); + return { + status: passIf(ok), + actual: `the API applied max_pages ${F.crawl.max_pages}, max_items ${F.crawl.max_items}; documented ${H.DOCUMENTED_DEFAULT}`, + evidence: `zenrows crawl --help:\n${doc.join("\n")}`, + }; + }, +); + +await liveCheck( + "x-cli-json", + "Every subcommand works with --json; create, get, wait, stop print {ok, crawl_id, status, crawl, note?}", + "valid JSON from all 8 subcommands; the envelope on the four", + async () => { + const problems = []; + for (const [sub, s] of Object.entries(shapes)) { + if (!s.keys) problems.push(`${sub}: stdout is not JSON`); + else if (["create", "get", "wait", "wait (ran out)", "stop"].includes(sub)) { + const extra = s.keys.filter((k) => !ENVELOPE_KEYS.has(k)); + const missing = ["ok", "crawl_id", "status", "crawl"].filter((k) => !s.keys.includes(k)); + if (extra.length || missing.length) + problems.push(`${sub}: extra [${extra}], missing [${missing}]`); + } + } + const missingSubs = [ + "create", + "get", + "list", + "results", + "content", + "download", + "stop", + "wait", + ].filter((s) => !shapes[s]); + if (missingSubs.length) problems.push(`not exercised: ${missingSubs.join(", ")}`); + return { + status: passIf(problems.length === 0), + actual: problems.join("; ") || "all 8 subcommands print JSON; the envelope holds", + evidence: JSON.stringify(shapes), + }; + }, +); + +H.naming({ + operations: { + create: "crawl create", + get: "crawl get", + list: "crawl list", + results: "crawl results", + content: "crawl content", + download: "crawl download", + stop: "crawl stop", + wait: "crawl wait", + }, + params: { + url: "", + depth: "--depth", + max_items: "--max-items", + max_pages: "--max-pages", + include_patterns: "--include-patterns", + exclude_patterns: "--exclude-patterns", + output_format: "--output-format", + idempotency_key: "--idempotency-key", + cursor: "--cursor", + limit: "--limit", + timeout: "--timeout", + }, +}); +record("17", "Naming", SKIP, "the API's names in this surface's casing", "computed by the report"); + +await check( + "18", + "Copy", + 'Crawl is "new", never "beta"; one "still evolving" line; the 429 limit is active jobs shared with Batch', + async () => { + const help = z(["crawl", "--help"]); + const helpText = `${help.stdout}${help.stderr}`; + const top = z(["--help"]); + const topCrawl = `${top.stdout}${top.stderr}`.split("\n").filter((l) => /crawl/i.test(l)); + const row = pkgReadme.split("\n").filter((l) => l.includes("`zenrows crawl`")); + const problems = []; + const beta = [...helpText.split("\n"), ...topCrawl, ...row, ...section.split("\n")].filter( + (l) => /\bbeta\b/i.test(l), + ); + if (beta.length) problems.push(`"beta": ${beta[0].trim().slice(0, 160)}`); + const evolving = pkgReadme.split("\n").filter((l) => /still evolving/i.test(l)).length; + if (evolving !== 1) problems.push(`${evolving} "still evolving" lines in the README (want 1)`); + if (!/active jobs[\s\S]{0,60}Batch/i.test(section)) + problems.push("the README does not read the 429 limit as active jobs shared with Batch"); + if (!/\bnew\b/i.test(helpText)) problems.push('`zenrows crawl --help` does not say "new"'); + return { + status: passIf(problems.length === 0), + actual: + problems.join("; ") || 'README and help say "new"; 1 "still evolving" line; 429 wording ok', + evidence: `${row.join("\n")}\n--- zenrows --help (crawl lines)\n${topCrawl.join("\n")}\n--- zenrows crawl --help\n${helpText}`, + }; + }, +); +console.log("cli: done"); diff --git a/acceptance/crawl/tools/build_failed.py b/acceptance/crawl/tools/build_failed.py new file mode 100644 index 0000000..3ac8717 --- /dev/null +++ b/acceptance/crawl/tools/build_failed.py @@ -0,0 +1,17 @@ +"""Record a source or build failure as a FAIL result, so the report shows it.""" +import json +import os +import sys + +out, surface = sys.argv[1], os.environ["SURFACE"] +log = "" +for name in ("fetch.log", "build.log"): + p = os.path.join(out, "build", name) + if os.path.exists(p): + with open(p, errors="replace") as f: + log += f.read()[-2500:] +d = os.path.join(out, "results", "checks") +os.makedirs(d, exist_ok=True) +with open(os.path.join(d, "000-build.json"), "w") as f: + json.dump({"surface": surface, "check": "build", "title": "Build the release artifact", "status": "FAIL", + "expected": "the artifact builds", "actual": "the source or builder step failed", "evidence": log}, f, indent=2) diff --git a/acceptance/crawl/tools/cleanup.py b/acceptance/crawl/tools/cleanup.py new file mode 100644 index 0000000..7228432 --- /dev/null +++ b/acceptance/crawl/tools/cleanup.py @@ -0,0 +1,64 @@ +"""Clean-up: stop every crawl this run started, then record each one's final state. + +Reads out/results/crawls.jsonl, the ledger the checks write. A crawl still running is +stopped; then each crawl is read for its final status and coverage.pages_fetched, which the +report uses for the cost table. Writes out/cleanup.json. +""" +import json +import os +import sys +import time +import urllib.error +import urllib.request + +OUT = sys.argv[1] +BASE = os.environ.get("CRAWL_API_BASE", "https://api.zenrows.com/v1").rstrip("/") +KEY = os.environ["ZENROWS_API_KEY"] + + +def call(method, path): + req = urllib.request.Request(BASE + path, method=method, headers={"X-API-Key": KEY}) + try: + with urllib.request.urlopen(req, timeout=60) as r: + return r.status, json.loads(r.read() or b"null") + except urllib.error.HTTPError as e: + try: + return e.code, json.loads(e.read() or b"null") + except ValueError: + return e.code, None + + +crawls = {} +ledger = os.path.join(OUT, "results", "crawls.jsonl") +if os.path.exists(ledger): + with open(ledger) as f: + for line in f: + if line.strip(): + row = json.loads(line) + entry = crawls.setdefault(row["crawl_id"], {"purposes": []}) + if row["purpose"] not in entry["purposes"]: + entry["purposes"].append(row["purpose"]) + +rows = [] +for crawl_id, entry in crawls.items(): + status, body = call("GET", f"/crawls/{crawl_id}?limit=1") + stopped = False + if status == 200 and body and body.get("status") == "running": + call("POST", f"/crawls/{crawl_id}/stop") + stopped = True + time.sleep(2) + status, body = call("GET", f"/crawls/{crawl_id}?limit=1") + row = {"crawl_id": crawl_id, **entry, "stopped_by_cleanup": stopped, "http": status} + if status == 200 and body: + row.update(status=body.get("status"), stop_reason=body.get("stop_reason"), + pages_fetched=body.get("coverage", {}).get("pages_fetched"), + output_format=body.get("output_format")) + rows.append(row) + print(f"{crawl_id} {row.get('status')} pages_fetched={row.get('pages_fetched')}" + f"{' (stopped by clean-up)' if stopped else ''}") + +still = [r["crawl_id"] for r in rows if r.get("status") == "running"] +with open(os.path.join(OUT, "cleanup.json"), "w") as f: + json.dump({"crawls": rows, "still_running": still, + "checked_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}, f, indent=2) +print(f"clean-up: {len(rows)} crawls, {len(still)} still running") diff --git a/acceptance/crawl/tools/report.py b/acceptance/crawl/tools/report.py new file mode 100644 index 0000000..fd085e1 --- /dev/null +++ b/acceptance/crawl/tools/report.py @@ -0,0 +1,190 @@ +"""Turn out/results/checks/*.json into out/crawl-first-install-report.md and out/results.json. + +Also computes check 17: the names this surface uses against the API's names. +""" +import glob +import json +import os +import re +import sys + +OUT = sys.argv[1] +SURFACE = os.environ["SURFACE"] +REPO = os.environ["REPO"] +NAMES = {"node": "Node SDK (`zenrows` on npm)", "python": "Python SDK (`zenrows` on PyPI)", + "go": "Go SDK (`github.com/zenrows/zenrows-go-sdk/service/crawl`)", "mcp": "MCP server (`@zenrows/mcp`)", + "cli": "CLI (`@zenrows/cli`)"} +CHECK_TITLES = { + "1": "Create", "2": "Create with html", "3": "Idempotency", "4": "Wait", "5": "Wait runs out", "6": "Get", + "7": "List", "8": "Results", "9": "Content", "10": "Download", "11": "Stop", "12": "Not found", + "13": "Invalid input", "14": "Gated option", "15": "Bad key", "16": "Errors carry the server code", + "17": "Naming", "18": "Copy", "19": "Failed crawl", "20": "Results after stop", "21": "Documented defaults", +} +# The API's operation and parameter names. Each surface uses them in its own casing. +CANON_OPS = ["create", "get", "list", "results", "content", "download", "stop", "wait"] +CANON_PARAMS = ["url", "depth", "max_items", "max_pages", "include_patterns", "exclude_patterns", "output_format", + "idempotency_key", "cursor", "limit", "timeout"] + + +def read(path, default=None): + try: + with open(path) as f: + return f.read().strip() + except OSError: + return default + + +def snake(name): + n = re.sub(r"^crawl[._ ]", "", name.strip()) + n = n.replace("()", "").replace("<", "").replace(">", "").lstrip("-").replace("-", "_") + return re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", n).lower() + + +def naming(rows): + p = os.path.join(OUT, "results", "naming.json") + if not os.path.exists(p): + return + with open(p) as f: + d = json.load(f) + diffs = [] + for op in CANON_OPS: + got = d["operations"].get(op) + if got is None: + diffs.append(f"operation `{op}` missing") + elif snake(got) != op: + diffs.append(f"operation `{op}` is `{got}`") + for prm in CANON_PARAMS: + got = d["params"].get(prm) + if got is None: + diffs.append(f"parameter `{prm}` missing") + elif snake(got) != prm: + diffs.append(f"parameter `{prm}` is `{got}`") + rows["17"] = {"surface": SURFACE, "check": "17", "title": "Naming", + "status": "FAIL" if diffs else "PASS", + "expected": "the API's operation and parameter names, in this surface's casing", + "actual": "; ".join(diffs) or "matches", + "evidence": json.dumps({k: d[k] for k in ("operations", "params")})} + + +def severity(check): + if check in ("17", "18", "21") or check.startswith("x-"): + return "minor" + if check.startswith("readme") or check == "build": + return "blocker" + return "major" + + +def first_line(s, n=110): + line = re.sub(r"\s+", " ", (s or "").split("\n")[0]) + return line if len(line) <= n else line[: n - 1] + "…" + + +def main(): + rows = {} + for p in sorted(glob.glob(os.path.join(OUT, "results", "checks", "*.json"))): + with open(p) as f: + r = json.load(f) + rows[r["check"]] = r + naming(rows) + env = json.loads(read(os.path.join(OUT, "results", "env.json"), "{}") or "{}") + cleanup = json.loads(read(os.path.join(OUT, "cleanup.json"), "null") or "null") + sha = read(os.path.join(OUT, "build", "sha"), "-") + origin = read(os.path.join(OUT, "build", "origin"), "-") + artifacts = [os.path.basename(p) for p in glob.glob(os.path.join(OUT, "build", "artifact", "*"))] + gomod = read(os.path.join(OUT, "results", "go-module.txt")) + + findings = [{"check": c, "severity": severity(c), "r": r} for c, r in rows.items() if r["status"] == "FAIL"] + order = {"blocker": 0, "major": 1, "minor": 2} + findings.sort(key=lambda f: order[f["severity"]]) + blocking = [f for f in findings if f["severity"] != "minor"] + counts = {} + for r in rows.values(): + counts[r["status"]] = counts.get(r["status"], 0) + 1 + + lines = [] + w = lines.append + w(f"# Crawl first-install test: {NAMES[SURFACE]}\n") + w(f"Run `{read(os.path.join(OUT, 'run_id'), '-')}`. The client was built as a release artifact, installed in a " + "clean container, set up from its README, then tested against the live Crawl API.\n") + + w("## 1. Verdict\n") + if blocking: + w("**Not ready.** " + "; ".join(f"{f['check']} {f['r']['title']}: {first_line(f['r']['actual'], 90)}" for f in blocking)) + else: + w("**Ready.**" + (f" {len(findings)} minor finding(s), see Findings." if findings else "")) + w("\n" + ", ".join(f"{v} {k}" for k, v in sorted(counts.items())) + ".\n") + + w("## 2. Environment\n") + w(f"- Command: `{os.environ.get('CMDLINE') or 'bash run.sh'}`") + w(f"- Repo: ZenRows/{REPO}; source: {origin}; SHA tested: `{sha}`") + w(f"- Artifact: {', '.join(artifacts) or ('go get ' + gomod.split()[0] if gomod else '-')}") + w(f"- `CRAWL_URL`: {os.environ.get('CRAWL_URL')}; `CRAWL_INCLUDE`: `{os.environ.get('CRAWL_INCLUDE')}`; " + f"API: {os.environ.get('CRAWL_API_BASE')}") + image = {"node": "NODE_IMAGE", "mcp": "NODE_IMAGE", "cli": "NODE_IMAGE", "python": "PYTHON_IMAGE", "go": "GO_IMAGE"}[SURFACE] + w(f"- Image: `{os.environ.get(image)}`; container: " + ", ".join(f"{k} {v}" for k, v in env.items())) + tools = {"node": f"typescript {os.environ.get('TYPESCRIPT_VERSION')}, pnpm {os.environ.get('PNPM_VERSION')}", + "mcp": f"@modelcontextprotocol/sdk {os.environ.get('MCP_SDK_VERSION')}", "cli": "none", + "python": f"mypy {os.environ.get('MYPY_VERSION')}, build {os.environ.get('PY_BUILD_VERSION')}", "go": "none"}[SURFACE] + w(f"- Pinned tools: {tools}\n") + + w("## 3. First-time install\n") + for check, r in rows.items(): + if check.startswith("readme") or check == "build": + w(f"- **{r['status']}** {r['title']}: {first_line(r['actual'], 200)}") + ev = r.get("evidence") or "" + if ev and (r["status"] != "PASS" or "substitutions" in ev): + w("\n ```\n " + "\n ".join(ev.splitlines()[:16]) + "\n ```") + w("") + + w("## 4. Feature matrix\n") + w("| # | Check | Result |") + w("| --- | --- | --- |") + for c in [str(i) for i in range(1, 22)]: + r = rows.get(c) + cell = f"**{r['status']}** {first_line(r['actual'])}".replace("|", "\\|") if r else "—" + w(f"| {c} | {CHECK_TITLES[c]} | {cell} |") + for c, r in rows.items(): + if c.startswith("x-"): + w(f"| extra | {r['title']} | " + f"**{r['status']}** {first_line(r['actual'])}".replace("|", "\\|") + " |") + w("") + + w("## 5. Findings\n") + if not findings: + w("None.\n") + for i, f in enumerate(findings, 1): + r = f["r"] + w(f"### {i}. {r['title']} ({f['severity']})\n") + w(f"- What happened: {r['actual']}") + w(f"- Expected: {r['expected']}") + w("- Evidence:\n\n ```\n " + "\n ".join((r.get("evidence") or "").splitlines()[:20]) + "\n ```\n") + + w("## 6. Cost\n") + if cleanup and cleanup["crawls"]: + w("| Crawl | Purpose | Final status | pages_fetched |") + w("| --- | --- | --- | --- |") + total = 0 + for c in cleanup["crawls"]: + total += c.get("pages_fetched") or 0 + w(f"| `{c['crawl_id']}` | {', '.join(c['purposes'])}{' (html)' if c.get('output_format') == 'html' else ''} | " + f"{c.get('status')}{'/' + c['stop_reason'] if c.get('stop_reason') else ''} | {c.get('pages_fetched')} |") + w(f"\nTotal pages fetched: **{total}**.\n") + else: + w("No crawls.\n") + + w("## 7. Clean-up\n") + if cleanup and cleanup["crawls"]: + stopped = [c["crawl_id"] for c in cleanup["crawls"] if c.get("stopped_by_cleanup")] + w(f"Checked at {cleanup['checked_at']}: {len(cleanup['crawls'])} crawls started by this run, " + f"{', '.join(cleanup['still_running']) or 'none'} still running. " + + (f"Clean-up stopped {', '.join(stopped)}." if stopped else "Clean-up had nothing to stop.") + "\n") + else: + w("No crawls to clean up.\n") + + with open(os.path.join(OUT, "crawl-first-install-report.md"), "w") as f: + f.write("\n".join(lines)) + with open(os.path.join(OUT, "results.json"), "w") as f: + json.dump(list(rows.values()), f, indent=2) + print(f"{SURFACE}: " + ", ".join(f"{k} {v}" for k, v in sorted(counts.items())) + f"; findings: {len(findings)}") + + +main() diff --git a/acceptance/crawl/tools/scan_key.py b/acceptance/crawl/tools/scan_key.py new file mode 100644 index 0000000..f6f0419 --- /dev/null +++ b/acceptance/crawl/tools/scan_key.py @@ -0,0 +1,21 @@ +"""Fail the run if the API key appears in any file under out/. Reads the key from the env.""" +import os +import sys + +key = os.environ.get("ZENROWS_API_KEY", "").encode() +hits = [] +for root, _, files in os.walk(sys.argv[1]): + if os.sep + "build" in root: + continue + for f in files: + p = os.path.join(root, f) + try: + with open(p, "rb") as fh: + if key and key in fh.read(): + hits.append(p) + except OSError: + pass +if hits: + print("THE API KEY APPEARS IN:", *hits, sep="\n ") + sys.exit(1) +print("key scan: the key appears in no result, log or report file") diff --git a/acceptance/crawl/versions.env b/acceptance/crawl/versions.env new file mode 100644 index 0000000..0ea853d --- /dev/null +++ b/acceptance/crawl/versions.env @@ -0,0 +1,23 @@ +# Pinned inputs for the Crawl acceptance test. Every value can be overridden from the environment. +SURFACE=cli +REPO=cli + +# The official images, pinned by digest so two runs agree. +NODE_IMAGE="${NODE_IMAGE:-node:20@sha256:8f693eaa7e0a8e71560c9a82b55fd54c2ae920a2ba5d2cde28bac7d1c01c9ba5}" +PYTHON_IMAGE="${PYTHON_IMAGE:-python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1}" +GO_IMAGE="${GO_IMAGE:-golang:1.25@sha256:699337d620559a59b4a2bb298ad59611e535d2ee755a34cf2d2a98f37578dc80}" + +# Tools the build and the checks install. +PNPM_VERSION="${PNPM_VERSION:-9.4.0}" +PY_BUILD_VERSION="${PY_BUILD_VERSION:-1.2.2.post1}" +MCP_SDK_VERSION="${MCP_SDK_VERSION:-1.27.1}" +TYPESCRIPT_VERSION="${TYPESCRIPT_VERSION:-5.7.3}" +MYPY_VERSION="${MYPY_VERSION:-1.15.0}" + +# What to crawl. +CRAWL_URL="${CRAWL_URL:-https://www.scrapingcourse.com/ecommerce/}" +CRAWL_INCLUDE="${CRAWL_INCLUDE:-/product/}" +CRAWL_API_BASE="${CRAWL_API_BASE:-https://api.zenrows.com/v1}" + +# REF unset tests this checkout. Set it to a SHA, branch or tag to test that commit from GitHub. +REF="${REF:-}" From 86d3e57e8b7b55126633f0f178ccbc8748e6acd0 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 14:27:23 +0200 Subject: [PATCH 12/19] feat(crawl)!: name the credits error CRAWL_INSUFFICIENT_CREDITS A 402 for lack of credit is CRAWL_INSUFFICIENT_CREDITS, after the Crawl API's insufficient_credits, instead of CRAWL_QUOTA_EXCEEDED. The MCP server uses the same name. CRAWL_KEY_CAP_REACHED is unchanged and already covers the API's api_key_cap_reached. --- README.md | 2 +- skills/crawl/SKILL.md | 4 ++-- skills/trace-debug/SKILL.md | 2 +- src/core/crawl-api.ts | 2 +- src/core/errors.ts | 2 +- tests/crawl-api.test.ts | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e0221c3..64c7159 100644 --- a/README.md +++ b/README.md @@ -210,7 +210,7 @@ Errors carry the API's code as `server_code`, `crawl_id` on any call about one crawl, and `retry_after` on a 429. When the account has reached its limit of active jobs (3 by default), shared with its Batch jobs, a create gets `CRAWL_TOO_MANY_CRAWLS` and creates nothing; retry after `retry_after` seconds. -Other codes: `CRAWL_QUOTA_EXCEEDED` and `CRAWL_KEY_CAP_REACHED` (402, with the +Other codes: `CRAWL_INSUFFICIENT_CREDITS` and `CRAWL_KEY_CAP_REACHED` (402, with the same claim, top-up and key-cap advice as other commands), `CRAWL_INVALID_REQUEST` (400/422, do not retry as is), `CRAWL_REQUEST_IN_FLIGHT` (409, retry once the first request ends), `CRAWL_NOT_FOUND`, diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index e573ab9..caea8f4 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -104,8 +104,8 @@ The "Retry?" column is for errors that remain after these retries. | --- | --- | --- | | `CRAWL_NOT_ENABLED` | 403 `REQS008`: Crawl is not enabled for this account | no | | `AUTH_INVALID` | 401: the API key was rejected | no, log in again | -| `CRAWL_QUOTA_EXCEEDED` | 402: the account is out of credits (claim the account or add credits) | after credits renew or are added | -| `CRAWL_KEY_CAP_REACHED` | 402 `AUTH014`: this API key reached its credit cap | after the cap resets or is raised | +| `CRAWL_INSUFFICIENT_CREDITS` | 402: the account is out of credits (claim the account or add credits) | after credits renew or are added | +| `CRAWL_KEY_CAP_REACHED` | 402 `AUTH014` or `api_key_cap_reached`: this API key reached its credit cap | after the cap resets or is raised | | `CRAWL_INVALID_REQUEST` | 400 or 422: fix the request. For `idempotency_key_reused`, send a new key or no key | no, not as is | | `CRAWL_REQUEST_IN_FLIGHT` | 409: the same Idempotency-Key is still in flight | yes, once the first request ends | | `CRAWL_NOT_FOUND` / `CRAWL_CONTENT_NOT_FOUND` | 404: wrong id, or the page has no content | no | diff --git a/skills/trace-debug/SKILL.md b/skills/trace-debug/SKILL.md index 65c0da2..0948509 100644 --- a/skills/trace-debug/SKILL.md +++ b/skills/trace-debug/SKILL.md @@ -47,7 +47,7 @@ zenrows trace export # JSON for sharing `failure_detail` in `zenrows batch status --json` say why. - `CRAWL_NOT_ENABLED` → the API answered 403 `REQS008`: Crawl is not enabled for this account. Do not retry; ask Zenrows support for access. -- `CRAWL_QUOTA_EXCEEDED` / `CRAWL_KEY_CAP_REACHED` → Crawl's 402: the +- `CRAWL_INSUFFICIENT_CREDITS` / `CRAWL_KEY_CAP_REACHED` → Crawl's 402: the account is out of credits, or this key reached its credit cap (`AUTH014`). Same advice as `POLICY_MAX_CREDITS_EXCEEDED` / `KEY_CREDIT_CAP_REACHED`. - `CRAWL_TOO_MANY_CRAWLS` → 429 `too_many_crawls`: the account has reached diff --git a/src/core/crawl-api.ts b/src/core/crawl-api.ts index 923a98f..2a631e9 100644 --- a/src/core/crawl-api.ts +++ b/src/core/crawl-api.ts @@ -372,7 +372,7 @@ function classifyProblem( } const acct = readAccount(); const quota = quotaExhausted(request, acct?.unclaimed ? acct.claimUrl : undefined, { status, detail }); - return new ToolkitError({ ...quota.toJSON(), code: "CRAWL_QUOTA_EXCEEDED" }); + return new ToolkitError({ ...quota.toJSON(), code: "CRAWL_INSUFFICIENT_CREDITS" }); } if (status === 409) { return new ToolkitError({ diff --git a/src/core/errors.ts b/src/core/errors.ts index 5b889c3..2a3c093 100644 --- a/src/core/errors.ts +++ b/src/core/errors.ts @@ -36,7 +36,7 @@ export type ErrorCode = | "BATCH_NOT_FOUND" | "BATCH_FAILED" | "CRAWL_NOT_ENABLED" - | "CRAWL_QUOTA_EXCEEDED" + | "CRAWL_INSUFFICIENT_CREDITS" | "CRAWL_KEY_CAP_REACHED" | "CRAWL_TOO_MANY_CRAWLS" | "CRAWL_NOT_FOUND" diff --git a/tests/crawl-api.test.ts b/tests/crawl-api.test.ts index 5cc733c..9de1f39 100644 --- a/tests/crawl-api.test.ts +++ b/tests/crawl-api.test.ts @@ -259,12 +259,12 @@ test("429 too_many_crawls maps to CRAWL_TOO_MANY_CRAWLS with retry_after, and cr assert.equal(calls.length, 1); }); -test("402 maps to CRAWL_QUOTA_EXCEEDED, and AUTH014 to CRAWL_KEY_CAP_REACHED, with the shared advice", async () => { +test("402 maps to CRAWL_INSUFFICIENT_CREDITS, and AUTH014 to CRAWL_KEY_CAP_REACHED, with the shared advice", async () => { const quota = problem(402, "AUTH004", "Usage limit reached."); await assert.rejects( () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: quota.impl }), (e: unknown) => - e instanceof ToolkitError && e.code === "CRAWL_QUOTA_EXCEEDED" && e.server_code === "AUTH004" && /zenrows usage/.test(e.next_action), + e instanceof ToolkitError && e.code === "CRAWL_INSUFFICIENT_CREDITS" && e.server_code === "AUTH004" && /zenrows usage/.test(e.next_action), ); const cap = problem(402, "AUTH014", "Key cap reached."); await assert.rejects( From 76420da26675a2d9195b5bd123e56a1c8444069e Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 14:29:43 +0200 Subject: [PATCH 13/19] test(crawl): check content_id in the naming check The naming check now covers content_id, the argument of the content operation, so a client that names it differently fails check 17. The MCP surface calls crawl_content with content_id. --- acceptance/crawl/surface/test.mjs | 1 + acceptance/crawl/tools/report.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/acceptance/crawl/surface/test.mjs b/acceptance/crawl/surface/test.mjs index 33cb3f5..80b7352 100644 --- a/acceptance/crawl/surface/test.mjs +++ b/acceptance/crawl/surface/test.mjs @@ -855,6 +855,7 @@ H.naming({ cursor: "--cursor", limit: "--limit", timeout: "--timeout", + content_id: "", }, }); record("17", "Naming", SKIP, "the API's names in this surface's casing", "computed by the report"); diff --git a/acceptance/crawl/tools/report.py b/acceptance/crawl/tools/report.py index fd085e1..94488f8 100644 --- a/acceptance/crawl/tools/report.py +++ b/acceptance/crawl/tools/report.py @@ -23,7 +23,7 @@ # The API's operation and parameter names. Each surface uses them in its own casing. CANON_OPS = ["create", "get", "list", "results", "content", "download", "stop", "wait"] CANON_PARAMS = ["url", "depth", "max_items", "max_pages", "include_patterns", "exclude_patterns", "output_format", - "idempotency_key", "cursor", "limit", "timeout"] + "idempotency_key", "cursor", "limit", "timeout", "content_id"] def read(path, default=None): From 8cbed3df3b08a66779948a1f194880f91fdaf27c Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 15:14:50 +0200 Subject: [PATCH 14/19] test(crawl): give check 9 a finished crawl to read, and count a refusal before the API as a pass on check 14 Check 9 read content from the crawl check 20 stops; a stop that lands before any page is fetched left nothing to read. It now falls back to a one-item html crawl that runs to the end. Check 14 passes when the client refuses the gated output format before calling the API, as well as on the API's 422. --- acceptance/crawl/surface/test.mjs | 40 ++++++++++++++++++------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/acceptance/crawl/surface/test.mjs b/acceptance/crawl/surface/test.mjs index 80b7352..c4e5fa4 100644 --- a/acceptance/crawl/surface/test.mjs +++ b/acceptance/crawl/surface/test.mjs @@ -621,21 +621,25 @@ await liveCheck( "crawl content by content id, then by content_url: the same HTML", async () => { if (!B) return { status: SKIP, actual: "no html crawl" }; - const row = (z(["crawl", "results", B.crawl_id, "--json"]).json?.results ?? []).find( - (x) => x.content_url, - ); - if (!row) - return stopInfo?.status === "stopped" - ? { - status: SKIP, - actual: - "the stop cancelled every page fetch, so no page has content to read (see check 20)", - } - : { status: FAIL, actual: "no result with content_url" }; + const firstFetched = (id) => + (z(["crawl", "results", id, "--json"]).json?.results ?? []).find((x) => x.content_url); + // The stop in check 20 can land before any page is fetched. Then read a + // one-item html crawl that runs to the end instead. + let src = B.crawl_id; + let row = firstFetched(src); + if (!row) { + const own = createWith([CRAWL_URL, "--depth", "1", "--max-items", "1", "--max-pages", "10", + "--include-patterns", INCLUDE, "--output-format", "html"]); + if (!own.json?.crawl_id) return { status: FAIL, actual: `create: exit ${own.code}`, evidence: show(own) }; + z(["crawl", "wait", own.json.crawl_id, "--json"]); + src = own.json.crawl_id; + row = firstFetched(src); + } + if (!row) return { status: FAIL, actual: `no result with content_url on ${src}` }; const id = row.content_url.split("/").pop(); - const a = z(["crawl", "content", B.crawl_id, id, "--out", "/work/a.html"]); - const b = z(["crawl", "content", B.crawl_id, row.content_url, "--out", "/work/b.html"]); - noteShape("content", z(["crawl", "content", B.crawl_id, id, "--json"])); + const a = z(["crawl", "content", src, id, "--out", "/work/a.html"]); + const b = z(["crawl", "content", src, row.content_url, "--out", "/work/b.html"]); + noteShape("content", z(["crawl", "content", src, id, "--json"])); const ha = existsSync("/work/a.html") ? readFileSync("/work/a.html", "utf8") : ""; const hb = existsSync("/work/b.html") ? readFileSync("/work/b.html", "utf8") : ""; const ok = a.code === 0 && b.code === 0 && ha === hb && / { +await liveCheck("14", "Gated option", "--output-format json is refused: INVALID_USAGE before the API, or 422 invalid_parameter", async () => { const r = z(["crawl", "create", CRAWL_URL, "--depth", "1", "--output-format", "json", "--json"]); if (r.json?.crawl_id) { ledger(r.json.crawl_id, "output_format json (accepted)"); @@ -721,9 +725,11 @@ await liveCheck("14", "Gated option", "--output-format json → 422 invalid_para actual: `accepted (${r.json.crawl_id}): the key is in a beta segment, not a bug; stopped it`, }; } + // Refusing a gated value before it reaches the API is the behavior we want. + const code = r.json?.error?.code; return { - status: SKIP, - actual: `the client does not allow it: ${r.json?.error?.code}: ${r.json?.error?.message}`, + status: passIf(r.code !== 0 && (code === "INVALID_USAGE" || code === "CRAWL_INVALID_REQUEST")), + actual: `exit ${r.code} ${code}: ${r.json?.error?.message}`, }; }); await check("15", "Bad key", "AUTH_INVALID", async () => { From 90fd3e2e25eb679d239bdf14c2945e948fa9c114 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 16:22:36 +0200 Subject: [PATCH 15/19] docs(acceptance): correct the download row and the crawl count --- acceptance/crawl/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/acceptance/crawl/README.md b/acceptance/crawl/README.md index 86de5a0..f6589e7 100644 --- a/acceptance/crawl/README.md +++ b/acceptance/crawl/README.md @@ -69,7 +69,7 @@ and signal checks. | 7 | List | `limit` 2, then the next page: newest first; `next_cursor` absent or null on the last page | | 8 | Results | every kept URL once, each matching `CRAWL_INCLUDE`; on a running crawl, what is kept so far (MCP, CLI: `partial: true`) | | 9 | Content | by content id and by `content_url`: the same HTML | -| 10 | Download | a status, and one line per result with `url`, `content_status`, `content` (MCP has no download) | +| 10 | Download | a status, and one line per result with `url`, `content_status`, `content` | | 11 | Stop | running → `stopped`, `stop_reason: user`; ended → answers as it ended | | 12 | Not found | 404 `crawl_not_found` (MCP, CLI: `CRAWL_NOT_FOUND` with `crawl_id` and `server_code`) | | 13 | Invalid input | `depth` 0 → 422 `invalid_parameter` (MCP, CLI: `CRAWL_INVALID_REQUEST`) | @@ -90,7 +90,8 @@ Every crawl uses `depth` 1 and `max_pages` 10 or less. A run starts five crawls: one URLs-only crawl, one idempotency crawl that is stopped at once, one html crawl that is stopped with its pages in flight (check 20), and one that keeps nothing (check 19). On the default `CRAWL_URL`, a URLs-only crawl fetches 1 page and the html crawl 6, so a run costs about **10 pages**, and at most 50 if every crawl reaches its -`max_pages`. The report lists each crawl id and its `pages_fetched`. +`max_pages`. Check 9 starts a sixth crawl, one html item, when the stop in check 20 lands +before any page is fetched. It adds at most 10 pages. The report lists each crawl id and its `pages_fetched`. An account can run 3 active jobs at once, shared with Batch jobs. A run has at most 2 crawls running at a time. If a create still gets a 429, the run waits `Retry-After` and tries again. From 14068a5d0e7c94c1b2d65eabc186f52092508293 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 16:40:35 +0200 Subject: [PATCH 16/19] fix(crawl): stream the download export and reject a malformed content id The download held the whole NDJSON export, page HTML included, in memory before it wrote the file. It now streams into .part, counts lines as they pass, and renames the file into place, so a failed export keeps any earlier file. A content id with a bad % escape threw a raw URIError; it is now INVALID_USAGE. --- src/cli/commands/crawl.ts | 25 ++++++++++++---- src/cli/output.ts | 52 ++++++++++++++++++++++++--------- src/core/crawl-api.ts | 44 +++++++++++++++++++++++++--- tests/crawl-api.test.ts | 57 ++++++++++++++++++++++++++++++++----- tests/crawl-command.test.ts | 47 ++++++++++++++++++++++++++++-- 5 files changed, 191 insertions(+), 34 deletions(-) diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index d01e366..1900f2f 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -31,7 +31,7 @@ import { } from "../../core/crawl-api.ts"; import { asNumber, asString, parse, type Command, type RunContext } from "../command.ts"; import { ToolkitError } from "../../core/errors.ts"; -import { printError, writeOut } from "../output.ts"; +import { printError, writeOut, writeOutStream } from "../output.ts"; /** The API's default for `max_pages` (and `max_items`) when the flag is unset. */ const DEFAULT_MAX_PAGES = 10; @@ -295,9 +295,7 @@ async function downloadCmd(rest: string[], ctx: RunContext): Promise { const file = asString(values.out) ?? `${id.replace(/[^A-Za-z0-9._-]+/g, "_")}.jsonl`; assertUsable("crawl"); const apiKey = requireApiKey(); - const { status, ndjson } = await downloadCrawl(id, { apiKey }); - writeOut(file, ndjson); - const count = ndjson.split("\n").filter((l) => l.trim()).length; + const { status, count } = await downloadCrawl(id, (ndjson) => writeOutStream(file, ndjson), { apiKey }); const partial = status === "running"; if (json) { log.out(JSON.stringify({ ok: true, crawl_id: id, status: status ?? null, partial, count, file }, null, 2)); @@ -336,8 +334,7 @@ async function contentCmd(rest: string[], ctx: RunContext): Promise { const json = ctx.json || values.json === true; const id = requireId(positionals[0], "content"); const ref = positionals[1]; - // Accept the content id or the result's whole content_url. - const contentId = ref ? (contentIdOf({ url: "", content_url: ref }) ?? ref) : undefined; + const contentId = ref ? contentIdOfRef(id, ref) : undefined; if (!contentId) { throw new ToolkitError({ code: "INVALID_USAGE", @@ -367,6 +364,22 @@ async function contentCmd(rest: string[], ctx: RunContext): Promise { return 0; } +/** The content id or the result's whole content_url, as the content id. */ +function contentIdOfRef(id: string, ref: string): string { + try { + return contentIdOf({ url: "", content_url: ref }) ?? ref; + } catch (err) { + if (!(err instanceof URIError)) throw err; + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `Invalid content id '${ref}'.`, + likely_cause: "It has a % that does not start a valid percent-encoded character.", + next_action: "Pass the id or content_url exactly as a fetched result shows it.", + suggested_commands: [`zenrows crawl results ${id}`], + }); + } +} + async function listCmd(rest: string[], ctx: RunContext): Promise { const { values } = parse(rest, { limit: { type: "string" }, cursor: { type: "string" }, json: { type: "boolean" } }); const json = ctx.json || values.json === true; diff --git a/src/cli/output.ts b/src/cli/output.ts index 224b7f8..589b561 100644 --- a/src/cli/output.ts +++ b/src/cli/output.ts @@ -2,7 +2,8 @@ * Output helpers: render normalized errors for humans and machines, and a tiny * key/value printer used by `status`, `config`, `policy`. */ -import { statSync, writeFileSync } from "node:fs"; +import { createWriteStream, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { pipeline } from "node:stream/promises"; import { ToolkitError, type ToolkitErrorShape } from "../core/errors.ts"; import { ANSI, c, log } from "../core/logger.ts"; @@ -12,6 +13,32 @@ import { ANSI, c, log } from "../core/logger.ts"; * parent directory doesn't exist (ENOENT). Everything else propagates. */ export function writeOut(filePath: string, data: string | Buffer): void { + assertNotDirectory(filePath); + try { + writeFileSync(filePath, data); + } catch (err) { + throw outError(filePath, err); + } +} + +/** + * `writeOut` for a body too large to hold in memory: streams into + * `.part`, then renames it into place, so a failed stream leaves any + * earlier file intact. + */ +export async function writeOutStream(filePath: string, chunks: AsyncIterable): Promise { + assertNotDirectory(filePath); + const part = `${filePath}.part`; + try { + await pipeline(chunks, createWriteStream(part)); + renameSync(part, filePath); + } catch (err) { + rmSync(part, { force: true }); + throw outError(filePath, err); + } +} + +function assertNotDirectory(filePath: string): void { if (statSync(filePath, { throwIfNoEntry: false })?.isDirectory()) { throw new ToolkitError({ code: "INVALID_USAGE", @@ -20,19 +47,16 @@ export function writeOut(filePath: string, data: string | Buffer): void { next_action: "Pass a file path, e.g. --out results.jsonl.", }); } - try { - writeFileSync(filePath, data); - } catch (err) { - if ((err as NodeJS.ErrnoException)?.code === "ENOENT") { - throw new ToolkitError({ - code: "INVALID_USAGE", - message: `Cannot write --out '${filePath}': the parent directory does not exist.`, - likely_cause: "A directory in the output path is missing.", - next_action: "Create the directory first, or choose an existing one.", - }); - } - throw err; - } +} + +function outError(filePath: string, err: unknown): unknown { + if ((err as NodeJS.ErrnoException)?.code !== "ENOENT") return err; + return new ToolkitError({ + code: "INVALID_USAGE", + message: `Cannot write --out '${filePath}': the parent directory does not exist.`, + likely_cause: "A directory in the output path is missing.", + next_action: "Create the directory first, or choose an existing one.", + }); } /** Print a ToolkitError. In --json mode emits the structured shape to stdout. */ diff --git a/src/core/crawl-api.ts b/src/core/crawl-api.ts index 2a631e9..063a4f1 100644 --- a/src/core/crawl-api.ts +++ b/src/core/crawl-api.ts @@ -13,6 +13,7 @@ * `output_format` takes `html` only (absent = URLs only). Response fields we do * not model pass through untouched. */ +import { Readable } from "node:stream"; import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "./errors.ts"; import { readAccount } from "./agent-account.ts"; import { registerSecret } from "./logger.ts"; @@ -148,6 +149,8 @@ interface RequestOpts extends CallOpts { crawlId?: string; /** Retry a 503 crawl_busy (stop only): the stop was not saved and the crawl still runs. */ retryBusy?: boolean; + /** Consume a 2xx body as a stream instead of reading it as text. Called again on each retry. */ + sink?: (body: AsyncIterable) => Promise; } interface RawResponse { @@ -199,6 +202,10 @@ async function attempt(url: string, init: RequestInit, opts: RequestOpts): Promi const timeout = setTimeout(() => controller.abort(), opts.timeoutMs ?? 60_000); try { const res = await (opts.fetchImpl ?? fetch)(url, { ...init, signal: controller.signal }); + if (opts.sink && res.ok) { + await opts.sink(res.body ?? Readable.from([])); + return { status: res.status, headers: res.headers, text: "" }; + } return { status: res.status, headers: res.headers, text: await res.text() }; } finally { clearTimeout(timeout); @@ -235,6 +242,7 @@ async function crawlFetch(method: string, path: string, opts: RequestOpts): Prom try { res = await attempt(url.toString(), init, opts); } catch (err) { + if (err instanceof ToolkitError) throw err; if (canRetry) { await sleep(backoffMs(n)); continue; @@ -460,17 +468,45 @@ export async function getCrawlContent( } /** - * Every result in one NDJSON file (`{url, content_status?, content?}` per line). - * `status` is the `X-Crawl-Status` header: `running` means the file is partial. + * Every result in one NDJSON file (`{url, content_status?, content?}` per line), + * streamed into `write` because page HTML makes it large. `count` is its + * non-blank lines. `status` is the `X-Crawl-Status` header: `running` means the + * file is partial. */ -export async function downloadCrawl(id: string, opts: CallOpts): Promise<{ status?: string; ndjson: string }> { +export async function downloadCrawl( + id: string, + write: (ndjson: AsyncIterable) => Promise, + opts: CallOpts, +): Promise<{ status?: string; count: number }> { + let count = 0; const res = await crawlFetch("GET", `/crawls/${encodeURIComponent(id)}/download`, { ...opts, crawlId: id, accept: "application/x-ndjson, application/problem+json;q=0.8", timeoutMs: opts.timeoutMs ?? 300_000, + sink: (body) => { + count = 0; + return write(countLines(body, () => count++)); + }, }); - return { status: res.headers.get("x-crawl-status") ?? undefined, ndjson: res.text }; + return { status: res.headers.get("x-crawl-status") ?? undefined, count }; +} + +/** Pass `chunks` through, calling `onLine` for each line that is not blank. */ +async function* countLines(chunks: AsyncIterable, onLine: () => void): AsyncGenerator { + let blank = true; + for await (const chunk of chunks) { + for (const b of chunk) { + if (b === 0x0a) { + if (!blank) onLine(); + blank = true; + } else if (b !== 0x20 && b !== 0x09 && b !== 0x0d) { + blank = false; + } + } + yield chunk; + } + if (!blank) onLine(); } /** diff --git a/tests/crawl-api.test.ts b/tests/crawl-api.test.ts index 9de1f39..94a5e4e 100644 --- a/tests/crawl-api.test.ts +++ b/tests/crawl-api.test.ts @@ -159,15 +159,58 @@ test("getCrawlContent returns the HTML body and its type", async () => { assert.equal(out.body, "hi"); }); -test("downloadCrawl returns the NDJSON and X-Crawl-Status", async () => { - const { impl, calls } = stubFetch(200, '{"url":"https://example.com/product/a"}\n{"url":"https://example.com/product/b"}\n', { - "content-type": "application/x-ndjson", - "x-crawl-status": "running", - }); - const out = await downloadCrawl("c_1", { apiKey: "k", fetchImpl: impl }); +/** A `write` for downloadCrawl that keeps the streamed text of the last attempt. */ +function collect() { + const sink = { text: "" }; + const write = async (chunks: AsyncIterable) => { + sink.text = ""; + for await (const c of chunks) sink.text += Buffer.from(c).toString(); + }; + return { sink, write }; +} + +test("downloadCrawl streams the NDJSON, counts its lines and returns X-Crawl-Status", async () => { + const ndjson = '{"url":"https://example.com/product/a"}\n\n{"url":"https://example.com/product/b"}'; + const { impl, calls } = stubFetch(200, ndjson, { "content-type": "application/x-ndjson", "x-crawl-status": "running" }); + const { sink, write } = collect(); + const out = await downloadCrawl("c_1", write, { apiKey: "k", fetchImpl: impl }); assert.match(calls[0]!.url, /\/crawls\/c_1\/download$/); assert.equal(out.status, "running"); - assert.equal(out.ndjson.trim().split("\n").length, 2); + assert.equal(out.count, 2); + assert.equal(sink.text, ndjson); +}); + +test("downloadCrawl restarts the write and the count when the body fails mid-stream", async () => { + let n = 0; + const impl = (async () => { + n++; + const body = new ReadableStream({ + start(c) { + c.enqueue(new TextEncoder().encode('{"url":"a"}\n')); + if (n === 1) c.error(new TypeError("terminated")); + else c.close(); + }, + }); + return new Response(body, { status: 200 }); + }) as unknown as typeof fetch; + const { sink, write } = collect(); + const { sleepImpl } = recordSleep(); + const out = await downloadCrawl("c_1", write, { apiKey: "k", fetchImpl: impl, sleepImpl }); + assert.equal(n, 2); + assert.equal(out.count, 1); + assert.equal(sink.text, '{"url":"a"}\n'); +}); + +test("downloadCrawl does not retry an error the write raises as a ToolkitError", async () => { + const { impl, calls } = stubFetch(200, '{"url":"a"}\n'); + const write = async () => { + throw new ToolkitError({ code: "INVALID_USAGE", message: "m", likely_cause: "c", next_action: "n" }); + }; + await assert.rejects( + () => downloadCrawl("c_1", write, { apiKey: "k", fetchImpl: impl, sleepImpl: recordSleep().sleepImpl }), + (e: unknown) => e instanceof ToolkitError && e.code === "INVALID_USAGE", + ); + assert.equal(calls.length, 1); }); test("403 REQS008 maps to CRAWL_NOT_ENABLED with the API's wording", async () => { diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index 178ba1a..65bb820 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -1,6 +1,6 @@ import { mock, test } from "node:test"; import assert from "node:assert/strict"; -import { existsSync, readFileSync } from "node:fs"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { crawl } from "../src/cli/commands/crawl.ts"; import { createWorkspace } from "../src/core/workspace.ts"; @@ -60,8 +60,10 @@ function json(body: unknown, status = 200, headers: Record = {}) async function captureOut(fn: () => unknown): Promise { const orig = process.stdout.write.bind(process.stdout); let buf = ""; - process.stdout.write = ((s: string | Uint8Array) => { - buf += typeof s === "string" ? s : Buffer.from(s).toString(); + // log.out writes strings. Bytes are the test runner's own reports, which an awaited write lets through. + process.stdout.write = ((s: string | Uint8Array, ...rest: unknown[]) => { + if (typeof s !== "string") return (orig as (...a: unknown[]) => boolean)(s, ...rest); + buf += s; return true; }) as typeof process.stdout.write; try { @@ -355,6 +357,33 @@ test("crawl download writes the NDJSON export to .jsonl", async () => { assert.equal(out.crawl_id, "c_1"); assert.equal(calls[0]!.url.pathname, "/v1/crawls/c_1/download"); assert.ok(existsSync(join(process.cwd(), "c_1.jsonl"))); + assert.ok(!existsSync(join(process.cwd(), "c_1.jsonl.part"))); + }, + ); +}); + +test("crawl download keeps an earlier file when the export fails", async () => { + await withCrawlWorkspace( + () => json({ code: "crawl_not_found", title: "Not found", status: 404 }, 404), + async () => { + writeFileSync("c_1.jsonl", "old\n"); + const { code, out } = await run(["download", "c_1"]); + assert.notEqual(code, 0); + assert.equal(out.ok, false); + assert.equal(readFileSync("c_1.jsonl", "utf8"), "old\n"); + assert.ok(!existsSync("c_1.jsonl.part")); + }, + ); +}); + +test("crawl download --out into a missing directory is INVALID_USAGE", async () => { + await withCrawlWorkspace( + () => new Response('{"url":"https://example.com/product/1"}\n', { status: 200 }), + async (calls) => { + const { code, out } = await run(["download", "c_1", "--out", "missing/c_1.jsonl"]); + assert.equal(code, 1); + assert.equal(out.error.code, "INVALID_USAGE"); + assert.equal(calls.length, 1); }, ); }); @@ -386,6 +415,18 @@ test("crawl content accepts a content_url and prints the page", async () => { ); }); +test("crawl content with a malformed % escape is INVALID_USAGE, before any network call", async () => { + await withCrawlWorkspace( + () => new Response("ok", { status: 200 }), + async (calls) => { + const { code, out } = await run(["content", "c_1", "/v1/crawls/c_1/contents/ct_%zz"]); + assert.equal(code, 1); + assert.equal(out.error.code, "INVALID_USAGE"); + assert.equal(calls.length, 0); + }, + ); +}); + test("crawl list and stop call the right endpoints", async () => { await withCrawlWorkspace( (url, init) => From 69803d2d2449ec5f9893c4812b5c2f57baf3d914 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 16:40:35 +0200 Subject: [PATCH 17/19] fix(acceptance): make the crawl harness portable and its clean-up resilient realpath --relative-to is GNU-only, so on macOS the exclude pathspec was empty and every run was tagged +local-changes. The python and go branches had no builder in this repo. Clean-up now records a crawl it cannot reach and goes on, and retries a stop answered 503 crawl_busy. The key scan skipped any path with /build in it; it now skips only out/build. --- acceptance/crawl/run.sh | 31 ++++++-------------- acceptance/crawl/tools/cleanup.py | 46 ++++++++++++++++++++++-------- acceptance/crawl/tools/scan_key.py | 7 +++-- 3 files changed, 46 insertions(+), 38 deletions(-) diff --git a/acceptance/crawl/run.sh b/acceptance/crawl/run.sh index 03d63bb..d4b2218 100644 --- a/acceptance/crawl/run.sh +++ b/acceptance/crawl/run.sh @@ -51,7 +51,8 @@ prepare_source() { # Tracked and untracked-but-not-ignored files, as a release build would see them. (cd "$root" && git ls-files -co --exclude-standard -z | tar --null -T - -cf -) | tar -C "$OUT_DIR/build/source" -xf - local sha; sha="$(git -C "$root" rev-parse HEAD)" - if [ -n "$(git -C "$root" status --porcelain -- . ":(exclude)$(realpath --relative-to="$root" "$HERE")")" ]; then + local harness; harness="$(git -C "$HERE" rev-parse --show-prefix)" + if [ -n "$(git -C "$root" status --porcelain -- . ":(exclude)${harness%/}")" ]; then sha="$sha+local-changes" fi echo "$sha" >"$OUT_DIR/build/sha" @@ -64,41 +65,25 @@ prepare_source() { # ---- 2. Builder: the release artifact, in its own container ---- build_artifact() { mkdir -p "$OUT_DIR/build/artifact" - case "$SURFACE" in - node|mcp|cli) - log "build: npm pack" - docker_run -v "$OUT_DIR/build:/out" -v "$HERE/builders:/builders:ro" \ - -e SURFACE -e PNPM_VERSION "$NODE_IMAGE" bash /builders/build-npm.sh >"$OUT_DIR/build/build.log" 2>&1 ;; - python) - log "build: the wheel" - docker_run -v "$OUT_DIR/build:/out" -v "$HERE/builders:/builders:ro" \ - -e PY_BUILD_VERSION "$PYTHON_IMAGE" bash /builders/build-python.sh >"$OUT_DIR/build/build.log" 2>&1 ;; - go) - # Go has no artifact: the clean container runs `go get` at a pushed commit, as a user would. - local sha; sha="$(cat "$OUT_DIR/build/sha")" - case "$sha" in *+local-changes) - log "go: local changes are not tested; go get installs the commit ${sha%+local-changes} from GitHub" ;; - esac - echo "${sha%+local-changes}" >"$OUT_DIR/build/go_ref" ;; - esac + log "build: npm pack" + docker_run -v "$OUT_DIR/build:/out" -v "$HERE/builders:/builders:ro" \ + -e SURFACE -e PNPM_VERSION "$NODE_IMAGE" bash /builders/build-npm.sh >"$OUT_DIR/build/build.log" 2>&1 } # ---- 3. Clean test container: only the artifact, the checks and the inputs go in ---- run_checks() { rm -rf "$OUT_DIR/results"; mkdir -p "$OUT_DIR/results" - local image - case "$SURFACE" in node|mcp|cli) image="$NODE_IMAGE" ;; python) image="$PYTHON_IMAGE" ;; go) image="$GO_IMAGE" ;; esac - log "checks: a clean $image" + log "checks: a clean $NODE_IMAGE" local rc=0 docker_run --init \ -v "$OUT_DIR/build/artifact:/artifact:ro" \ -v "$HERE/surface:/harness:ro" \ -v "$OUT_DIR/results:/results" \ -e ZENROWS_API_KEY -e CRAWL_URL -e CRAWL_INCLUDE -e CRAWL_API_BASE \ - -e SURFACE -e GO_REF="$(cat "$OUT_DIR/build/go_ref" 2>/dev/null || true)" \ + -e SURFACE \ -e MCP_SDK_VERSION -e TYPESCRIPT_VERSION -e MYPY_VERSION \ -w /work --tmpfs /work:exec,size=2g \ - "$image" bash /harness/run-in-container.sh >"$OUT_DIR/results/container.log" 2>&1 || rc=$? + "$NODE_IMAGE" bash /harness/run-in-container.sh >"$OUT_DIR/results/container.log" 2>&1 || rc=$? log "checks: the container exited $rc (out/results/container.log)" } diff --git a/acceptance/crawl/tools/cleanup.py b/acceptance/crawl/tools/cleanup.py index 7228432..ef309dd 100644 --- a/acceptance/crawl/tools/cleanup.py +++ b/acceptance/crawl/tools/cleanup.py @@ -2,8 +2,10 @@ Reads out/results/crawls.jsonl, the ledger the checks write. A crawl still running is stopped; then each crawl is read for its final status and coverage.pages_fetched, which the -report uses for the cost table. Writes out/cleanup.json. +report uses for the cost table. Writes out/cleanup.json. A crawl the API cannot be reached +for is recorded with its error, not stopped, and the clean-up goes on with the next one. """ +import http.client import json import os import sys @@ -14,18 +16,33 @@ OUT = sys.argv[1] BASE = os.environ.get("CRAWL_API_BASE", "https://api.zenrows.com/v1").rstrip("/") KEY = os.environ["ZENROWS_API_KEY"] +BUSY_RETRIES = 3 +MAX_RETRY_AFTER = 30 def call(method, path): + """(status, body, Retry-After). A network error or timeout raises.""" req = urllib.request.Request(BASE + path, method=method, headers={"X-API-Key": KEY}) try: with urllib.request.urlopen(req, timeout=60) as r: - return r.status, json.loads(r.read() or b"null") + return r.status, json.loads(r.read() or b"null"), None except urllib.error.HTTPError as e: try: - return e.code, json.loads(e.read() or b"null") + body = json.loads(e.read() or b"null") except ValueError: - return e.code, None + body = None + return e.code, body, e.headers.get("Retry-After") + + +def stop(crawl_id): + # 503 crawl_busy: the stop was not saved and the crawl still runs. + for attempt in range(BUSY_RETRIES + 1): + status, body, retry_after = call("POST", f"/crawls/{crawl_id}/stop") + busy = status == 503 and isinstance(body, dict) and body.get("code") == "crawl_busy" + if not busy or attempt == BUSY_RETRIES: + return + wait = retry_after.strip() if retry_after else "" + time.sleep(min(int(wait), MAX_RETRY_AFTER) if wait.isdigit() else 1) crawls = {} @@ -41,21 +58,26 @@ def call(method, path): rows = [] for crawl_id, entry in crawls.items(): - status, body = call("GET", f"/crawls/{crawl_id}?limit=1") - stopped = False - if status == 200 and body and body.get("status") == "running": - call("POST", f"/crawls/{crawl_id}/stop") - stopped = True - time.sleep(2) - status, body = call("GET", f"/crawls/{crawl_id}?limit=1") + stopped, status, body, error = False, None, None, None + try: + status, body, _ = call("GET", f"/crawls/{crawl_id}?limit=1") + if status == 200 and body and body.get("status") == "running": + stop(crawl_id) + stopped = True + time.sleep(2) + status, body, _ = call("GET", f"/crawls/{crawl_id}?limit=1") + except (OSError, http.client.HTTPException) as e: + stopped, status, body, error = False, None, None, f"{type(e).__name__}: {e}" row = {"crawl_id": crawl_id, **entry, "stopped_by_cleanup": stopped, "http": status} + if error: + row["error"] = error if status == 200 and body: row.update(status=body.get("status"), stop_reason=body.get("stop_reason"), pages_fetched=body.get("coverage", {}).get("pages_fetched"), output_format=body.get("output_format")) rows.append(row) print(f"{crawl_id} {row.get('status')} pages_fetched={row.get('pages_fetched')}" - f"{' (stopped by clean-up)' if stopped else ''}") + f"{' (stopped by clean-up)' if stopped else ''}{f' ({error})' if error else ''}") still = [r["crawl_id"] for r in rows if r.get("status") == "running"] with open(os.path.join(OUT, "cleanup.json"), "w") as f: diff --git a/acceptance/crawl/tools/scan_key.py b/acceptance/crawl/tools/scan_key.py index f6f0419..65a704d 100644 --- a/acceptance/crawl/tools/scan_key.py +++ b/acceptance/crawl/tools/scan_key.py @@ -4,9 +4,10 @@ key = os.environ.get("ZENROWS_API_KEY", "").encode() hits = [] -for root, _, files in os.walk(sys.argv[1]): - if os.sep + "build" in root: - continue +for root, dirs, files in os.walk(sys.argv[1]): + # The builders' source copy and artifact: what goes in, not what the run wrote. + if root == sys.argv[1] and "build" in dirs: + dirs.remove("build") for f in files: p = os.path.join(root, f) try: From e3a9ecb79b139814da22ca2416bc0ddaf43608a7 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Tue, 13 Oct 2026 11:06:12 +0200 Subject: [PATCH 18/19] feat(crawl): optional --depth with 100-page and 100-item defaults, and stop retries on 429, 502, 503 and 504 Folds the two stacked changes into this branch so the crawl command ships as one pull request: - --depth is optional and unbounded; --max-pages and --max-items default to 100 - crawl stop retries 429, 502, 503 and 504, waiting Retry-After up to 30 s, or 1 s when it is missing or unusable --- README.md | 16 ++++---- acceptance/crawl/README.md | 8 ++-- acceptance/crawl/surface/harness.mjs | 2 +- acceptance/crawl/surface/test.mjs | 6 ++- docs/capabilities.md | 2 +- registry/capabilities.json | 2 +- skills/crawl/SKILL.md | 16 +++++--- src/cli/commands/crawl.ts | 33 +++++++---------- src/core/crawl-api.ts | 34 ++++++++--------- tests/crawl-api.test.ts | 55 ++++++++++++++++++++++++++-- tests/crawl-command.test.ts | 35 ++++++++++++++++-- 11 files changed, 145 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 64c7159..fe59566 100644 --- a/README.md +++ b/README.md @@ -164,8 +164,8 @@ Crawl is still evolving: new features are coming, limits may be tuned, and the c Crawl (`https://api.zenrows.com/v1/crawls`) takes one start URL and returns the URLs it finds behind it, optionally with each page's HTML. Use it when you know the listing or section but not the item -URLs. Crawl follows the links on each page up to `--depth` hops and stays on -the start URL's registrable domain (subdomains count). Without +URLs. Crawl follows the links on each page, up to `--depth` hops when you set +it, and stays on the start URL's registrable domain (subdomains count). Without `--output-format html` it returns URLs only. An account without Crawl access gets `CRAWL_NOT_ENABLED` ("Crawl is not enabled for this account", HTTP 403 `REQS008`). @@ -183,9 +183,10 @@ zenrows crawl list # your crawls, newest fi zenrows crawl stop # stop a running crawl ``` -`--depth` (link hops) is required; `--max-items` and `--max-pages` cap the -crawl (API default 10 each), and `--max-pages` bounds the cost, since each page -is one fetch. The local policy `max_pages_per_run` caps `--max-pages` (default +`--depth` (link hops, 1 to 100000) is optional: leave it out and the crawl +follows links until `--max-items` or `--max-pages` stops it, or nothing is left +to open. `--max-items` and `--max-pages` cap the crawl (API default 100 each), +and `--max-pages` bounds the cost, since each page is one fetch. The local policy `max_pages_per_run` caps `--max-pages` (default 1000). `--output-format html` uses up the page budget: each kept page is one more fetch, and `--max-pages` counts it. `--include-patterns` / `--exclude-patterns` are repeatable substring filters on the URL. `get` and @@ -223,8 +224,9 @@ waiting the `Retry-After` seconds (at most 30) or 250 ms x 2^n (+/-20% jitter, at most 10 s). Reads retry on 429, 502, 503, 504 and network errors (a timeout counts). `create` retries only with `--idempotency-key`, and only on 502, 503, 504 and -network errors, never on 429. `stop` retries only a 503 `crawl_busy`, which -means the stop was not saved and the crawl still runs. +network errors, never on 429. `stop` retries on 429, 502, 503 (a `crawl_busy` +means the stop was not saved and the crawl still runs) and 504, waiting the +`Retry-After` seconds (at most 30) or 1 s; repeating a stop is safe. ## 11. Browser Sessions diff --git a/acceptance/crawl/README.md b/acceptance/crawl/README.md index f6589e7..06388b8 100644 --- a/acceptance/crawl/README.md +++ b/acceptance/crawl/README.md @@ -60,7 +60,7 @@ and signal checks. | # | Feature | Expected | | --- | --- | --- | -| 1 | Create | returns at once: `running`, a `crawl_id`, the echoed parameters; only the fields set are sent | +| 1 | Create | returns at once: `running`, a `crawl_id`, the echoed parameters; only the fields set are sent (no `--depth` sends no `depth`) | | 2 | Create with html | results get `content_status` and, once fetched, `content_url` | | 3 | Idempotency | same key and body → same `crawl_id`; different body → 422 `idempotency_key_reused` (MCP, CLI: `CRAWL_INVALID_REQUEST` with the advice to send a new key) | | 4 | Wait | returns the ended crawl; `failed` is returned, not raised (CLI: exit 1) | @@ -80,13 +80,15 @@ and signal checks. | 18 | Copy | Crawl is "new", never "beta"; one "still evolving" line; the 429 limit reads as the account's active jobs, shared with Batch | | 19 | Failed crawl | an include pattern that matches nothing ends `failed` with `no_items_found`; SDKs return it, not raise it; MCP and CLI say `ok: false` (CLI: exit 1, `CRAWL_FAILED`) | | 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results ends at once (MCP and CLI: `partial: false`, no note to call again), and the same URLs come back once the pages in flight finish | -| 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (10) | +| 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (100) | Extra checks for this surface: every subcommand prints JSON with `--json` (`create`, `get`, `wait` and `stop` print `{ok, crawl_id, status, crawl, note?}`), Ctrl-C during `crawl wait` prints the resume hint and exits 130, and `--wait` and `--html` are rejected as unknown flags. ## What it costs -Every crawl uses `depth` 1 and `max_pages` 10 or less. A run starts five crawls: the README run, +Every crawl uses `depth` 1, and `max_pages` 10 or less except check 19's, which leaves `max_pages` +and `max_items` at the API default (100) for check 21; at `depth` 1 without html it fetches one +page. A run starts five crawls: the README run, one URLs-only crawl, one idempotency crawl that is stopped at once, one html crawl that is stopped with its pages in flight (check 20), and one that keeps nothing (check 19). On the default `CRAWL_URL`, a URLs-only crawl fetches 1 page and the html crawl 6, so a run costs about **10 pages**, and at most 50 if every crawl reaches its diff --git a/acceptance/crawl/surface/harness.mjs b/acceptance/crawl/surface/harness.mjs index ff958b9..763d581 100644 --- a/acceptance/crawl/surface/harness.mjs +++ b/acceptance/crawl/surface/harness.mjs @@ -13,7 +13,7 @@ export const API_BASE = process.env.CRAWL_API_BASE || "https://api.zenrows.com/v /** An include pattern no URL contains: the crawl keeps nothing and ends failed (check 19). */ export const NO_MATCH = "/crawl-acceptance-matches-nothing/"; /** The default every client documents for max_pages and max_items (check 21). */ -export const DOCUMENTED_DEFAULT = 10; +export const DOCUMENTED_DEFAULT = 100; export const PASS = "PASS"; export const FAIL = "FAIL"; export const SKIP = "SKIP"; diff --git a/acceptance/crawl/surface/test.mjs b/acceptance/crawl/surface/test.mjs index c4e5fa4..f1eb617 100644 --- a/acceptance/crawl/surface/test.mjs +++ b/acceptance/crawl/surface/test.mjs @@ -228,6 +228,7 @@ await check( // ---------- check 1, offline half: what the CLI sends ---------- const before = cap.requests.length; +await zAsync(["crawl", "create", "https://example.com/", "--json"], capEnv); await zAsync(["crawl", "create", "https://example.com/", "--depth", "1", "--json"], capEnv); await zAsync( [ @@ -249,11 +250,12 @@ await zAsync( capEnv, ); const posts = cap.requests.slice(before).filter((r) => r.method === "POST"); -const sent = { minimal: posts[0]?.body, full: posts[1]?.body }; +const sent = { bare: posts[0]?.body, minimal: posts[1]?.body, full: posts[2]?.body }; const sentOk = + H.sameKeys(sent.bare, ["url"]) && H.sameKeys(sent.minimal, ["url", "depth"]) && H.sameKeys(sent.full, ["url", "depth", "max_items", "max_pages", "include_patterns"]); -const sentLine = `sent (stand-in server): minimal=${JSON.stringify(sent.minimal)} full=${JSON.stringify(sent.full)}`; +const sentLine = `sent (stand-in server): bare=${JSON.stringify(sent.bare)} minimal=${JSON.stringify(sent.minimal)} full=${JSON.stringify(sent.full)}`; await cap.close(); // ---------- live ---------- diff --git a/docs/capabilities.md b/docs/capabilities.md index bc52ad3..fd9ced3 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -22,6 +22,6 @@ Classification is based on the public Zenrows documentation: | `protected_fetch` | Fetch `GET https://api.zenrows.com/v1/` with `mode`, `js_render`, `premium_proxy`, `proxy_country`, `wait`/`wait_for`, `js_instructions`, `response_type`, `screenshot`, `original_status`, … | available | | `extract` | Extract `GET https://api.zenrows.com/v1/` via `extract=auto` (domain-gated open beta; CLI falls back to `autoparse`), plus `autoparse`, `css_extractor`, `outputs`, `response_type=markdown\|plaintext` | beta | | `batch` | Batch `https://async.api.zenrows.com/v1` (separate host, `X-API-Key` header) — open beta. Local JSONL spec validation + credit estimation work with no key. | beta | -| `crawl` | Crawl `https://api.zenrows.com/v1/crawls` (`X-API-Key` header; same host as Fetch/Extract). Follows the links on each page from one start URL up to `depth`, on the start URL's registrable domain (subdomains count); returns URLs, and with `output_format: "html"` each page's HTML. Subcommands: `create`, `get`, `list`, `results`, `content`, `download`, `stop`, `wait`. Without Crawl access the API returns 403 `REQS008` → `CRAWL_NOT_ENABLED`. | available | +| `crawl` | Crawl `https://api.zenrows.com/v1/crawls` (`X-API-Key` header; same host as Fetch/Extract). Follows the links on each page from one start URL, up to `depth` when set (optional; no depth limit when omitted), on the start URL's registrable domain (subdomains count); returns URLs, and with `output_format: "html"` each page's HTML. Subcommands: `create`, `get`, `list`, `results`, `content`, `download`, `stop`, `wait`. Without Crawl access the API returns 403 `REQS008` → `CRAWL_NOT_ENABLED`. | available | | `browser` | Browser Sessions REST API `https://mcp.zenrows.com/browser/sessions/*` (Bearer; same backend as `@zenrows/mcp` `browser_*`); CDP via `zenrows browser connect`. Escalation-only (prefer fetch/extract); on by default, opt out via `policy.allow_browser`; bills by bandwidth + session time (15-min max) | available | | `mcp` | Hosted `https://mcp.zenrows.com/mcp` + local `npx -y @zenrows/mcp` | available | diff --git a/registry/capabilities.json b/registry/capabilities.json index 24a1b4f..22b726a 100644 --- a/registry/capabilities.json +++ b/registry/capabilities.json @@ -35,7 +35,7 @@ "command": "zenrows crawl", "backend": "Crawl — https://api.zenrows.com/v1/crawls (X-API-Key header; same host as Fetch/Extract)", "requiresAuth": true, - "notes": "Crawl is still evolving: new features are coming, limits may be tuned, and the changelog announces each change. Follows the links on each page from one start URL up to depth, on the start URL's registrable domain (subdomains count), and returns the URLs it keeps, optionally with each page's HTML (create/get/list/results/content/download/stop/wait). An account without Crawl access gets 403 REQS008 (CRAWL_NOT_ENABLED). When the account has reached its limit of active jobs (3 by default), shared with its Batch jobs, a create gets 429 too_many_crawls (CRAWL_TOO_MANY_CRAWLS); retry after retry_after seconds." + "notes": "Crawl is still evolving: new features are coming, limits may be tuned, and the changelog announces each change. Follows the links on each page from one start URL, up to depth when set (optional; without it max_pages / max_items stop the crawl, both default 100), on the start URL's registrable domain (subdomains count), and returns the URLs it keeps, optionally with each page's HTML (create/get/list/results/content/download/stop/wait). An account without Crawl access gets 403 REQS008 (CRAWL_NOT_ENABLED). When the account has reached its limit of active jobs (3 by default), shared with its Batch jobs, a create gets 429 too_many_crawls (CRAWL_TOO_MANY_CRAWLS); retry after retry_after seconds." }, "browser": { "key": "browser", diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index caea8f4..9ed7fbc 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -7,8 +7,8 @@ requires_backend_capabilities: [crawl] # Crawl (new) -Give Crawl one start URL; it follows the links on that page (and, with a -higher depth, on the pages behind it) and returns the URLs it keeps. Use it +Give Crawl one start URL; it follows the links on that page (and on the +pages behind it, up to the depth you set) and returns the URLs it keeps. Use it when you know where the items live (a category page, a blog index, a docs section) but not the item URLs themselves. Once you have the URLs, read pages with `zenrows fetch` / `zenrows extract`, or ask Crawl for each page's HTML @@ -27,10 +27,13 @@ zenrows crawl results --json # every kept URL zenrows crawl results --out urls.jsonl # same, as JSONL ``` -- `--depth N` is required: link hops from the start URL. `1` = the start - page's links; `2` = also the links on each of those pages. +- `--depth N` (1 to 100000) is optional: link hops from the start URL. `1` = + the start page's links; `2` = also the links on each of those pages. Leave + it out for no depth limit: the crawl follows links until `--max-items` or + `--max-pages` stops it, or nothing is left to open. A crawl created without + a depth has no `depth` field in its output. - `--max-items` (kept URLs) and `--max-pages` (fetched pages) cap the crawl; - both default to 10. `--max-pages` bounds the cost: each page is one fetch. + both default to 100. `--max-pages` bounds the cost: each page is one fetch. The local policy `max_pages_per_run` caps `--max-pages` (default 1000); above it the CLI refuses with `POLICY_LIMIT_EXCEEDED` before any request. - `--include-patterns` / `--exclude-patterns` are substring matches on the @@ -97,7 +100,8 @@ The CLI retries transient failures, as the Crawl SDKs do: up to 3 retries, waiting the `Retry-After` seconds (at most 30) or 250 ms x 2^n (+/-20% jitter, at most 10 s). Reads retry on 429, 502, 503, 504 and network errors (a timeout counts). `create` retries only with `--idempotency-key`, and only on 502, 503, -504 and network errors, never on 429. `stop` retries only a 503 `crawl_busy`. +504 and network errors, never on 429. `stop` retries on 429, 502, 503 (including +`crawl_busy`) and 504, waiting the `Retry-After` seconds (at most 30) or 1 s. The "Retry?" column is for errors that remain after these retries. | Code | Meaning | Retry? | diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index 1900f2f..eeb615b 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -34,7 +34,7 @@ import { ToolkitError } from "../../core/errors.ts"; import { printError, writeOut, writeOutStream } from "../output.ts"; /** The API's default for `max_pages` (and `max_items`) when the flag is unset. */ -const DEFAULT_MAX_PAGES = 10; +const DEFAULT_MAX_PAGES = 100; /** Largest `depth` / `max_items` / `max_pages` the API takes. */ const MAX_LIMIT = 100_000; @@ -42,13 +42,14 @@ export const crawl: Command = { name: "crawl", summary: "Crawl a site from one start URL and collect the URLs behind it (new).", usage: - "zenrows crawl --depth N|get |list|results |content |download |stop |wait >", + "zenrows crawl [--depth N]|get |list|results |content |download |stop |wait >", help: [ "Crawl is new. Cloud (needs a key with Crawl access):", - " create --depth [flags] start a crawl from one URL (returns at once)", - " --depth link hops to follow from the start URL (up to 100000, required)", - " --max-items stop after keeping n URLs (API default 10)", - " --max-pages stop after fetching n pages (API default 10; bounds cost);", + " create [flags] start a crawl from one URL (returns at once)", + " --depth link hops to follow from the start URL (1 to 100000);", + " leave it out for no depth limit (the caps below stop the crawl)", + " --max-items stop after keeping n URLs (API default 100)", + " --max-pages stop after fetching n pages (API default 100; bounds cost);", " the local policy max_pages_per_run caps it (default 1000)", " --include-patterns

keep only URLs containing this substring (repeatable)", " --exclude-patterns

drop URLs containing this substring (repeatable)", @@ -101,7 +102,7 @@ export const crawl: Command = { message: `Unknown crawl subcommand: ${sub ?? "(none)"}`, likely_cause: "Subcommand not recognized.", next_action: - "Use create --depth N | get | list | results | content | download | stop | wait .", + "Use create [--depth N] | get | list | results | content | download | stop | wait .", suggested_commands: ["zenrows crawl --help"], }); } @@ -133,21 +134,13 @@ async function createCmd(rest: string[], ctx: RunContext): Promise { code: "INVALID_USAGE", message: "Provide the start URL.", likely_cause: "No positional was given.", - next_action: "Usage: zenrows crawl create --depth 1", + next_action: "Usage: zenrows crawl create [--depth N]", suggested_commands: ["zenrows crawl create https://example.com/ --depth 1"], }); } + const params: CreateCrawlParams = { url }; const depth = limitFlag("depth", values.depth); - if (depth === undefined) { - throw new ToolkitError({ - code: "INVALID_USAGE", - message: "--depth is required.", - likely_cause: "Crawl has no default depth yet; every crawl states how many link hops to follow.", - next_action: "Pass --depth 1 to collect the start page's links, --depth 2 to also open each of those.", - suggested_commands: [`zenrows crawl create ${url} --depth 1`], - }); - } - const params: CreateCrawlParams = { url, depth }; + if (depth !== undefined) params.depth = depth; const maxItems = limitFlag("max-items", values["max-items"]); if (maxItems !== undefined) params.max_items = maxItems; const maxPages = limitFlag("max-pages", values["max-pages"]); @@ -188,7 +181,7 @@ async function createCmd(rest: string[], ctx: RunContext): Promise { const runId = newRunId(); const startedAt = new Date().toISOString(); - log.step(`Starting crawl of ${url} (depth ${depth})…`); + log.step(`Starting crawl of ${url} (${depth === undefined ? "no depth limit" : `depth ${depth}`})…`); try { const created = await createCrawl(params, { apiKey, idempotencyKey }); const follow = values.follow === true; @@ -444,7 +437,7 @@ export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null { insufficient_credits: "The account ran out of credits. Check `zenrows usage`, add credits, then start a new crawl.", seed_unreachable: "The start URL could not be fetched (the site may be blocking it). Check it with `zenrows fetch `.", no_items_found: - "Nothing matched. Loosen --include-patterns / --exclude-patterns, raise --depth, or check the start page with `zenrows fetch `.", + "Nothing matched. Loosen --include-patterns / --exclude-patterns, raise or drop --depth, or check the start page with `zenrows fetch `.", }; return new ToolkitError({ code: "CRAWL_FAILED", diff --git a/src/core/crawl-api.ts b/src/core/crawl-api.ts index 063a4f1..1889eea 100644 --- a/src/core/crawl-api.ts +++ b/src/core/crawl-api.ts @@ -54,7 +54,8 @@ export interface Crawl { /** running | completed | stopped | failed. Terminal = anything but `running`. */ status: string; url: string; - depth: number; + /** Absent when the crawl was created without one: no depth limit. */ + depth?: number; max_items: number; max_pages: number; coverage: CrawlCoverage; @@ -106,7 +107,8 @@ export interface CrawlStop { /** Create body. Only the fields the caller set are sent. */ export interface CreateCrawlParams { url: string; - depth: number; + /** Link hops from the start URL. Unset = no depth limit: max_pages / max_items stop the crawl. */ + depth?: number; max_items?: number; max_pages?: number; include_patterns?: string[]; @@ -147,8 +149,11 @@ interface RequestOpts extends CallOpts { accept?: string; /** Set on every call about one crawl, so its errors carry `crawl_id`. */ crawlId?: string; - /** Retry a 503 crawl_busy (stop only): the stop was not saved and the crawl still runs. */ - retryBusy?: boolean; + /** + * Retry a stop on `STOP_RETRY_STATUSES`, waiting Retry-After or 1 s. A stop is + * safe to repeat: an ended crawl answers 200 with its status. + */ + retryStop?: boolean; /** Consume a 2xx body as a stream instead of reading it as text. Called again on each retry. */ sink?: (body: AsyncIterable) => Promise; } @@ -165,11 +170,13 @@ const MAX_RETRY_AFTER_SECONDS = 30; const READ_RETRY_STATUSES: ReadonlySet = new Set([429, 502, 503, 504]); // 429 on create is the active-job limit: a retry only waits for a slot that may never free. const CREATE_RETRY_STATUSES: ReadonlySet = new Set([502, 503, 504]); +// Any 503 on stop, crawl_busy included: crawl_busy means the stop was not saved and the crawl still runs. +const STOP_RETRY_STATUSES: ReadonlySet = new Set([429, 502, 503, 504]); /** * The statuses to retry, or undefined when the request must run once: a GET is - * safe to repeat, a create only with an Idempotency-Key. Stop retries only a 503 - * crawl_busy (see `crawlFetch`). + * safe to repeat, a create only with an Idempotency-Key. Stop retries on its own + * statuses and wait rule (see `retryStop`). */ function retryStatuses(method: string, headers: Record): ReadonlySet | undefined { if (method === "GET") return READ_RETRY_STATUSES; @@ -186,14 +193,6 @@ function retryAfterMs(value: string | null): number | undefined { return value && /^\d+$/.test(value.trim()) ? Math.min(Number(value), MAX_RETRY_AFTER_SECONDS) * 1000 : undefined; } -function problemCode(body: string): string | undefined { - try { - return (JSON.parse(body) as ProblemJson).code; - } catch { - return undefined; - } -} - const defaultSleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); /** One HTTP attempt with its own timeout; the body is read inside it. */ @@ -262,7 +261,7 @@ async function crawlFetch(method: string, path: string, opts: RequestOpts): Prom await sleep(retryAfterMs(retryAfter) ?? backoffMs(n)); continue; } - if (opts.retryBusy && n < MAX_RETRIES && res.status === 503 && problemCode(res.text) === "crawl_busy") { + if (opts.retryStop && n < MAX_RETRIES && STOP_RETRY_STATUSES.has(res.status)) { await sleep(retryAfterMs(retryAfter) ?? 1000); continue; } @@ -421,7 +420,8 @@ function classifyProblem( /** Start a crawl (202). Sends only the fields set in `params`. */ export function createCrawl(params: CreateCrawlParams, opts: CallOpts & { idempotencyKey?: string }): Promise { - const body: Record = { url: params.url, depth: params.depth }; + const body: Record = { url: params.url }; + if (params.depth !== undefined) body.depth = params.depth; if (params.max_items !== undefined) body.max_items = params.max_items; if (params.max_pages !== undefined) body.max_pages = params.max_pages; if (params.include_patterns?.length) body.include_patterns = params.include_patterns; @@ -450,7 +450,7 @@ export function listCrawls(opts: CallOpts & { cursor?: string; limit?: number }) /** Stop a running crawl. Idempotent: an ended crawl answers as it ended. */ export function stopCrawl(id: string, opts: CallOpts): Promise { - return crawlJson("POST", `/crawls/${encodeURIComponent(id)}/stop`, { ...opts, crawlId: id, retryBusy: true }); + return crawlJson("POST", `/crawls/${encodeURIComponent(id)}/stop`, { ...opts, crawlId: id, retryStop: true }); } /** The page of one kept URL (HTML for `output_format: html`). */ diff --git a/tests/crawl-api.test.ts b/tests/crawl-api.test.ts index 94a5e4e..5a9cd0e 100644 --- a/tests/crawl-api.test.ts +++ b/tests/crawl-api.test.ts @@ -100,6 +100,14 @@ test("createCrawl posts /crawls with X-API-Key and sends only the fields the cal assert.ok(!("output_format" in body)); }); +test("createCrawl leaves depth out when the caller set none", async () => { + const { depth: _depth, ...noDepth } = crawlBody; + const { impl, calls } = stubFetch(202, noDepth); + const c = await createCrawl({ url: "https://example.com/products/" }, { apiKey: "k", fetchImpl: impl }); + assert.deepEqual(JSON.parse(String(calls[0]!.init?.body)), { url: "https://example.com/products/" }); + assert.equal(c.depth, undefined); +}); + test("createCrawl sends output_format html and Idempotency-Key when asked", async () => { const { impl, calls } = stubFetch(202, crawlBody); await createCrawl({ url: "https://example.com/products/", depth: 2, max_pages: 5, output_format: "html" }, { @@ -400,11 +408,52 @@ test("create with an Idempotency-Key is retried on 503 with the same key", async assert.ok(calls.every((c) => (c.init?.headers as Record)["Idempotency-Key"] === "k1")); }); -test("stop is not retried on a plain 503", async () => { - const { impl, calls } = seqFetch([unavailable, { status: 200, payload: { crawl_id: "c_1", status: "stopped" } }]); +const stopped = { status: 200, payload: { crawl_id: "c_1", status: "stopped" } }; + +for (const status of [429, 502, 504]) { + test(`stop retries a ${status} once, waiting 1 s without Retry-After, then returns the stopped crawl`, async () => { + const { impl, calls } = seqFetch([{ status, payload: { code: "transient", status } }, stopped]); + const { waits, sleepImpl } = recordSleep(); + const s = await stopCrawl("c_1", { apiKey: "k", fetchImpl: impl, sleepImpl }); + assert.equal(s.status, "stopped"); + assert.equal(calls.length, 2); + assert.deepEqual(waits, [1000]); + }); +} + +test("stop retries a plain 503 and honors Retry-After", async () => { + const { impl, calls } = seqFetch([{ ...unavailable, headers: { "retry-after": "2" } }, stopped]); + const { waits, sleepImpl } = recordSleep(); + const s = await stopCrawl("c_1", { apiKey: "k", fetchImpl: impl, sleepImpl }); + assert.equal(s.status, "stopped"); + assert.equal(calls.length, 2); + assert.deepEqual(waits, [2000]); +}); + +test("stop gives up after 3 retries and maps the last status as before", async () => { + const cases: Array<[number, string]> = [ + [429, "CRAWL_TOO_MANY_CRAWLS"], + [502, "CRAWL_FAILED"], + [503, "CRAWL_FAILED"], + [504, "CRAWL_FAILED"], + ]; + for (const [status, code] of cases) { + const { impl, calls } = seqFetch([{ status, payload: { code: "transient", status } }]); + const { waits, sleepImpl } = recordSleep(); + await assert.rejects( + () => stopCrawl("c_1", { apiKey: "k", fetchImpl: impl, sleepImpl }), + (e: unknown) => e instanceof ToolkitError && e.code === code && e.crawl_id === "c_1", + ); + assert.equal(calls.length, 4, `${status}: 1 attempt + 3 retries`); + assert.deepEqual(waits, [1000, 1000, 1000]); + } +}); + +test("stop is not retried on a 500", async () => { + const { impl, calls } = seqFetch([{ status: 500, payload: { code: "internal_error" } }, stopped]); await assert.rejects( () => stopCrawl("c_1", { apiKey: "k", fetchImpl: impl, sleepImpl: recordSleep().sleepImpl }), - (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_FAILED" && e.crawl_id === "c_1", + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_FAILED", ); assert.equal(calls.length, 1); }); diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index 65bb820..90374e0 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -123,11 +123,11 @@ test("crawl create takes --output-format html only and --timeout in positive sec ); }); -test("crawl create requires --depth as a whole number up to 100000", async () => { +test("crawl create takes --depth as a whole number up to 100000", async () => { await withCrawlWorkspace( () => json(crawlBody, 202), async (calls) => { - for (const args of [[], ["--depth", "1.5"], ["--depth", "100001"], ["--depth", "x"]]) { + for (const args of [["--depth", "1.5"], ["--depth", "100001"], ["--depth", "x"]]) { const { code, out } = await run(["create", "https://example.com/products/", ...args]); assert.equal(code, 1, args.join(" ")); assert.equal(out.error.code, "INVALID_USAGE", args.join(" ")); @@ -141,7 +141,7 @@ test("crawl create leaves the lower bound to the API: --depth 0 comes back as CR await withCrawlWorkspace( () => json( - { type: "about:blank", title: "Invalid parameter", status: 422, code: "invalid_parameter", detail: "'depth' is required, and at least 1." }, + { type: "about:blank", title: "Invalid parameter", status: 422, code: "invalid_parameter", detail: "'depth' is between 1 and 100000. Leave it out for no depth limit." }, 422, ), async (calls) => { @@ -196,6 +196,35 @@ test("crawl create sends only the flags the caller set as the create body", asyn ); }); +test("crawl create without --depth sends no depth and prints a crawl that has none", async () => { + const { depth: _depth, ...noDepth } = crawlBody; + await withCrawlWorkspace( + () => json({ ...noDepth, max_items: 100, max_pages: 100 }, 202, { location: "/v1/crawls/c_1" }), + async (calls) => { + const { code, out } = await run(["create", "https://example.com/products/"]); + assert.equal(code, 0); + assert.equal(out.ok, true); + assert.equal(calls.length, 1); + assert.deepEqual(JSON.parse(String(calls[0]!.init?.body)), { url: "https://example.com/products/" }); + assert.ok(!("depth" in out.crawl), "a crawl without a depth limit echoes no depth"); + assert.equal(out.crawl.max_pages, 100); + }, + ); +}); + +test("crawl create without --max-pages counts the API default of 100 against the page cap", async () => { + await withCrawlWorkspace( + () => json(crawlBody, 202), + async (calls) => { + const { code, out } = await run(["create", "https://example.com/products/"]); + assert.equal(code, 1); + assert.equal(out.error.code, "POLICY_LIMIT_EXCEEDED"); + assert.equal(calls.length, 0); + }, + { max_pages_per_run: 99 }, + ); +}); + test("crawl create honors the domain policy and the page cap before any network call", async () => { await withCrawlWorkspace( () => json(crawlBody, 202), From 01c105d2341239ded2a7f12c281d7a04dade0ee0 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Tue, 13 Oct 2026 11:58:19 +0200 Subject: [PATCH 19/19] fix(crawl): name a 429 too_many_crawls only when the API says so The Crawl API sends 429 only on create, with the code too_many_crawls. Any other 429, on a stop or a read, now maps to CRAWL_RATE_LIMITED with retry_after, instead of naming the active-job limit and "Nothing was created". --- README.md | 7 ++++--- registry/capabilities.json | 2 +- skills/crawl/SKILL.md | 7 +++++-- src/core/crawl-api.ts | 12 +++++++++++- src/core/errors.ts | 1 + tests/crawl-api.test.ts | 20 +++++++++++++++++++- tests/crawl-command.test.ts | 30 ++++++++++++++++++++++++++++++ 7 files changed, 71 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index fe59566..9a92c60 100644 --- a/README.md +++ b/README.md @@ -209,9 +209,10 @@ with `CRAWL_FAILED`. Errors carry the API's code as `server_code`, `crawl_id` on any call about one crawl, and `retry_after` on a 429. When the account has reached its limit of -active jobs (3 by default), shared with its Batch jobs, a create gets -`CRAWL_TOO_MANY_CRAWLS` and creates nothing; retry after `retry_after` seconds. -Other codes: `CRAWL_INSUFFICIENT_CREDITS` and `CRAWL_KEY_CAP_REACHED` (402, with the +active jobs (3 by default), shared with its Batch jobs, a create gets 429 +`too_many_crawls`, mapped to `CRAWL_TOO_MANY_CRAWLS`, and creates nothing; retry +after `retry_after` seconds. Any other 429 is `CRAWL_RATE_LIMITED`: requests +are being throttled; retry after `retry_after` seconds. Other codes: `CRAWL_INSUFFICIENT_CREDITS` and `CRAWL_KEY_CAP_REACHED` (402, with the same claim, top-up and key-cap advice as other commands), `CRAWL_INVALID_REQUEST` (400/422, do not retry as is), `CRAWL_REQUEST_IN_FLIGHT` (409, retry once the first request ends), `CRAWL_NOT_FOUND`, diff --git a/registry/capabilities.json b/registry/capabilities.json index 22b726a..fce5b92 100644 --- a/registry/capabilities.json +++ b/registry/capabilities.json @@ -35,7 +35,7 @@ "command": "zenrows crawl", "backend": "Crawl — https://api.zenrows.com/v1/crawls (X-API-Key header; same host as Fetch/Extract)", "requiresAuth": true, - "notes": "Crawl is still evolving: new features are coming, limits may be tuned, and the changelog announces each change. Follows the links on each page from one start URL, up to depth when set (optional; without it max_pages / max_items stop the crawl, both default 100), on the start URL's registrable domain (subdomains count), and returns the URLs it keeps, optionally with each page's HTML (create/get/list/results/content/download/stop/wait). An account without Crawl access gets 403 REQS008 (CRAWL_NOT_ENABLED). When the account has reached its limit of active jobs (3 by default), shared with its Batch jobs, a create gets 429 too_many_crawls (CRAWL_TOO_MANY_CRAWLS); retry after retry_after seconds." + "notes": "Crawl is still evolving: new features are coming, limits may be tuned, and the changelog announces each change. Follows the links on each page from one start URL, up to depth when set (optional; without it max_pages / max_items stop the crawl, both default 100), on the start URL's registrable domain (subdomains count), and returns the URLs it keeps, optionally with each page's HTML (create/get/list/results/content/download/stop/wait). An account without Crawl access gets 403 REQS008 (CRAWL_NOT_ENABLED). When the account has reached its limit of active jobs (3 by default), shared with its Batch jobs, a create gets 429 too_many_crawls (CRAWL_TOO_MANY_CRAWLS); retry after retry_after seconds. Any other 429 is CRAWL_RATE_LIMITED (requests are being throttled); retry after retry_after seconds." }, "browser": { "key": "browser", diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index 9ed7fbc..cac5d8f 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -89,7 +89,9 @@ under `crawl`. `get` adds `results` and `next_cursor`. `download` prints When the account has reached its limit of active jobs (3 by default), shared with its Batch jobs, a create gets 429 `too_many_crawls` → `CRAWL_TOO_MANY_CRAWLS`; nothing was created. Retry after `retry_after` -seconds (about 30), or stop one of your crawls or Batch jobs first. +seconds (about 30), or stop one of your crawls or Batch jobs first. Any other +429 is `CRAWL_RATE_LIMITED`: requests are being throttled; retry after +`retry_after` seconds. ## Errors @@ -113,7 +115,8 @@ The "Retry?" column is for errors that remain after these retries. | `CRAWL_INVALID_REQUEST` | 400 or 422: fix the request. For `idempotency_key_reused`, send a new key or no key | no, not as is | | `CRAWL_REQUEST_IN_FLIGHT` | 409: the same Idempotency-Key is still in flight | yes, once the first request ends | | `CRAWL_NOT_FOUND` / `CRAWL_CONTENT_NOT_FOUND` | 404: wrong id, or the page has no content | no | -| `CRAWL_TOO_MANY_CRAWLS` | 429: the account's active-job limit | yes, after `retry_after` | +| `CRAWL_TOO_MANY_CRAWLS` | 429 `too_many_crawls` on `create`: the account's active-job limit | yes, after `retry_after` | +| `CRAWL_RATE_LIMITED` | any other 429: requests are being throttled | yes, after `retry_after` | | `CRAWL_BUSY` | 503 `crawl_busy` on `stop`: the stop was not saved yet and the crawl still runs | yes, run the stop again after `retry_after` | | `CRAWL_FAILED` | the crawl ended `failed`, or another error | 5xx: yes, with backoff | | `BACKEND_UNAVAILABLE` | the API could not be reached | yes | diff --git a/src/core/crawl-api.ts b/src/core/crawl-api.ts index 1889eea..b0f2871 100644 --- a/src/core/crawl-api.ts +++ b/src/core/crawl-api.ts @@ -362,7 +362,7 @@ function classifyProblem( suggested_commands: ["zenrows crawl list"], }); } - if (status === 429) { + if (status === 429 && serverCode === "too_many_crawls") { return new ToolkitError({ code: "CRAWL_TOO_MANY_CRAWLS", message: "The account has reached its limit of active jobs.", @@ -371,6 +371,16 @@ function classifyProblem( suggested_commands: ["zenrows crawl list"], }); } + if (status === 429) { + // The Crawl API sends 429 only on create, as too_many_crawls. Any other 429 + // comes from something in front of it and says nothing about active jobs. + return new ToolkitError({ + code: "CRAWL_RATE_LIMITED", + message: "The Crawl API is rate limiting requests.", + likely_cause: cause, + next_action: retryAfter !== undefined ? `Retry after ${retryAfter} seconds.` : "Retry later.", + }); + } if (status === 402) { // Same advice as the shared credit errors, under Crawl's own codes. const detail = problem.detail || problem.title || undefined; diff --git a/src/core/errors.ts b/src/core/errors.ts index 2a3c093..1c4da84 100644 --- a/src/core/errors.ts +++ b/src/core/errors.ts @@ -39,6 +39,7 @@ export type ErrorCode = | "CRAWL_INSUFFICIENT_CREDITS" | "CRAWL_KEY_CAP_REACHED" | "CRAWL_TOO_MANY_CRAWLS" + | "CRAWL_RATE_LIMITED" | "CRAWL_NOT_FOUND" | "CRAWL_CONTENT_NOT_FOUND" | "CRAWL_INVALID_REQUEST" diff --git a/tests/crawl-api.test.ts b/tests/crawl-api.test.ts index 5a9cd0e..681bbd2 100644 --- a/tests/crawl-api.test.ts +++ b/tests/crawl-api.test.ts @@ -432,7 +432,7 @@ test("stop retries a plain 503 and honors Retry-After", async () => { test("stop gives up after 3 retries and maps the last status as before", async () => { const cases: Array<[number, string]> = [ - [429, "CRAWL_TOO_MANY_CRAWLS"], + [429, "CRAWL_RATE_LIMITED"], [502, "CRAWL_FAILED"], [503, "CRAWL_FAILED"], [504, "CRAWL_FAILED"], @@ -449,6 +449,24 @@ test("stop gives up after 3 retries and maps the last status as before", async ( } }); +test("a 429 without too_many_crawls on stop maps to CRAWL_RATE_LIMITED after the retries, not to the active-job limit", async () => { + for (const payload of [{ code: "rate_limited", status: 429 }, { title: "Too Many Requests", status: 429 }]) { + const { impl, calls } = seqFetch([{ status: 429, payload, headers: { "retry-after": "7" } }]); + await assert.rejects( + () => stopCrawl("c_1", { apiKey: "k", fetchImpl: impl, sleepImpl: recordSleep().sleepImpl }), + (e: unknown) => + e instanceof ToolkitError && + e.code === "CRAWL_RATE_LIMITED" && + e.message === "The Crawl API is rate limiting requests." && + e.retry_after === 7 && + e.crawl_id === "c_1" && + /Retry after 7 seconds/.test(e.next_action) && + !/Nothing was created|active jobs/.test(`${e.message} ${e.likely_cause} ${e.next_action}`), + ); + assert.equal(calls.length, 4, "1 attempt + 3 retries"); + } +}); + test("stop is not retried on a 500", async () => { const { impl, calls } = seqFetch([{ status: 500, payload: { code: "internal_error" } }, stopped]); await assert.rejects( diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index 90374e0..b46952f 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -154,6 +154,36 @@ test("crawl create leaves the lower bound to the API: --depth 0 comes back as CR ); }); +test("crawl create on 429 too_many_crawls returns CRAWL_TOO_MANY_CRAWLS; crawl stop on another 429 returns CRAWL_RATE_LIMITED", async () => { + await withCrawlWorkspace( + () => + json( + { type: "about:blank", title: "Too many crawls", status: 429, code: "too_many_crawls", detail: "The account has 3 active jobs." }, + 429, + { "retry-after": "30" }, + ), + async (calls) => { + const { code, out } = await run(["create", "https://example.com/products/"]); + assert.equal(code, 1); + assert.equal(out.error.code, "CRAWL_TOO_MANY_CRAWLS"); + assert.equal(out.error.retry_after, 30); + assert.match(out.error.next_action, /Nothing was created/); + assert.equal(calls.length, 1); + }, + ); + await withCrawlWorkspace( + () => json({ title: "Too Many Requests", status: 429 }, 429, { "retry-after": "0" }), + async (calls) => { + const { code, out } = await run(["stop", "c_1"]); + assert.equal(code, 1); + assert.equal(out.error.code, "CRAWL_RATE_LIMITED"); + assert.equal(out.error.retry_after, 0); + assert.doesNotMatch(`${out.error.message} ${out.error.likely_cause} ${out.error.next_action}`, /Nothing was created/); + assert.equal(calls.length, 4, "stop still retries a 429 three times"); + }, + ); +}); + test("crawl create sends only the flags the caller set as the create body", async () => { await withCrawlWorkspace( () => json(crawlBody, 202, { location: "/v1/crawls/c_1" }),