From 7be627fc7b81587b94d3597b0c2e2090d4462d0a Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Thu, 8 Oct 2026 16:53:04 +0200 Subject: [PATCH 1/3] feat(crawl): add `zenrows crawl` for the Crawl API Crawl takes one start URL, follows the links on each page up to `depth` on the start URL's domain, and returns the URLs it keeps, optionally with each page's HTML. The new `crawl` command drives https://api.zenrows.com/v1/crawls with the X-API-Key header: - start --depth N [--max-items N] [--max-pages N] [--include P]... [--exclude P]... [--html] [--follow|--wait] [--timeout ms] [--idempotency-key K] - status, wait, results (all pages, one page via --cursor/--limit, or the NDJSON export via --download), content, list, stop src/core/crawl-api.ts is the client (base override ZENROWS_CRAWL_API_BASE, otherwise the configured apiBase). Error responses map to new codes: CRAWL_NOT_ENABLED (403 REQS008), CRAWL_NOT_FOUND (404), CRAWL_QUOTA_EXCEEDED (429 too_many_crawls), CRAWL_INVALID (400/409/422), CRAWL_FAILED (a failed crawl or another error) and CRAWL_TIMEOUT (wait gave up; the crawl keeps running). The guards that asserted `crawl` is an unknown command and that no crawl skill exists are narrowed to `discover` and `monitor`. Adds the `crawl` capability, a crawl skill, README and docs sections, unit tests, and an opt-in e2e test (`npm run test:e2e`) that drives the built CLI against a live API. Co-Authored-By: Claude Opus 5.5 --- README.md | 52 +++- docs/capabilities.md | 1 + docs/contributing.md | 32 +++ package.json | 1 + registry/capabilities.json | 9 + registry/skills.json | 11 + skills/crawl/SKILL.md | 73 +++++ skills/trace-debug/SKILL.md | 8 + skills/zenrows/SKILL.md | 5 + src/cli/commands/crawl.ts | 473 +++++++++++++++++++++++++++++++ src/cli/index.ts | 3 +- src/core/crawl-api.ts | 459 ++++++++++++++++++++++++++++++ src/core/errors.ts | 6 + tests/capabilities.test.ts | 2 + tests/crawl-api.test.ts | 286 +++++++++++++++++++ tests/crawl-command.test.ts | 324 +++++++++++++++++++++ tests/e2e/crawl.e2e.test.ts | 141 +++++++++ tests/production-surface.test.ts | 5 +- tests/registry.test.ts | 7 +- 19 files changed, 1883 insertions(+), 15 deletions(-) create mode 100644 skills/crawl/SKILL.md create mode 100644 src/cli/commands/crawl.ts create mode 100644 src/core/crawl-api.ts create mode 100644 tests/crawl-api.test.ts create mode 100644 tests/crawl-command.test.ts create mode 100644 tests/e2e/crawl.e2e.test.ts diff --git a/README.md b/README.md index 4fd37d3..0973eaf 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ that endpoint with extraction parameters, not a separate product. | `zenrows fetch` | Fetch — retrieve a protected page | **available** — `GET https://api.zenrows.com/v1/` | | `zenrows extract` | Extract — `extract=auto` (domain-gated open beta; falls back to Autoparse) / CSS / Markdown | **beta** — same `/v1/` | | `zenrows batch` | Batch — fan out over many URLs | beta — cloud works with beta access; local validate/estimate always | +| `zenrows crawl` | Crawl — collect the URLs behind one start page, optionally with each page's HTML | **available** — `https://api.zenrows.com/v1/crawls` | | `zenrows browser` | Browser Sessions REST API (same backend as MCP `browser_*`) | **available** — escalation-only; bills by bandwidth + time | | `zenrows mcp` | MCP server config (remote + local) | **available** | | Zenrows CLI | this repo | available | @@ -156,7 +157,36 @@ An Extract task costs the same as a regular one (1 credit at base tier), so keys: `html` (the raw page) and `parsed` (the structured data) — validate a sample with `zenrows extract ` before running the full batch. -## 10. Browser Sessions +## 10. Crawl + +Zenrows **Crawl** (`https://api.zenrows.com/v1/crawls`) takes one start URL and +returns the URLs it finds behind it, optionally with each page's HTML. Use it +when you know the listing or section but not the item URLs. Crawl follows the +links on each page up to `--depth` hops and stays on the start URL's domain. +Without `--html` it returns URLs only. An account without Crawl access gets +`CRAWL_NOT_ENABLED` ("Crawl is not enabled for this account", HTTP 403 +`REQS008`). + +```bash +zenrows crawl start https://example.com/products/ --depth 1 \ + --max-items 20 --include /product/ --follow # start and wait for the end +zenrows crawl results --out urls.jsonl # every kept URL (paginated) +zenrows crawl start --depth 2 --html # also fetch each kept page's HTML +zenrows crawl content # one page's HTML +zenrows crawl results --download # NDJSON export (URLs + HTML) → .jsonl +zenrows crawl status # status, coverage, stop reason +zenrows crawl list # your crawls, newest first +zenrows crawl stop # stop a running crawl +``` + +`--depth` (link hops) is required; `--max-items` and `--max-pages` cap the +crawl (API default 10 each), and `--max-pages` bounds the cost, since each page +is one fetch. `--include` / `--exclude` are repeatable substring filters on the +URL. When the account has too many crawls running, a start gets +`CRAWL_QUOTA_EXCEEDED` and creates nothing; retry after `Retry-After`. Set +`ZENROWS_CRAWL_API_BASE` to point the CLI at another deployment. + +## 11. Browser Sessions Escalation only — **prefer `fetch`/`extract` for the vast majority of cases**; they cost less. Use the browser for logins, forms, and multi-step JS flows that @@ -170,7 +200,7 @@ bring-your-own Playwright/Puppeteer. close interactive sessions with `zenrows browser close`. On by default; opt out with `zenrows policy set allow_browser false`. -## 11. MCP +## 12. MCP ```bash zenrows mcp status @@ -188,7 +218,7 @@ both servers in `mcp.json` (no secrets) and ships MCP-native skills under `agent-plugin/skills/`. It is included in the published `@zenrows/cli` npm tarball (`node_modules/@zenrows/cli/agent-plugin`). -## 12. Plugins +## 13. Plugins ```bash zenrows plugin list @@ -201,7 +231,7 @@ MCP-native skills). See [`agent-plugin/README.md`](agent-plugin/README.md). Legacy per-client snippets from `zenrows plugin install` remain available. CLI-oriented skills stay under repo-root `skills/`. -## 13. Skills +## 14. Skills Agent-readable playbooks that teach agents how to choose primitives. @@ -213,28 +243,28 @@ zenrows skill validate zenrows The master skill `skills/zenrows/SKILL.md` teaches the full decision tree. -## 14. Templates +## 15. Templates ```bash zenrows template list zenrows template create protected-fetch-node --output ./my-project ``` -## 15. Workflows +## 16. Workflows ```bash zenrows workflow list zenrows workflow explain competitor-intelligence ``` -## 16. Recipes +## 17. Recipes ```bash zenrows recipe list zenrows recipe run fetch-protected-page ``` -## 17. Evals +## 18. Evals Reproducible, transparent benchmarks. **No competitor keys are bundled** and nothing is hardcoded to make Zenrows win — comparison evals require you to supply @@ -249,7 +279,7 @@ zenrows eval report protected-fetch-smoke Reports write `input.json`, `results.json`, `report.md`, `failures.jsonl`, `cost.json`, and `traces/` under `.zenrows/evals//`. -## 18. Security +## 19. Security - API keys are never printed or written into run artifacts/assets. - Secrets live in `.zenrows/secrets.json` (0600, gitignored); logs are redacted. @@ -259,12 +289,12 @@ Reports write `input.json`, `results.json`, `report.md`, `failures.jsonl`, - Destructive `uninstall` requires `--yes`. Browser is on by default; opt out with `zenrows policy set allow_browser false`. Experimental features are off by default. -## 19. Capability matrix +## 20. Capability matrix See [`registry/capabilities.json`](registry/capabilities.json). `zenrows status --json` emits it. -## 20. Contributing +## 21. Contributing The CLI is TypeScript with **zero runtime dependencies** (native `fetch`, `node:util` `parseArgs`, `node:test`). Node 20+ runs the published build; diff --git a/docs/capabilities.md b/docs/capabilities.md index fd7939b..2557ee7 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -22,5 +22,6 @@ Classification is based on the public Zenrows documentation: | `protected_fetch` | Fetch `GET https://api.zenrows.com/v1/` with `mode`, `js_render`, `premium_proxy`, `proxy_country`, `wait`/`wait_for`, `js_instructions`, `response_type`, `screenshot`, `original_status`, … | available | | `extract` | Extract `GET https://api.zenrows.com/v1/` via `extract=auto` (domain-gated open beta; CLI falls back to `autoparse`), plus `autoparse`, `css_extractor`, `outputs`, `response_type=markdown\|plaintext` | beta | | `batch` | Batch `https://async.api.zenrows.com/v1` (separate host, `X-API-Key` header) — open beta. Local JSONL spec validation + credit estimation work with no key. | beta | +| `crawl` | Crawl `https://api.zenrows.com/v1/crawls` (`X-API-Key` header; same host as Fetch/Extract). Follows the links on each page from one start URL up to `depth`, on the start URL's domain; returns URLs, and with `output_format: "html"` each page's HTML. Without Crawl access the API returns 403 `REQS008` → `CRAWL_NOT_ENABLED`. | available | | `browser` | Browser Sessions REST API `https://mcp.zenrows.com/browser/sessions/*` (Bearer; same backend as `@zenrows/mcp` `browser_*`); CDP via `zenrows browser connect`. Escalation-only (prefer fetch/extract); on by default, opt out via `policy.allow_browser`; bills by bandwidth + session time (15-min max) | available | | `mcp` | Hosted `https://mcp.zenrows.com/mcp` + local `npx -y @zenrows/mcp` | available | diff --git a/docs/contributing.md b/docs/contributing.md index bcd9989..cc3a519 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -37,6 +37,38 @@ npm run build # tsc → dist/ (for publishing) 4. Add a skill + recipe/eval and declare `requires_backend_capabilities`. 5. Add tests. Never fake backend behavior; return a normalized error instead. +## Running the Crawl e2e test + +`tests/e2e/crawl.e2e.test.ts` drives the built CLI (`zenrows crawl …`) against a +live Zenrows API: it starts one small crawl of the target you give it (depth 1, +3 items, 5 pages, HTML output), waits for it, reads its results, one page's +HTML and the NDJSON export, lists crawls, stops the ended crawl and checks a +missing id. It is not part of `npm test`, and it skips unless `ZENROWS_E2E=1`, +`ZENROWS_API_KEY` and `ZENROWS_E2E_CRAWL_URL` are all set. It bills a few +pages on the key's account. + +| Variable | Value | +| --- | --- | +| `ZENROWS_E2E` | `1` to run the test | +| `ZENROWS_API_KEY` | a key with Crawl access | +| `ZENROWS_E2E_CRAWL_URL` | the start URL to crawl, e.g. `https://example.com/products/` | +| `ZENROWS_E2E_CRAWL_INCLUDE` | optional include pattern, e.g. `/product/`; every result URL must contain it | +| `ZENROWS_CRAWL_API_BASE` | optional API base, default `https://api.zenrows.com/v1` | + +Read the key from a file rather than typing it, so it stays out of your shell +history: + +```bash +export ZENROWS_API_KEY="$(tr -d '\n' < path/to/api.key)" +ZENROWS_E2E=1 ZENROWS_E2E_CRAWL_URL=https://example.com/products/ \ + ZENROWS_E2E_CRAWL_INCLUDE=/product/ npm run test:e2e +``` + +To target a local or staging deployment, set `ZENROWS_CRAWL_API_BASE` to its +`/v1` base. When the account has too many crawls running, the test waits +`Retry-After` and retries the start for up to 5 minutes, so run concurrent e2e +runs on one account one at a time. + ## Rules - Never print or persist API keys. Redact secrets in logs and artifacts. diff --git a/package.json b/package.json index ef1173b..8b10701 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "zenrows": "node bin/zenrows.ts", "dev": "node bin/zenrows.ts", "test": "npm run build && node --test --import ./dist/tests/setup.js dist/tests/*.test.js", + "test:e2e": "npm run build && node --test dist/tests/e2e/*.test.js", "typecheck": "tsc --noEmit", "build": "rm -rf dist && tsc && chmod +x dist/bin/zenrows.js", "prepack": "npm run build", diff --git a/registry/capabilities.json b/registry/capabilities.json index 231d875..36c8a1f 100644 --- a/registry/capabilities.json +++ b/registry/capabilities.json @@ -28,6 +28,15 @@ "requiresAuth": true, "notes": "Open beta. Cloud subcommands (create/status/results/cancel/wait/retry-failed) call the Batch API; `zenrows batch estimate` works locally with no API key." }, + "crawl": { + "key": "crawl", + "label": "Crawl", + "status": "available", + "command": "zenrows crawl", + "backend": "Crawl — https://api.zenrows.com/v1/crawls (X-API-Key header; same host as Fetch/Extract)", + "requiresAuth": true, + "notes": "Follows the links on each page from one start URL up to depth, on the start URL's domain, and returns the URLs it keeps, optionally with each page's HTML (start/status/wait/results/content/list/stop). An account without Crawl access gets 403 REQS008 (CRAWL_NOT_ENABLED). Too many crawls running on the account gives 429 too_many_crawls (CRAWL_QUOTA_EXCEEDED); retry after Retry-After." + }, "browser": { "key": "browser", "label": "Browser Sessions", diff --git a/registry/skills.json b/registry/skills.json index 458cea6..2658341 100644 --- a/registry/skills.json +++ b/registry/skills.json @@ -44,6 +44,17 @@ "path": "skills/batch-jobs", "tags": ["batch", "scale", "jsonl"] }, + { + "name": "crawl", + "type": "skill", + "description": "Collect the URLs behind one start page with Crawl: start, wait, read results and page HTML, stop.", + "status": "available", + "requires_backend_capabilities": ["crawl"], + "requires_auth": true, + "version": "0.1.0", + "path": "skills/crawl", + "tags": ["crawl", "links", "urls"] + }, { "name": "interact-browser", "type": "skill", diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md new file mode 100644 index 0000000..0d8136e --- /dev/null +++ b/skills/crawl/SKILL.md @@ -0,0 +1,73 @@ +--- +name: crawl +description: Collect the URLs behind one start page with Crawl — start a crawl, wait for it, read the kept URLs and optionally each page's HTML, stop it. Use when the user has a listing or section URL but not the item URLs. +version: 0.1.0 +requires_backend_capabilities: [crawl] +--- + +# Crawl + +Give Crawl one start URL; it follows the links on that page (and, with a +higher depth, on the pages behind it) and returns the URLs it keeps. Use it +when you know where the items live (a category page, a blog index, a docs +section) but not the item URLs themselves. Once you have the URLs, read pages +with `zenrows fetch` / `zenrows extract`, or ask Crawl for each page's HTML +with `--html`. + +Crawl stays on the start URL's domain. An account without Crawl access gets +403 `REQS008` → `CRAWL_NOT_ENABLED` ("Crawl is not enabled for this account"); +do not retry it. + +## Start, wait, read + +``` +zenrows crawl start https://example.com/products/ --depth 1 \ + --max-items 20 --max-pages 25 --include /product/ --follow --json +zenrows crawl results --json # every kept URL (crawl must have ended) +zenrows crawl results --out urls.jsonl # same, as JSONL +``` + +- `--depth N` is required: link hops from the start URL. `1` = the start + page's links; `2` = also the links on each of those pages. +- `--max-items` (kept URLs) and `--max-pages` (fetched pages) cap the crawl; + both default to 10. `--max-pages` bounds the cost: each page is one fetch. +- `--include` / `--exclude` are substring matches on the normalized URL and + repeat: `--include /product/ --exclude ?add-to-cart`. +- `start` returns at once. `--follow` (or `zenrows crawl wait `) polls + until the crawl ends; on `--timeout` the crawl keeps running (it is not + stopped) and you get `CRAWL_TIMEOUT`. + +## Page HTML + +``` +zenrows crawl start --depth 1 --html --follow +zenrows crawl results --json # fetched results carry content_url +zenrows crawl content # one page's HTML (or pass the content_url) +zenrows crawl results --download --out pages.jsonl # every URL + HTML, NDJSON +``` + +A result's `content_status` is `pending`, `fetched` or `failed`; only +`fetched` results have content. + +## While it runs, and after + +``` +zenrows crawl status # status, coverage, stop_reason / error +zenrows crawl results --cursor --json # one page; keep polling with next_cursor +zenrows crawl list # your crawls, newest first +zenrows crawl stop # stop it; kept URLs stay readable +``` + +A crawl ends `completed` (nothing left, or a cap hit: `stop_reason` +`max_items` / `max_pages`), `stopped` (you stopped it) or `failed` (`error.code` +and `error.detail` say why). Only `failed` exits non-zero (`CRAWL_FAILED`). + +## Limits + +When the account has too many crawls running, a start gets 429 +`too_many_crawls` → `CRAWL_QUOTA_EXCEEDED`; nothing was created. Retry after +`Retry-After` (about 30s), or stop one of your crawls first. + +Rules: start small (`--depth 1`, low caps) and check the results before a +deep crawl. Mind the cost ([[cost-control]]). Read errors with +[[trace-debug]]. diff --git a/skills/trace-debug/SKILL.md b/skills/trace-debug/SKILL.md index 11d0b01..e786019 100644 --- a/skills/trace-debug/SKILL.md +++ b/skills/trace-debug/SKILL.md @@ -45,6 +45,14 @@ zenrows trace export # JSON for sharing auto-created Free plan. - `BATCH_FAILED` → the batch run ended `failed`; `failure_reason` and `failure_detail` in `zenrows batch status --json` say why. +- `CRAWL_NOT_ENABLED` → the API answered 403 `REQS008`: Crawl is not + enabled for this account. Do not retry; ask Zenrows support for access. +- `CRAWL_QUOTA_EXCEEDED` → 429 `too_many_crawls`: the account has too many + crawls running. Nothing was created; retry after `Retry-After` (see + `zenrows crawl list`). +- `CRAWL_FAILED` → the crawl ended `failed`; `error.code` in + `zenrows crawl status --json` says why, with `error.detail`. See + [[crawl]]. - `PARAM_CONFLICT_AUTO_MANUAL` → drop the managed flags or add `--manual`. - `CAPABILITY_UNAVAILABLE` → the primitive is not available on this account (e.g. beta/invite-only); use the local-spec path where offered. diff --git a/skills/zenrows/SKILL.md b/skills/zenrows/SKILL.md index d328eb8..58f87df 100644 --- a/skills/zenrows/SKILL.md +++ b/skills/zenrows/SKILL.md @@ -34,6 +34,10 @@ If the user has a known URL and wants page content: If the user has a known URL and wants structured data: → Use Extract. (zenrows extract | --autoparse | --css) +If the user has a start page (listing, index, section) but not the item URLs: + → Use Crawl. (zenrows crawl start --depth 1 --follow, + then `zenrows crawl results `) + If the user has many URLs (bulk): → Use Batch. (validate one page with fetch/extract first, then `zenrows batch create `) @@ -70,6 +74,7 @@ Run `zenrows status` for the live capability matrix. As of this toolkit: | Protected Fetch | `zenrows fetch` | available (`GET /v1/`) | | Extract (extract=auto / Autoparse/CSS/Markdown) | `zenrows extract` | beta (same `/v1/`; extract=auto falls back to autoparse) | | Batch | `zenrows batch` | beta — runs cloud jobs (create/status/results); cheapest path at scale. Estimate specs locally with no key | +| Crawl | `zenrows crawl` | available — collect the URLs behind one start page, optionally with each page's HTML | | Browser | `zenrows browser` | available (Browser Sessions REST API / MCP) | | MCP | `zenrows mcp` | available (remote + local server) | diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts new file mode 100644 index 0000000..a21775b --- /dev/null +++ b/src/cli/commands/crawl.ts @@ -0,0 +1,473 @@ +/** + * `zenrows crawl` — Crawl API. + * + * Give Crawl one start URL and read back the URLs it finds behind it, optionally + * with each page's HTML. A crawl is an async job: `start` returns at once + * (or polls with `--follow`), then `status` / `wait` / `results` / `content` + * read it and `stop` ends it. An account without Crawl access gets 403 REQS008 + * → CRAWL_NOT_ENABLED. + */ +import { log } from "../../core/logger.ts"; +import { requireApiKey } from "../../core/auth.ts"; +import { ensureApiKey } from "../../core/ensure-key.ts"; +import { assertUsable } from "../../core/capabilities.ts"; +import { assertDomainAllowed, assertWithinLimits, loadPolicy } from "../../core/policy.ts"; +import { newRunId, writeRun } from "../../core/artifacts.ts"; +import { + contentIdOf, + createCrawl, + downloadCrawl, + getCrawl, + getCrawlContent, + listAllResults, + listCrawls, + stopCrawl, + waitForCrawl, + type Crawl, + type CrawlStop, + type CreateCrawlParams, +} from "../../core/crawl-api.ts"; +import { asNumber, asString, parse, type Command, type RunContext } from "../command.ts"; +import { ToolkitError } from "../../core/errors.ts"; +import { printError, writeOut } from "../output.ts"; +import { normalizeTimeout } from "./fetch.ts"; + +/** The API's default for `max_pages` (and `max_items`) when the flag is unset. */ +const DEFAULT_MAX_PAGES = 10; +/** Largest `depth` / `max_items` / `max_pages` the API takes. */ +const MAX_LIMIT = 100_000; + +export const crawl: Command = { + name: "crawl", + summary: "Crawl a site from one start URL and collect the URLs behind it.", + usage: + "zenrows crawl --depth N|status |results |content |list|stop |wait >", + help: [ + "Cloud (needs a key with Crawl access):", + " start --depth [flags] start a crawl from one URL (returns at once)", + " --depth link hops to follow from the start URL (1-100000, required)", + " --max-items stop after keeping n URLs (API default 10)", + " --max-pages stop after fetching n pages (API default 10; bounds cost)", + " --include keep only URLs containing this substring (repeatable)", + " --exclude drop URLs containing this substring (repeatable)", + " --html also fetch each kept URL's page HTML (read with `content`)", + " --follow poll until the crawl ends (alias: --wait)", + " --timeout with --follow: give up waiting after ms (default 600000)", + " --idempotency-key make a retried start create no second crawl", + " --no-signup do not auto-create a Free plan account if no key exists", + " status show status, coverage, stop reason / error", + " wait [--timeout ] poll until the crawl ends (it is not stopped on timeout)", + " results all kept URLs (crawl must have ended); paginated", + " --out write results as JSONL instead of printing", + " --cursor [--limit ] read one page only (works while running; prints next_cursor)", + " --download fetch the NDJSON export (with page HTML for --html crawls)", + " into --out, or .jsonl", + " content print one kept URL's page (content_id or content_url)", + " --out write it to a file instead", + " list [--limit ] [--cursor c] list your crawls, newest first", + " stop stop a running crawl (idempotent)", + " --json print structured output", + ].join("\n"), + async run(argv: string[], ctx: RunContext): Promise { + const [sub, ...rest] = argv; + try { + switch (sub) { + case "start": + return await startCmd(rest, ctx); + case "status": + return await statusCmd(rest, ctx); + case "wait": + return await waitCmd(rest, ctx); + case "results": + return await resultsCmd(rest, ctx); + case "content": + return await contentCmd(rest, ctx); + case "list": + return await listCmd(rest, ctx); + case "stop": + return await stopCmd(rest, ctx); + default: + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `Unknown crawl subcommand: ${sub ?? "(none)"}`, + likely_cause: "Subcommand not recognized.", + next_action: + "Use start --depth N | status | wait | results | content | list | stop .", + suggested_commands: ["zenrows crawl --help"], + }); + } + } catch (err) { + printError(err, ctx.json); + return 1; + } + }, +}; + +async function startCmd(rest: string[], ctx: RunContext): Promise { + const { values, positionals } = parse(rest, { + depth: { type: "string" }, + "max-items": { type: "string" }, + "max-pages": { type: "string" }, + include: { type: "string", multiple: true }, + exclude: { type: "string", multiple: true }, + html: { type: "boolean" }, + follow: { type: "boolean" }, + wait: { type: "boolean" }, // alias for --follow + timeout: { type: "string" }, + "idempotency-key": { type: "string" }, + "no-signup": { type: "boolean" }, + json: { type: "boolean" }, + }); + const json = ctx.json || values.json === true; + const follow = values.follow === true || values.wait === true; + const url = positionals[0]; + if (!url) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: "Provide the start URL.", + likely_cause: "No positional was given.", + next_action: "Usage: zenrows crawl start --depth 1", + suggested_commands: ["zenrows crawl start https://example.com/ --depth 1"], + }); + } + const depth = limitFlag("depth", values.depth); + if (depth === undefined) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: "--depth is required.", + likely_cause: "Crawl has no default depth yet; every crawl states how many link hops to follow.", + next_action: "Pass --depth 1 to collect the start page's links, --depth 2 to also open each of those.", + suggested_commands: [`zenrows crawl start ${url} --depth 1`], + }); + } + const params: CreateCrawlParams = { url, depth }; + const maxItems = limitFlag("max-items", values["max-items"]); + if (maxItems !== undefined) params.max_items = maxItems; + const maxPages = limitFlag("max-pages", values["max-pages"]); + if (maxPages !== undefined) params.max_pages = maxPages; + const include = patterns(values.include); + if (include.length) params.include_patterns = include; + const exclude = patterns(values.exclude); + if (exclude.length) params.exclude_patterns = exclude; + if (values.html === true) params.output_format = "html"; + const timeoutMs = normalizeTimeout(values.timeout); + const idempotencyKey = asString(values["idempotency-key"]); + + assertUsable("crawl"); + + // Pre-flight governance, all local: the start URL must pass the domain policy, + // and max_pages (each page is one fetch) must fit the per-run page cap. + const policy = loadPolicy(); + assertDomainAllowed(url, policy); + assertWithinLimits({ pages: maxPages ?? DEFAULT_MAX_PAGES }, policy, "crawl"); + + const apiKey = await ensureApiKey(values["no-signup"] ? { ...policy, auto_signup: false } : policy, { + onProvision: (a) => { + log.info("No API key found — created a Zenrows Free plan account for you."); + log.dim(`Claim it anytime (keeps your usage): ${a.claimUrl}`); + }, + }); + + const runId = newRunId(); + const startedAt = new Date().toISOString(); + log.step(`Starting crawl of ${url} (depth ${depth})…`); + try { + const created = await createCrawl(params, { apiKey, idempotencyKey }); + const finished: Crawl = follow ? await waitForCrawl(created.crawl_id, { apiKey, timeoutMs }) : created; + const failure = crawlFailure(finished); + const runDir = writeRun({ + runId, + command: "zenrows crawl start", + capability: "crawl", + url, + startedAt, + finishedAt: new Date().toISOString(), + status: failure ? "error" : "ok", + request: { ...params }, + result: { crawlId: created.crawl_id, status: finished.status, coverage: finished.coverage }, + ...(failure ? { error: failure.toJSON() } : {}), + }); + const code = printCrawl(finished, json, `Started crawl ${created.crawl_id}`); + if (!json) { + if (runDir) log.dim(` artifact: ${runDir}`); + if (!follow) log.dim(` next: zenrows crawl wait ${created.crawl_id}`); + } + return code; + } catch (err) { + writeRun({ + runId, + command: "zenrows crawl start", + capability: "crawl", + url, + startedAt, + finishedAt: new Date().toISOString(), + status: "error", + request: { ...params }, + error: err instanceof ToolkitError ? err.toJSON() : { message: String(err) }, + }); + throw err; + } +} + +async function statusCmd(rest: string[], ctx: RunContext): Promise { + const { values, positionals } = parse(rest, { json: { type: "boolean" } }); + const id = requireId(positionals[0], "status"); + assertUsable("crawl"); + const apiKey = requireApiKey(); + const c = await getCrawl(id, { apiKey, limit: 1 }); + return printCrawl(c, ctx.json || values.json === true, `Crawl ${id}`); +} + +async function waitCmd(rest: string[], ctx: RunContext): Promise { + const { values, positionals } = parse(rest, { timeout: { type: "string" }, json: { type: "boolean" } }); + const id = requireId(positionals[0], "wait"); + const timeoutMs = normalizeTimeout(values.timeout); + assertUsable("crawl"); + const apiKey = requireApiKey(); + const c = await waitForCrawl(id, { apiKey, timeoutMs }); + return printCrawl(c, ctx.json || values.json === true, `Crawl ${id} finished`); +} + +async function resultsCmd(rest: string[], ctx: RunContext): Promise { + const { values, positionals } = parse(rest, { + out: { type: "string" }, + cursor: { type: "string" }, + limit: { type: "string" }, + download: { type: "boolean" }, + json: { type: "boolean" }, + }); + const json = ctx.json || values.json === true; + const id = requireId(positionals[0], "results"); + const outFile = asString(values.out); + const cursor = asString(values.cursor); + const limit = pageLimit(values.limit, 10_000); + assertUsable("crawl"); + const apiKey = requireApiKey(); + + if (values.download === true) { + const file = outFile ?? `${id.replace(/[^A-Za-z0-9._-]+/g, "_")}.jsonl`; + const { crawlStatus, ndjson } = await downloadCrawl(id, { apiKey }); + writeOut(file, ndjson); + const lines = ndjson.split("\n").filter((l) => l.trim()).length; + const partial = crawlStatus === "running"; + if (json) { + log.out(JSON.stringify({ ok: true, crawlId: id, crawlStatus, partial, count: lines, file }, null, 2)); + } else { + log.success(`Downloaded ${lines} result(s) → ${file}`); + if (partial) log.warn(" The crawl is still running: this file is partial. Download again once it ends."); + } + return 0; + } + + // One page, as the API returns it — the way to follow a crawl while it runs. + if (cursor !== undefined || limit !== undefined) { + const page = await getCrawl(id, { apiKey, cursor, limit }); + const body = { ok: true, crawlId: id, status: page.status, count: page.results.length, results: page.results, next_cursor: page.next_cursor }; + if (outFile) { + writeOut(outFile, toJsonl(page.results)); + log.success(`Wrote ${page.results.length} result(s) → ${outFile}`); + log.info(`next_cursor: ${page.next_cursor ?? "null (last page)"}`); + } else if (json) { + log.out(JSON.stringify(body, null, 2)); + } else { + log.info(`${page.results.length} result(s) for crawl ${id} (status: ${page.status}):`); + log.out(toJsonl(page.results).trimEnd()); + log.info(`next_cursor: ${page.next_cursor ?? "null (last page)"}`); + } + return 0; + } + + const { crawl: c, results } = await listAllResults(id, { apiKey }); + if (outFile) { + writeOut(outFile, toJsonl(results)); + log.success(`Wrote ${results.length} result(s) → ${outFile}`); + } else if (json) { + log.out(JSON.stringify({ ok: true, crawlId: id, status: c.status, count: results.length, results }, null, 2)); + } else { + log.info(`${results.length} result(s) for crawl ${id} (status: ${c.status}):`); + log.out(toJsonl(results).trimEnd()); + } + return 0; +} + +async function contentCmd(rest: string[], ctx: RunContext): Promise { + const { values, positionals } = parse(rest, { out: { type: "string" }, json: { type: "boolean" } }); + const json = ctx.json || values.json === true; + const id = requireId(positionals[0], "content"); + const ref = positionals[1]; + // Accept the content id or the result's whole content_url. + const contentId = ref ? (contentIdOf({ url: "", content_url: ref }) ?? ref) : undefined; + if (!contentId) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: "Missing content id.", + likely_cause: "No positional was provided.", + next_action: + "Pass the id from a fetched result's content_url (the last path segment), e.g. `zenrows crawl content ct_…`. Only crawls started with --html have content.", + suggested_commands: [`zenrows crawl results ${id}`], + }); + } + assertUsable("crawl"); + const apiKey = requireApiKey(); + const { contentType, body } = await getCrawlContent(id, contentId, { apiKey }); + const outFile = asString(values.out); + if (outFile) { + writeOut(outFile, body); + if (json) { + log.out(JSON.stringify({ ok: true, crawlId: id, contentId, contentType, bytes: Buffer.byteLength(body), file: outFile }, null, 2)); + } else { + log.success(`Wrote ${Buffer.byteLength(body)} bytes (${contentType || "unknown type"}) → ${outFile}`); + } + } else if (json) { + log.out(JSON.stringify({ ok: true, crawlId: id, contentId, contentType, bytes: Buffer.byteLength(body), content: body }, null, 2)); + } else { + log.out(body); + } + return 0; +} + +async function listCmd(rest: string[], ctx: RunContext): Promise { + const { values } = parse(rest, { limit: { type: "string" }, cursor: { type: "string" }, json: { type: "boolean" } }); + const json = ctx.json || values.json === true; + const limit = pageLimit(values.limit, 100); + const cursor = asString(values.cursor); + assertUsable("crawl"); + const apiKey = requireApiKey(); + const page = await listCrawls({ apiKey, limit, cursor }); + const crawls = page.crawls ?? []; + if (json) { + log.out(JSON.stringify({ ok: true, count: crawls.length, crawls, next_cursor: page.next_cursor ?? null }, null, 2)); + return 0; + } + if (!crawls.length) log.info("No crawls."); + for (const c of crawls) { + const cov = c.coverage; + log.out( + `${c.crawl_id} ${c.status.padEnd(9)} ${c.created_at} ${cov ? `${cov.items_found} items/${cov.pages_fetched} pages` : ""} ${c.url}`, + ); + } + if (page.next_cursor) log.dim(` more: zenrows crawl list --cursor ${page.next_cursor}`); + return 0; +} + +async function stopCmd(rest: string[], ctx: RunContext): Promise { + const { values, positionals } = parse(rest, { json: { type: "boolean" } }); + const id = requireId(positionals[0], "stop"); + assertUsable("crawl"); + const apiKey = requireApiKey(); + const s = await stopCrawl(id, { apiKey }); + return printCrawl(s, ctx.json || values.json === true, `Stopped crawl ${id}`); +} + +/** + * The error for a crawl that ended `failed`, or null for any other state. The + * crawl's own `error.code` picks the next action. + */ +export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null { + if (c.status !== "failed") return null; + const code = c.error?.code; + const why = [code, c.error?.detail].filter(Boolean).join(": "); + const next: Record = { + insufficient_credits: "The account ran out of credits. Check `zenrows usage`, add credits, then start a new crawl.", + seed_unreachable: "The start URL could not be fetched (the site may be blocking it). Check it with `zenrows fetch `.", + no_items_found: + "Nothing matched. Loosen --include / --exclude, raise --depth, or check the start page with `zenrows fetch `.", + }; + return new ToolkitError({ + code: "CRAWL_FAILED", + message: `Crawl ${c.crawl_id} failed.`, + likely_cause: why || "The crawl ended in status failed without a reason.", + next_action: (code && next[code]) || "Read the error, fix the cause, then start a new crawl.", + suggested_commands: [`zenrows crawl status ${c.crawl_id}`], + }); +} + +/** + * Print a crawl (or a stop answer), structured under --json, and return the exit + * code: 1 when it ended `failed`, else 0. `stopped` is a deliberate outcome, so + * it exits 0 but prints as a warning. + */ +function printCrawl(c: Crawl | CrawlStop, json: boolean, headline: string): number { + const err = crawlFailure(c); + const crawlFields = { ...c } as Record; + // A status read asks for one result to keep polls cheap; that page is not "the results". + delete crawlFields.results; + delete crawlFields.next_cursor; + if (json) { + log.out( + JSON.stringify( + { + ok: !err, + crawlId: c.crawl_id, + status: c.status, + ...(c.stop_reason ? { stop_reason: c.stop_reason } : {}), + crawl: crawlFields, + ...(err ? { error: err.toJSON() } : {}), + }, + null, + 2, + ), + ); + return err ? 1 : 0; + } + const line = `${headline} · status: ${c.status ?? "unknown"}${c.stop_reason ? ` (${c.stop_reason})` : ""}`; + if (err) log.error(line); + else if (c.status === "stopped") log.warn(line); + else log.success(line); + const cov = (c as Crawl).coverage; + if (cov) { + log.info(` ${cov.items_found} item(s) found · ${cov.pages_fetched} page(s) fetched · ${cov.pages_failed} failed`); + } + if (err) printError(err, false); + return err ? 1 : 0; +} + +/** A positive integer flag up to the API's 100,000 ceiling; undefined when unset. */ +function limitFlag(name: string, v: unknown): number | undefined { + if (v === undefined) return undefined; + const n = asNumber(v); + if (n === undefined || !Number.isInteger(n) || n < 1 || n > MAX_LIMIT) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `Invalid --${name} value '${String(v)}'.`, + likely_cause: `--${name} takes a whole number from 1 to ${MAX_LIMIT}.`, + next_action: `Pass e.g. --${name} ${name === "depth" ? 1 : 10}.`, + }); + } + return n; +} + +/** `--limit` for a paged read: a whole number from 1 to `max`. */ +function pageLimit(v: unknown, max: number): number | undefined { + if (v === undefined) return undefined; + const n = asNumber(v); + if (n === undefined || !Number.isInteger(n) || n < 1 || n > max) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `Invalid --limit value '${String(v)}'.`, + likely_cause: `--limit takes a whole number from 1 to ${max}.`, + next_action: `Pass e.g. --limit ${Math.min(max, 100)}.`, + }); + } + return n; +} + +function patterns(v: unknown): string[] { + const list = Array.isArray(v) ? v : typeof v === "string" ? [v] : []; + return list.filter((p): p is string => typeof p === "string" && p.length > 0); +} + +function toJsonl(rows: unknown[]): string { + return rows.map((r) => JSON.stringify(r)).join("\n") + (rows.length ? "\n" : ""); +} + +function requireId(id: string | undefined, sub: string): string { + if (id) return id; + throw new ToolkitError({ + code: "INVALID_USAGE", + message: "Missing crawl id.", + likely_cause: "No positional was provided.", + next_action: `Usage: zenrows crawl ${sub} . List your crawls with \`zenrows crawl list\`.`, + suggested_commands: ["zenrows crawl list"], + }); +} diff --git a/src/cli/index.ts b/src/cli/index.ts index e8e2bdc..2185bcb 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -27,6 +27,7 @@ import { policy } from "./commands/policy.ts"; import { fetch_ } from "./commands/fetch.ts"; import { extract } from "./commands/extract.ts"; import { batch } from "./commands/batch.ts"; +import { crawl } from "./commands/crawl.ts"; import { browser } from "./commands/browser.ts"; import { mcp } from "./commands/mcp.ts"; import { plugin } from "./commands/plugin.ts"; @@ -43,7 +44,7 @@ const evalCmd = makeAssetCommand("eval", "Reproducible benchmark/test suites."); /** Ordered groups for help output. */ const GROUPS: Array<{ title: string; commands: Command[] }> = [ { title: "Setup", commands: [init, signup, account, login, logout, status, usage, update, uninstall, config, policy] }, - { title: "Primitives", commands: [fetch_, extract, batch, browser] }, + { title: "Primitives", commands: [fetch_, extract, batch, crawl, browser] }, { title: "Distribution", commands: [mcp, plugin, skill, template, workflow, recipe, evalCmd, trace] }, ]; diff --git a/src/core/crawl-api.ts b/src/core/crawl-api.ts new file mode 100644 index 0000000..f434f51 --- /dev/null +++ b/src/core/crawl-api.ts @@ -0,0 +1,459 @@ +/** + * Client for the Zenrows Crawl API. + * + * Crawl lives on the same host as Fetch and Extract (`https://api.zenrows.com/v1`) + * under `/crawls`. Auth is the `X-API-Key` header, never the `apikey` query + * param: Crawl answers any query parameter it does not know with 400. Bodies are + * JSON; errors come back as `application/problem+json` (RFC 9457) and we branch + * on `status` + `code` only. A crawl is an async job: create it, then poll it + * until its status leaves `running`. `fetchImpl` / `sleepImpl` are injectable + * for tests. + * + * `output_format` takes `html` only (absent = URLs only). Response fields we do + * not model pass through untouched. + */ +import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "./errors.ts"; +import { readAccount } from "./agent-account.ts"; +import { registerSecret } from "./logger.ts"; +import { CLI_USER_AGENT, loadConfig } from "./config.ts"; +import { agentClientHeader } from "./agent-client.ts"; + +/** Production Crawl API base (no trailing slash): the Fetch/Extract host. */ +export const DEFAULT_CRAWL_API_BASE = "https://api.zenrows.com/v1"; +/** Env var to override the Crawl API base (local/staging testing). */ +export const CRAWL_API_BASE_ENV = "ZENROWS_CRAWL_API_BASE"; + +/** + * Resolve the Crawl API base, trimmed of trailing `/`. `ZENROWS_CRAWL_API_BASE` + * wins; otherwise Crawl follows the Fetch/Extract `apiBase` (config or + * `ZENROWS_API_BASE`), since both live on the same API host. + */ +export function crawlBase(): string { + const env = process.env[CRAWL_API_BASE_ENV]; + const base = env && env.trim() ? env.trim() : loadConfig().apiBase || DEFAULT_CRAWL_API_BASE; + return base.replace(/\/+$/, ""); +} + +export interface CrawlCoverage { + pages_fetched: number; + pages_failed: number; + items_found: number; +} + +/** Why a crawl ended `failed` (part of the crawl, not an error response). */ +export interface CrawlRunError { + code: string; + detail: string; +} + +/** One crawl. Enum fields are extensible: expect values not listed here. */ +export interface Crawl { + crawl_id: string; + /** running | completed | stopped | failed. Terminal = anything but `running`. */ + status: string; + url: string; + depth: number; + max_items: number; + max_pages: number; + coverage: CrawlCoverage; + created_at: string; + /** max_items | max_pages | user. Absent when nothing was left to open. */ + stop_reason?: string; + error?: CrawlRunError; + include_patterns?: string[]; + exclude_patterns?: string[]; + output_format?: string; + duplicates_removed?: number; + finished_at?: string; + [k: string]: unknown; +} + +/** One URL the crawl kept. */ +export interface CrawlResult { + url: string; + /** pending | fetched | failed — only when the crawl has an output format. */ + content_status?: string; + /** e.g. `/v1/crawls/c_x/contents/ct_y`, present once fetched. */ + content_url?: string; + [k: string]: unknown; +} + +/** `GET /crawls/{id}`: the crawl plus one page of results. */ +export interface CrawlWithResults extends Crawl { + results: CrawlResult[]; + /** Never null while the crawl runs; null once it ended and this page is the last. */ + next_cursor: string | null; +} + +export interface CrawlList { + crawls: Crawl[]; + /** Absent on the last page. */ + next_cursor?: string; +} + +/** `POST /crawls/{id}/stop`: where the crawl stands, without counts. */ +export interface CrawlStop { + crawl_id: string; + status: string; + stop_reason?: string; + error?: CrawlRunError; + finished_at?: string; + [k: string]: unknown; +} + +/** Create body. Only the fields the caller set are sent. */ +export interface CreateCrawlParams { + url: string; + depth: number; + max_items?: number; + max_pages?: number; + include_patterns?: string[]; + exclude_patterns?: string[]; + /** `html` returns each kept URL's page; absent = URLs only. */ + output_format?: "html"; +} + +/** RFC 9457 problem body. */ +interface ProblemJson { + type?: string; + title?: string; + status?: number; + detail?: string; + code?: string; +} + +/** The API's "Crawl is not enabled for this account" code (403). */ +export const CRAWL_NOT_ENABLED_CODE = "REQS008"; + +/** A crawl stops progressing once its status is anything but `running`. */ +export function isTerminal(status: string | undefined): boolean { + return !!status && status !== "running"; +} + +interface RequestOpts { + apiKey: string; + body?: unknown; + query?: Record; + headers?: Record; + accept?: string; + fetchImpl?: typeof fetch; + timeoutMs?: number; +} + +interface RawResponse { + status: number; + headers: Headers; + text: string; +} + +/** + * Perform a Crawl API request and return the raw body. Sets `X-API-Key`, adds a + * JSON content-type when a body is present, and on a non-2xx parses the + * problem+json body into a normalized ToolkitError. + */ +async function crawlFetch(method: string, path: string, opts: RequestOpts): Promise { + registerSecret(opts.apiKey); + const url = new URL(crawlBase() + path); + for (const [k, v] of Object.entries(opts.query ?? {})) { + if (v !== undefined && v !== null) url.searchParams.set(k, String(v)); + } + const doFetch = opts.fetchImpl ?? fetch; + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), opts.timeoutMs ?? 60_000); + + const headers: Record = { + "X-API-Key": opts.apiKey, + Accept: opts.accept ?? "application/json", + "User-Agent": CLI_USER_AGENT, + ...agentClientHeader(), + ...(opts.headers ?? {}), + }; + if (opts.body !== undefined) headers["Content-Type"] = "application/json"; + + let res: Response; + let text: string; + try { + res = await doFetch(url.toString(), { + method, + headers, + body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined, + signal: controller.signal, + }); + text = await res.text(); + } catch (err) { + throw new ToolkitError({ + code: "BACKEND_UNAVAILABLE", + message: "Could not reach the Zenrows Crawl API.", + likely_cause: err instanceof Error ? err.message : String(err), + next_action: + "Check connectivity and retry. Override the host with ZENROWS_CRAWL_API_BASE if you are testing against staging.", + suggested_commands: ["zenrows status"], + }); + } finally { + clearTimeout(timeout); + } + + if (res.status < 200 || res.status >= 300) { + throw problemToError(res.status, text, method, path, res.headers.get("retry-after")); + } + return { status: res.status, headers: res.headers, text }; +} + +/** `crawlFetch` + JSON parse. */ +async function crawlJson(method: string, path: string, opts: RequestOpts): Promise { + const { text } = await crawlFetch(method, path, opts); + try { + return JSON.parse(text) as T; + } catch { + throw new ToolkitError({ + code: "CRAWL_FAILED", + message: "The Crawl API response was not valid JSON.", + likely_cause: text.slice(0, 240) || "Empty response body.", + next_action: "Retry, or check the crawl with `zenrows crawl status `.", + }); + } +} + +/** Map a problem+json body + HTTP status to a normalized ToolkitError. */ +function problemToError( + status: number, + body: string, + method: string, + path: string, + retryAfter: string | null, +): ToolkitError { + let problem: ProblemJson = {}; + try { + problem = JSON.parse(body) as ProblemJson; + } catch { + // non-JSON error body — fall through with an empty problem + } + const serverCode = problem.code ?? ""; + const detail = problem.detail || problem.title || body.slice(0, 240) || `HTTP ${status}`; + const cause = `HTTP ${status}${serverCode ? ` (${serverCode})` : ""} for ${method} ${path}: ${detail}`; + + if (status === 403 && serverCode === CRAWL_NOT_ENABLED_CODE) { + return new ToolkitError({ + code: "CRAWL_NOT_ENABLED", + message: "Crawl is not enabled for this account.", + likely_cause: `${cause}. Crawl is not enabled for this account yet.`, + next_action: + "Ask Zenrows support to enable Crawl for this account. Meanwhile fetch known URLs with `zenrows fetch`.", + suggested_commands: ["zenrows fetch "], + }); + } + if (status === 401) { + return new ToolkitError({ + code: "AUTH_INVALID", + message: "Zenrows rejected the API key for the Crawl API.", + likely_cause: cause, + next_action: "Re-check your key and log in again.", + suggested_commands: ["zenrows login --api-key "], + }); + } + if (status === 404) { + const content = serverCode === "content_not_found"; + return new ToolkitError({ + code: "CRAWL_NOT_FOUND", + message: content ? "Crawl content not found." : "Crawl not found.", + likely_cause: content + ? `${cause}. The page was not fetched (yet), its fetch failed, or the crawl ran without --html.` + : `${cause}. The id may be wrong or the crawl is not owned by this account.`, + next_action: content + ? "Read the result's content_status with `zenrows crawl results `; only `fetched` results have content." + : "Check the crawl id, or list your crawls with `zenrows crawl list`.", + suggested_commands: ["zenrows crawl list"], + }); + } + if (status === 429) { + const wait = retryAfter && /^\d+$/.test(retryAfter.trim()) ? ` Retry after ${retryAfter.trim()}s.` : ""; + return new ToolkitError({ + code: "CRAWL_QUOTA_EXCEEDED", + message: "Too many crawls running.", + likely_cause: `${cause}. The account has too many crawls running.${wait}`, + next_action: + "Retry after Retry-After, or stop one of your crawls with `zenrows crawl stop ` first. Nothing was created.", + suggested_commands: ["zenrows crawl list"], + }); + } + if (status === 402 && isKeyCapReached(serverCode)) { + return keyCapReached(`${method} ${path}`, { status: 402, detail: problem.detail || problem.title || undefined }); + } + if (status === 402) { + const acct = readAccount(); + return quotaExhausted(`${method} ${path}`, acct?.unclaimed ? acct.claimUrl : undefined, { + status: 402, + detail: problem.detail || problem.title || undefined, + }); + } + if (status === 400 || status === 409 || status === 422) { + return new ToolkitError({ + code: "CRAWL_INVALID", + message: `Crawl rejected the request (HTTP ${status}).`, + likely_cause: `${cause}.`, + next_action: + status === 409 + ? "A request with the same Idempotency-Key is still in flight; retry once it has finished." + : "Fix the reported parameter and retry. See `zenrows crawl --help` for the accepted flags and ranges.", + }); + } + return new ToolkitError({ + code: "CRAWL_FAILED", + message: `Crawl request failed (HTTP ${status}).`, + likely_cause: `${cause}.`, + next_action: + status >= 500 ? "Retry with a short backoff." : "Fix the reported problem and retry.", + }); +} + +interface CallOpts { + apiKey: string; + fetchImpl?: typeof fetch; + timeoutMs?: number; +} + +/** Start a crawl (202). Sends only the fields set in `params`. */ +export function createCrawl(params: CreateCrawlParams, opts: CallOpts & { idempotencyKey?: string }): Promise { + const body: Record = { url: params.url, depth: params.depth }; + if (params.max_items !== undefined) body.max_items = params.max_items; + if (params.max_pages !== undefined) body.max_pages = params.max_pages; + if (params.include_patterns?.length) body.include_patterns = params.include_patterns; + if (params.exclude_patterns?.length) body.exclude_patterns = params.exclude_patterns; + if (params.output_format !== undefined) body.output_format = params.output_format; + return crawlJson("POST", "/crawls", { + apiKey: opts.apiKey, + body, + headers: opts.idempotencyKey ? { "Idempotency-Key": opts.idempotencyKey } : undefined, + fetchImpl: opts.fetchImpl, + timeoutMs: opts.timeoutMs, + }); +} + +/** Read a crawl and one page of its results. */ +export function getCrawl(id: string, opts: CallOpts & { cursor?: string; limit?: number }): Promise { + return crawlJson("GET", `/crawls/${encodeURIComponent(id)}`, { + apiKey: opts.apiKey, + query: { cursor: opts.cursor, limit: opts.limit }, + fetchImpl: opts.fetchImpl, + timeoutMs: opts.timeoutMs, + }); +} + +/** One page of the account's crawls, newest first. */ +export function listCrawls(opts: CallOpts & { cursor?: string; limit?: number }): Promise { + return crawlJson("GET", "/crawls", { + apiKey: opts.apiKey, + query: { cursor: opts.cursor, limit: opts.limit }, + fetchImpl: opts.fetchImpl, + timeoutMs: opts.timeoutMs, + }); +} + +/** Stop a running crawl. Idempotent: an ended crawl answers as it ended. */ +export function stopCrawl(id: string, opts: CallOpts): Promise { + return crawlJson("POST", `/crawls/${encodeURIComponent(id)}/stop`, { + apiKey: opts.apiKey, + fetchImpl: opts.fetchImpl, + timeoutMs: opts.timeoutMs, + }); +} + +/** The page of one kept URL (HTML for `output_format: html`). */ +export async function getCrawlContent( + id: string, + contentId: string, + opts: CallOpts, +): Promise<{ contentType: string; body: string }> { + const res = await crawlFetch("GET", `/crawls/${encodeURIComponent(id)}/contents/${encodeURIComponent(contentId)}`, { + apiKey: opts.apiKey, + accept: "text/html, application/json;q=0.9, application/problem+json;q=0.8", + fetchImpl: opts.fetchImpl, + timeoutMs: opts.timeoutMs, + }); + return { contentType: res.headers.get("content-type") ?? "", body: res.text }; +} + +/** + * Every result in one NDJSON file (`{url, content_status?, content?}` per line). + * `crawlStatus` is the `X-Crawl-Status` header: `running` means the file is partial. + */ +export async function downloadCrawl(id: string, opts: CallOpts): Promise<{ crawlStatus?: string; ndjson: string }> { + const res = await crawlFetch("GET", `/crawls/${encodeURIComponent(id)}/download`, { + apiKey: opts.apiKey, + accept: "application/x-ndjson, application/problem+json;q=0.8", + fetchImpl: opts.fetchImpl, + timeoutMs: opts.timeoutMs ?? 300_000, + }); + return { crawlStatus: res.headers.get("x-crawl-status") ?? undefined, ndjson: res.text }; +} + +/** + * The content id of a fetched result: the last segment of its `content_url`. + * Undefined when the result has no content yet. + */ +export function contentIdOf(result: CrawlResult): string | undefined { + if (!result.content_url) return undefined; + const last = result.content_url.split("?")[0]!.replace(/\/+$/, "").split("/").pop(); + return last ? decodeURIComponent(last) : undefined; +} + +/** + * Read every result, following `next_cursor` until it is null. Only call this + * on an ended crawl: while a crawl runs `next_cursor` is never null, so this + * refuses a `running` crawl instead of looping forever (wait first). + */ +export async function listAllResults( + id: string, + opts: CallOpts & { pageSize?: number }, +): Promise<{ crawl: CrawlWithResults; results: CrawlResult[] }> { + const results: CrawlResult[] = []; + let cursor: string | undefined; + let first: CrawlWithResults | undefined; + for (;;) { + const page = await getCrawl(id, { ...opts, cursor, limit: opts.pageSize }); + if (!isTerminal(page.status)) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `Crawl ${id} is still running.`, + likely_cause: "Reading all results needs an ended crawl: while it runs, more results keep arriving.", + next_action: "Wait for it first (`zenrows crawl wait `), or stop it with `zenrows crawl stop `.", + suggested_commands: [`zenrows crawl wait ${id}`, `zenrows crawl status ${id}`], + }); + } + first ??= page; + results.push(...(page.results ?? [])); + if (page.next_cursor === null || page.next_cursor === undefined || page.next_cursor === cursor) break; + cursor = page.next_cursor; + } + return { crawl: first!, results }; +} + +/** + * Poll `GET /crawls/{id}?limit=1` until the status leaves `running`, backing off + * 2s → ×1.5 → capped at 15s. On timeout throws CRAWL_TIMEOUT + * and leaves the crawl running. `sleepImpl` is injectable for tests. + */ +export async function waitForCrawl( + id: string, + opts: CallOpts & { sleepImpl?: (ms: number) => Promise }, +): Promise { + const sleep = opts.sleepImpl ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); + const timeoutMs = opts.timeoutMs ?? 600_000; + const deadline = Date.now() + timeoutMs; + let delay = 2000; + for (;;) { + const crawl = await getCrawl(id, { apiKey: opts.apiKey, fetchImpl: opts.fetchImpl, limit: 1 }); + if (isTerminal(crawl.status)) return crawl; + if (Date.now() + delay > deadline) { + throw new ToolkitError({ + code: "CRAWL_TIMEOUT", + message: `Timed out waiting for crawl ${id} to finish.`, + likely_cause: `The crawl was still running after ${Math.round(timeoutMs / 1000)}s. It keeps running; it was not stopped.`, + next_action: "Re-check progress with `zenrows crawl status `, wait again with a larger --timeout, or stop it.", + suggested_commands: [`zenrows crawl status ${id}`, `zenrows crawl wait ${id}`, `zenrows crawl stop ${id}`], + }); + } + await sleep(delay); + delay = Math.min(delay * 1.5, 15_000); + } +} diff --git a/src/core/errors.ts b/src/core/errors.ts index b36f109..95d3a22 100644 --- a/src/core/errors.ts +++ b/src/core/errors.ts @@ -35,6 +35,12 @@ export type ErrorCode = | "BATCH_QUOTA_EXCEEDED" | "BATCH_NOT_FOUND" | "BATCH_FAILED" + | "CRAWL_NOT_ENABLED" + | "CRAWL_QUOTA_EXCEEDED" + | "CRAWL_NOT_FOUND" + | "CRAWL_INVALID" + | "CRAWL_FAILED" + | "CRAWL_TIMEOUT" | "PLUGIN_UNSUPPORTED" | "MCP_CLIENT_UNSUPPORTED" | "ASSET_NOT_FOUND" diff --git a/tests/capabilities.test.ts b/tests/capabilities.test.ts index 1aa8e9d..73c7da6 100644 --- a/tests/capabilities.test.ts +++ b/tests/capabilities.test.ts @@ -11,6 +11,7 @@ test("capability matrix loads with the honest classifications", () => { assert.equal(caps.mcp?.status, "available"); // Batch is a real product in open beta. assert.equal(caps.batch?.status, "beta"); + assert.equal(caps.crawl?.status, "available"); }); test("isUsable true for available + open beta, false for absent capabilities", () => { @@ -25,6 +26,7 @@ test("assertUsable passes for available + experimental + beta", () => { assert.doesNotThrow(() => assertUsable("browser")); // available (billing/escalation handled by docs, opt-out via policy) assert.doesNotThrow(() => assertUsable("extract")); assert.doesNotThrow(() => assertUsable("batch")); + assert.doesNotThrow(() => assertUsable("crawl")); assert.throws( () => assertUsable("nope"), (e: unknown) => e instanceof ToolkitError && e.code === "CAPABILITY_UNAVAILABLE", diff --git a/tests/crawl-api.test.ts b/tests/crawl-api.test.ts new file mode 100644 index 0000000..2dc80b3 --- /dev/null +++ b/tests/crawl-api.test.ts @@ -0,0 +1,286 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + CRAWL_API_BASE_ENV, + DEFAULT_CRAWL_API_BASE, + contentIdOf, + crawlBase, + createCrawl, + downloadCrawl, + getCrawl, + getCrawlContent, + listAllResults, + listCrawls, + stopCrawl, + waitForCrawl, + type CrawlWithResults, +} from "../src/core/crawl-api.ts"; +import { ToolkitError } from "../src/core/errors.ts"; + +const crawlBody = { + crawl_id: "c_1", + status: "running", + url: "https://example.com/products/", + depth: 1, + max_items: 10, + max_pages: 10, + coverage: { pages_fetched: 0, pages_failed: 0, items_found: 0 }, + created_at: "2026-10-15T09:00:00Z", +}; + +/** Build a fetch stub that returns a fixed body, recording each call. */ +function stubFetch(status: number, payload: unknown, headers: Record = {}) { + const calls: Array<{ url: string; init?: RequestInit }> = []; + const impl = (async (url: string, init?: RequestInit) => { + calls.push({ url, init }); + return new Response(typeof payload === "string" ? payload : JSON.stringify(payload), { + status, + headers: { "content-type": "application/json", ...headers }, + }); + }) as unknown as typeof fetch; + return { impl, calls }; +} + +function problem(status: number, code: string, detail = "nope", headers: Record = {}) { + return stubFetch(status, { code, title: code, detail, status, instance: "urn:zenrows:request:x" }, { + "content-type": "application/problem+json", + ...headers, + }); +} + +test("crawlBase defaults to the production API and honors ZENROWS_CRAWL_API_BASE", () => { + assert.equal(crawlBase(), DEFAULT_CRAWL_API_BASE); + process.env[CRAWL_API_BASE_ENV] = "https://example.com/v1/"; + try { + assert.equal(crawlBase(), "https://example.com/v1"); + } finally { + delete process.env[CRAWL_API_BASE_ENV]; + } +}); + +test("createCrawl posts /crawls with X-API-Key and sends only the fields the caller set", async () => { + const { impl, calls } = stubFetch(202, crawlBody, { location: "/v1/crawls/c_1" }); + const c = await createCrawl( + { url: "https://example.com/products/", depth: 1, max_items: 3, include_patterns: ["/product/"], exclude_patterns: [] }, + { apiKey: "zr-key", fetchImpl: impl }, + ); + assert.equal(c.crawl_id, "c_1"); + const call = calls[0]!; + assert.equal(call.url, `${DEFAULT_CRAWL_API_BASE}/crawls`); + assert.equal(call.init?.method, "POST"); + const headers = call.init?.headers as Record; + assert.equal(headers["X-API-Key"], "zr-key"); + assert.equal(headers["Content-Type"], "application/json"); + assert.equal(headers["Idempotency-Key"], undefined); + assert.ok(!call.url.includes("apikey"), "the key never rides the query string"); + const body = JSON.parse(String(call.init?.body)); + assert.deepEqual(body, { url: "https://example.com/products/", depth: 1, max_items: 3, include_patterns: ["/product/"] }); + assert.ok(!("discovery" in body)); + assert.ok(!("output_format" in body)); +}); + +test("createCrawl sends output_format html and Idempotency-Key when asked", async () => { + const { impl, calls } = stubFetch(202, crawlBody); + await createCrawl({ url: "https://example.com/products/", depth: 2, max_pages: 5, output_format: "html" }, { + apiKey: "k", + fetchImpl: impl, + idempotencyKey: "idem-1", + }); + const body = JSON.parse(String(calls[0]!.init?.body)); + assert.deepEqual(body, { url: "https://example.com/products/", depth: 2, max_pages: 5, output_format: "html" }); + assert.equal((calls[0]!.init?.headers as Record)["Idempotency-Key"], "idem-1"); +}); + +test("getCrawl passes cursor + limit and parses results, tolerating unknown fields", async () => { + const { impl, calls } = stubFetch(200, { + ...crawlBody, + status: "a_future_status", + brand_new_field: { x: 1 }, + results: [{ url: "https://example.com/product/a", content_status: "fetched", content_url: "/v1/crawls/c_1/contents/ct_9" }], + next_cursor: "cur_2", + }); + const page = await getCrawl("c_1", { apiKey: "k", cursor: "cur_1", limit: 50, fetchImpl: impl }); + const u = new URL(calls[0]!.url); + assert.equal(u.pathname, "/v1/crawls/c_1"); + assert.equal(u.searchParams.get("cursor"), "cur_1"); + assert.equal(u.searchParams.get("limit"), "50"); + assert.equal(calls[0]!.init?.method, "GET"); + assert.equal(page.results[0]!.content_status, "fetched"); + assert.equal(contentIdOf(page.results[0]!), "ct_9"); + assert.equal(page.next_cursor, "cur_2"); + assert.equal(page.brand_new_field && typeof page.brand_new_field, "object"); +}); + +test("listCrawls reads one page; next_cursor is absent on the last page", async () => { + const { impl, calls } = stubFetch(200, { crawls: [{ ...crawlBody, status: "completed" }] }); + const page = await listCrawls({ apiKey: "k", limit: 5, fetchImpl: impl }); + assert.equal(new URL(calls[0]!.url).pathname, "/v1/crawls"); + assert.equal(new URL(calls[0]!.url).searchParams.get("limit"), "5"); + assert.equal(new URL(calls[0]!.url).searchParams.get("cursor"), null); + assert.equal(page.crawls.length, 1); + assert.equal(page.next_cursor, undefined); +}); + +test("stopCrawl posts /crawls/{id}/stop with no body", async () => { + const { impl, calls } = stubFetch(200, { crawl_id: "c_1", status: "stopped", stop_reason: "user" }); + const s = await stopCrawl("c_1", { apiKey: "k", fetchImpl: impl }); + assert.equal(s.status, "stopped"); + assert.equal(calls[0]!.init?.method, "POST"); + assert.equal(calls[0]!.init?.body, undefined); + assert.match(calls[0]!.url, /\/crawls\/c_1\/stop$/); +}); + +test("getCrawlContent returns the HTML body and its type", async () => { + const { impl, calls } = stubFetch(200, "hi", { "content-type": "text/html; charset=utf-8" }); + const out = await getCrawlContent("c_1", "ct_9", { apiKey: "k", fetchImpl: impl }); + assert.match(calls[0]!.url, /\/crawls\/c_1\/contents\/ct_9$/); + assert.match(out.contentType, /text\/html/); + assert.equal(out.body, "hi"); +}); + +test("downloadCrawl returns the NDJSON and X-Crawl-Status", async () => { + const { impl, calls } = stubFetch(200, '{"url":"https://example.com/product/a"}\n{"url":"https://example.com/product/b"}\n', { + "content-type": "application/x-ndjson", + "x-crawl-status": "running", + }); + const out = await downloadCrawl("c_1", { apiKey: "k", fetchImpl: impl }); + assert.match(calls[0]!.url, /\/crawls\/c_1\/download$/); + assert.equal(out.crawlStatus, "running"); + assert.equal(out.ndjson.trim().split("\n").length, 2); +}); + +test("403 REQS008 maps to CRAWL_NOT_ENABLED with the API's wording", async () => { + const { impl } = problem(403, "REQS008", "Crawl is not enabled for this account."); + await assert.rejects( + () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: impl }), + (e: unknown) => + e instanceof ToolkitError && e.code === "CRAWL_NOT_ENABLED" && e.message === "Crawl is not enabled for this account.", + ); +}); + +test("404 crawl_not_found maps to CRAWL_NOT_FOUND", async () => { + const { impl } = problem(404, "crawl_not_found"); + await assert.rejects( + () => getCrawl("c_nope", { apiKey: "k", fetchImpl: impl }), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_NOT_FOUND" && /crawl_not_found/.test(e.likely_cause), + ); +}); + +test("404 content_not_found maps to CRAWL_NOT_FOUND with content guidance", async () => { + const { impl } = problem(404, "content_not_found"); + await assert.rejects( + () => getCrawlContent("c_1", "ct_x", { apiKey: "k", fetchImpl: impl }), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_NOT_FOUND" && /content_status/.test(e.next_action), + ); +}); + +test("422 invalid_parameter and 400 unknown_parameter map to CRAWL_INVALID", async () => { + for (const [status, code] of [ + [422, "invalid_parameter"], + [400, "unknown_parameter"], + [409, "idempotency_request_in_flight"], + ] as const) { + const { impl } = problem(status, code, "'depth' is out of range."); + await assert.rejects( + () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: impl }), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_INVALID" && e.likely_cause.includes(code), + ); + } +}); + +test("429 too_many_crawls maps to CRAWL_QUOTA_EXCEEDED and carries Retry-After", async () => { + const { impl } = problem(429, "too_many_crawls", "Too many crawls.", { "retry-after": "30" }); + await assert.rejects( + () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: impl }), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_QUOTA_EXCEEDED" && /Retry after 30s/.test(e.likely_cause), + ); +}); + +test("401 maps to AUTH_INVALID and 500 to CRAWL_FAILED", async () => { + const a = problem(401, "AUTH002"); + await assert.rejects(() => listCrawls({ apiKey: "k", fetchImpl: a.impl }), (e: unknown) => e instanceof ToolkitError && e.code === "AUTH_INVALID"); + const b = problem(500, "internal_error"); + await assert.rejects(() => getCrawl("c", { apiKey: "k", fetchImpl: b.impl }), (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_FAILED"); +}); + +test("a transport failure maps to BACKEND_UNAVAILABLE", async () => { + const impl = (async () => { + throw new TypeError("fetch failed"); + }) as unknown as typeof fetch; + await assert.rejects(() => getCrawl("c", { apiKey: "k", fetchImpl: impl }), (e: unknown) => e instanceof ToolkitError && e.code === "BACKEND_UNAVAILABLE"); +}); + +/** A stub that serves `pages` in order, recording the cursor each call sent. */ +function pagedFetch(pages: Array>) { + const cursors: Array = []; + const limits: Array = []; + let i = 0; + const impl = (async (url: string) => { + const u = new URL(url); + cursors.push(u.searchParams.get("cursor")); + limits.push(u.searchParams.get("limit")); + const page = pages[Math.min(i++, pages.length - 1)]!; + return new Response(JSON.stringify({ ...crawlBody, ...page }), { status: 200, headers: { "content-type": "application/json" } }); + }) as unknown as typeof fetch; + return { impl, cursors, limits, calls: () => i }; +} + +test("listAllResults follows next_cursor and stops on null", async () => { + const { impl, cursors } = pagedFetch([ + { status: "completed", results: [{ url: "https://example.com/product/1" }], next_cursor: "c1" }, + { status: "completed", results: [{ url: "https://example.com/product/2" }], next_cursor: "c2" }, + { status: "completed", results: [{ url: "https://example.com/product/3" }], next_cursor: null }, + ]); + const { results, crawl } = await listAllResults("c_1", { apiKey: "k", fetchImpl: impl }); + assert.deepEqual(results.map((r) => r.url), ["https://example.com/product/1", "https://example.com/product/2", "https://example.com/product/3"]); + assert.deepEqual(cursors, [null, "c1", "c2"]); + assert.equal(crawl.status, "completed"); +}); + +test("listAllResults refuses a running crawl instead of polling forever", async () => { + const { impl, calls } = pagedFetch([{ status: "running", results: [], next_cursor: "c1" }]); + await assert.rejects( + () => listAllResults("c_1", { apiKey: "k", fetchImpl: impl }), + (e: unknown) => e instanceof ToolkitError && e.code === "INVALID_USAGE" && /still running/.test(e.message), + ); + assert.equal(calls(), 1); +}); + +test("waitForCrawl polls with limit=1 until the status leaves running, backing off", async () => { + const { impl, limits, calls } = pagedFetch([ + { status: "running", results: [], next_cursor: "c" }, + { status: "running", results: [], next_cursor: "c" }, + { status: "running", results: [], next_cursor: "c" }, + { status: "completed", results: [], next_cursor: null }, + ]); + const sleeps: number[] = []; + const c = await waitForCrawl("c_1", { apiKey: "k", fetchImpl: impl, sleepImpl: async (ms) => void sleeps.push(ms) }); + assert.equal(c.status, "completed"); + assert.equal(calls(), 4); + assert.ok(limits.every((l) => l === "1")); + assert.deepEqual(sleeps, [2000, 3000, 4500]); +}); + +test("waitForCrawl treats failed and stopped as terminal", async () => { + for (const status of ["failed", "stopped"]) { + const { impl } = pagedFetch([{ status, results: [], next_cursor: null }]); + const c = await waitForCrawl("c_1", { apiKey: "k", fetchImpl: impl, sleepImpl: async () => {} }); + assert.equal(c.status, status); + } +}); + +test("waitForCrawl times out with CRAWL_TIMEOUT and does not stop the crawl", async () => { + const methods: string[] = []; + const impl = (async (_url: string, init?: RequestInit) => { + methods.push(init?.method ?? "GET"); + return new Response(JSON.stringify({ ...crawlBody, results: [], next_cursor: "c" }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }) as unknown as typeof fetch; + await assert.rejects( + () => waitForCrawl("c_1", { apiKey: "k", fetchImpl: impl, timeoutMs: 1, sleepImpl: async () => {} }), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_TIMEOUT" && /not stopped/.test(e.likely_cause), + ); + assert.ok(methods.every((m) => m === "GET"), "no stop call on timeout"); +}); diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts new file mode 100644 index 0000000..768fb98 --- /dev/null +++ b/tests/crawl-command.test.ts @@ -0,0 +1,324 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { crawl } from "../src/cli/commands/crawl.ts"; +import { createWorkspace } from "../src/core/workspace.ts"; +import { savePolicy, defaultPolicy } from "../src/core/policy.ts"; +import { saveApiKey } from "../src/core/auth.ts"; +import { tempRoot } from "./helpers.ts"; + +const ctx = { json: true, yes: false }; + +const crawlBody = { + crawl_id: "c_1", + status: "running", + url: "https://example.com/products/", + depth: 1, + max_items: 10, + max_pages: 10, + coverage: { pages_fetched: 0, pages_failed: 0, items_found: 0 }, + created_at: "2026-10-15T09:00:00Z", +}; + +type Handler = (url: URL, init?: RequestInit) => Response; + +/** + * Run `fn` in a fresh workspace with a saved key and the given policy; every + * network call goes to `handler` and is recorded. + */ +function withCrawlWorkspace( + handler: Handler, + fn: (calls: Array<{ url: URL; init?: RequestInit }>) => Promise, + policy: Partial> = {}, +): Promise { + const { root, cleanup } = tempRoot(); + const cwd = process.cwd(); + createWorkspace(root); + savePolicy({ ...defaultPolicy(), ...policy }, root); + saveApiKey("0".repeat(41), root); + process.chdir(root); + const calls: Array<{ url: URL; init?: RequestInit }> = []; + const orig = globalThis.fetch; + globalThis.fetch = (async (u: string, init?: RequestInit) => { + const url = new URL(u); + calls.push({ url, init }); + return handler(url, init); + }) as unknown as typeof fetch; + return fn(calls).finally(() => { + globalThis.fetch = orig; + process.chdir(cwd); + cleanup(); + }); +} + +function json(body: unknown, status = 200, headers: Record = {}): Response { + return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json", ...headers } }); +} + +/** Capture stdout (log.out) for the duration of `fn`. */ +async function captureOut(fn: () => unknown): Promise { + const orig = process.stdout.write.bind(process.stdout); + let buf = ""; + process.stdout.write = ((s: string | Uint8Array) => { + buf += typeof s === "string" ? s : Buffer.from(s).toString(); + return true; + }) as typeof process.stdout.write; + try { + await fn(); + } finally { + process.stdout.write = orig; + } + return buf; +} + +async function run(args: string[]): Promise<{ code: number; out: Record }> { + let code = -1; + const raw = await captureOut(async () => { + code = await crawl.run(args, ctx); + }); + return { code, out: JSON.parse(raw) as Record }; +} + +test("crawl start rejects flags it does not declare, before any network call", async () => { + await withCrawlWorkspace( + () => json(crawlBody, 202), + async (calls) => { + for (const flag of ["--pagination", "--discovery", "--json-output"]) { + const { code, out } = await run(["start", "https://example.com/products/", "--depth", "1", flag]); + assert.equal(code, 1, flag); + assert.equal(out.error.code, "UNKNOWN_FLAG", flag); + } + assert.equal(calls.length, 0); + }, + ); +}); + +test("crawl start requires --depth and a whole number in range", async () => { + await withCrawlWorkspace( + () => json(crawlBody, 202), + async (calls) => { + for (const args of [[], ["--depth", "0"], ["--depth", "1.5"], ["--depth", "100001"], ["--depth", "x"]]) { + const { code, out } = await run(["start", "https://example.com/products/", ...args]); + assert.equal(code, 1, args.join(" ")); + assert.equal(out.error.code, "INVALID_USAGE", args.join(" ")); + } + assert.equal(calls.length, 0); + }, + ); +}); + +test("crawl start sends only the flags the caller set as the create body", async () => { + await withCrawlWorkspace( + () => json(crawlBody, 202, { location: "/v1/crawls/c_1" }), + async (calls) => { + const { code, out } = await run([ + "start", + "https://example.com/products/", + "--depth", + "1", + "--max-items", + "3", + "--max-pages", + "5", + "--include", + "/product/", + "--include", + "/item/", + "--exclude", + "?add", + "--html", + ]); + assert.equal(code, 0); + assert.equal(out.ok, true); + assert.equal(out.crawlId, "c_1"); + assert.equal(out.status, "running"); + assert.equal(calls.length, 1); + assert.equal(calls[0]!.url.pathname, "/v1/crawls"); + assert.deepEqual(JSON.parse(String(calls[0]!.init?.body)), { + url: "https://example.com/products/", + depth: 1, + max_items: 3, + max_pages: 5, + include_patterns: ["/product/", "/item/"], + exclude_patterns: ["?add"], + output_format: "html", + }); + }, + ); +}); + +test("crawl start honors the domain policy and the page cap before any network call", async () => { + await withCrawlWorkspace( + () => json(crawlBody, 202), + async (calls) => { + const blocked = await run(["start", "https://blocked.example.com/", "--depth", "1"]); + assert.equal(blocked.code, 1); + const overCap = await run(["start", "https://example.com/products/", "--depth", "1", "--max-pages", "50"]); + assert.equal(overCap.code, 1); + assert.equal(overCap.out.error.code, "POLICY_LIMIT_EXCEEDED"); + assert.equal(calls.length, 0); + }, + { blocked_domains: ["blocked.example.com"], max_pages_per_run: 20 }, + ); +}); + +test("crawl start --follow waits for the end and exits 1 with CRAWL_FAILED on a failed crawl", async () => { + await withCrawlWorkspace( + (url, init) => + init?.method === "POST" + ? json(crawlBody, 202) + : json({ + ...crawlBody, + status: "failed", + error: { code: "seed_unreachable", detail: "The site may be blocking it; check the URL." }, + results: [], + next_cursor: null, + }), + async (calls) => { + const { code, out } = await run(["start", "https://example.com/products/", "--depth", "1", "--follow"]); + assert.equal(code, 1); + assert.equal(out.ok, false); + assert.equal(out.status, "failed"); + assert.equal(out.error.code, "CRAWL_FAILED"); + assert.match(out.error.likely_cause, /seed_unreachable/); + assert.equal(calls[1]!.url.searchParams.get("limit"), "1"); + assert.equal(out.crawl.results, undefined, "status output carries no partial result page"); + }, + ); +}); + +test("crawl start surfaces 403 REQS008 as CRAWL_NOT_ENABLED", async () => { + await withCrawlWorkspace( + () => + json( + { code: "REQS008", title: "Crawl is not enabled for this account.", detail: "Contact support.", status: 403 }, + 403, + { "content-type": "application/problem+json" }, + ), + async () => { + const { code, out } = await run(["start", "https://example.com/products/", "--depth", "1"]); + assert.equal(code, 1); + assert.equal(out.error.code, "CRAWL_NOT_ENABLED"); + assert.equal(out.error.message, "Crawl is not enabled for this account."); + }, + ); +}); + +test("crawl status exits 0 for completed and stopped crawls", async () => { + for (const [status, stop_reason] of [ + ["completed", "max_items"], + ["stopped", "user"], + ]) { + await withCrawlWorkspace( + () => json({ ...crawlBody, status, stop_reason, results: [{ url: "https://example.com/product/a" }], next_cursor: "c" }), + async () => { + const { code, out } = await run(["status", "c_1"]); + assert.equal(code, 0, status); + assert.equal(out.ok, true); + assert.equal(out.stop_reason, stop_reason); + }, + ); + } +}); + +test("crawl status on an unknown id exits 1 with CRAWL_NOT_FOUND", async () => { + await withCrawlWorkspace( + () => json({ code: "crawl_not_found", title: "Crawl not found", detail: "No crawl c_x.", status: 404 }, 404), + async () => { + const { code, out } = await run(["status", "c_x"]); + assert.equal(code, 1); + assert.equal(out.error.code, "CRAWL_NOT_FOUND"); + }, + ); +}); + +test("crawl results reads every page of an ended crawl and writes JSONL with --out", async () => { + const pages: Record = { + "": { ...crawlBody, status: "completed", results: [{ url: "https://example.com/product/1" }], next_cursor: "p2" }, + p2: { ...crawlBody, status: "completed", results: [{ url: "https://example.com/product/2" }], next_cursor: null }, + }; + await withCrawlWorkspace( + (url) => json(pages[url.searchParams.get("cursor") ?? ""]), + async () => { + const { code, out } = await run(["results", "c_1"]); + assert.equal(code, 0); + assert.equal(out.count, 2); + assert.deepEqual(out.results.map((r: { url: string }) => r.url), ["https://example.com/product/1", "https://example.com/product/2"]); + + const file = join(process.cwd(), "urls.jsonl"); + await captureOut(() => crawl.run(["results", "c_1", "--out", file], ctx)); + assert.equal(readFileSync(file, "utf8"), '{"url":"https://example.com/product/1"}\n{"url":"https://example.com/product/2"}\n'); + }, + ); +}); + +test("crawl results --cursor reads one page and returns next_cursor while running", async () => { + await withCrawlWorkspace( + () => json({ ...crawlBody, results: [{ url: "https://example.com/product/3" }], next_cursor: "p4" }), + async (calls) => { + const { code, out } = await run(["results", "c_1", "--cursor", "p3", "--limit", "500"]); + assert.equal(code, 0); + assert.equal(out.next_cursor, "p4"); + assert.equal(calls[0]!.url.searchParams.get("cursor"), "p3"); + assert.equal(calls[0]!.url.searchParams.get("limit"), "500"); + }, + ); +}); + +test("crawl results --download writes the NDJSON export to .jsonl", async () => { + await withCrawlWorkspace( + () => + new Response('{"url":"https://example.com/product/1","content_status":"fetched","content":""}\n', { + status: 200, + headers: { "content-type": "application/x-ndjson", "x-crawl-status": "completed" }, + }), + async (calls) => { + const { code, out } = await run(["results", "c_1", "--download"]); + assert.equal(code, 0); + assert.equal(out.count, 1); + assert.equal(out.partial, false); + assert.equal(calls[0]!.url.pathname, "/v1/crawls/c_1/download"); + assert.ok(existsSync(join(process.cwd(), "c_1.jsonl"))); + }, + ); +}); + +test("crawl content accepts a content_url and prints the page", async () => { + await withCrawlWorkspace( + () => new Response("ok", { status: 200, headers: { "content-type": "text/html" } }), + async (calls) => { + const { code, out } = await run(["content", "c_1", "/v1/crawls/c_1/contents/ct_9"]); + assert.equal(code, 0); + assert.equal(out.content, "ok"); + assert.equal(calls[0]!.url.pathname, "/v1/crawls/c_1/contents/ct_9"); + }, + ); +}); + +test("crawl list and stop call the right endpoints", async () => { + await withCrawlWorkspace( + (url, init) => + init?.method === "POST" + ? json({ crawl_id: "c_1", status: "stopped", stop_reason: "user" }) + : json({ crawls: [{ ...crawlBody, status: "completed" }], next_cursor: "n2" }), + async (calls) => { + const list = await run(["list", "--limit", "5"]); + assert.equal(list.code, 0); + assert.equal(list.out.count, 1); + assert.equal(list.out.next_cursor, "n2"); + assert.equal(calls[0]!.url.searchParams.get("limit"), "5"); + + const stop = await run(["stop", "c_1"]); + assert.equal(stop.code, 0); + assert.equal(stop.out.status, "stopped"); + assert.equal(calls[1]!.url.pathname, "/v1/crawls/c_1/stop"); + }, + ); +}); + +test("unknown crawl subcommand is INVALID_USAGE", async () => { + const { code, out } = await run(["discover"]); + assert.equal(code, 1); + assert.equal(out.error.code, "INVALID_USAGE"); +}); diff --git a/tests/e2e/crawl.e2e.test.ts b/tests/e2e/crawl.e2e.test.ts new file mode 100644 index 0000000..14d5a72 --- /dev/null +++ b/tests/e2e/crawl.e2e.test.ts @@ -0,0 +1,141 @@ +/** + * End-to-end test for `zenrows crawl` against a live Zenrows API. + * + * Drives the built CLI binary (`dist/bin/zenrows.js`) the way a user does, so it + * covers argument parsing, the HTTP client and the output together. Skipped + * unless all are set: + * + * ZENROWS_E2E=1 + * ZENROWS_API_KEY= + * ZENROWS_E2E_CRAWL_URL= + * + * Optional: `ZENROWS_E2E_CRAWL_INCLUDE` (an include pattern every result URL + * must contain) and `ZENROWS_CRAWL_API_BASE` (default https://api.zenrows.com/v1). + * Run with `npm run test:e2e`, which does NOT load tests/setup.ts (that file + * scrubs every ZENROWS_* variable to keep the unit suite hermetic). + * + * The test starts one real, small crawl (bills a few pages on the key's account). + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const START_URL = process.env.ZENROWS_E2E_CRAWL_URL; +const INCLUDE = process.env.ZENROWS_E2E_CRAWL_INCLUDE || undefined; +const enabled = process.env.ZENROWS_E2E === "1" && !!process.env.ZENROWS_API_KEY && !!START_URL; +const skip = enabled + ? false + : "set ZENROWS_E2E=1, ZENROWS_API_KEY and ZENROWS_E2E_CRAWL_URL to run the Crawl e2e test"; + +const BIN = fileURLToPath(new URL("../../bin/zenrows.js", import.meta.url)); +/** How long to keep retrying a start that hits the account's active-crawl limit. */ +const QUOTA_RETRY_MS = 5 * 60_000; + +interface CliResult { + code: number; + out: Record; +} + +/** Run `zenrows --json` in `cwd` and parse its JSON stdout. */ +function cli(cwd: string, args: string[]): Promise { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [BIN, ...args, "--json"], { + cwd, + env: { ...process.env, ZENROWS_CONFIG_HOME: cwd, ZENROWS_TELEMETRY: "off" }, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (d) => (stdout += d)); + child.stderr.on("data", (d) => (stderr += d)); + child.on("error", reject); + child.on("close", (code) => { + try { + resolve({ code: code ?? -1, out: JSON.parse(stdout) as Record }); + } catch { + reject(new Error(`zenrows ${args[0]} ${args[1] ?? ""} printed no JSON (exit ${code}): ${stdout.slice(0, 500)} ${stderr.slice(-500)}`)); + } + }); + }); +} + +function log(msg: string): void { + process.stdout.write(`${msg}\n`); +} + +test("zenrows crawl end to end against a live API", { skip, timeout: 20 * 60_000 }, async () => { + const dir = mkdtempSync(join(tmpdir(), "zr-crawl-e2e-")); + try { + // 1. Start a small crawl and wait for it. Other runs may hold the account's + // active-crawl slots: on CRAWL_QUOTA_EXCEEDED wait Retry-After and try again. + const deadline = Date.now() + QUOTA_RETRY_MS; + let start: CliResult; + for (;;) { + start = await cli(dir, [ + "crawl", "start", START_URL!, + "--depth", "1", "--max-items", "3", "--max-pages", "5", + ...(INCLUDE ? ["--include", INCLUDE] : []), + "--html", "--follow", "--timeout", "600000", + ]); + if (start.out.error?.code !== "CRAWL_QUOTA_EXCEEDED" || Date.now() > deadline) break; + const wait = Number(/Retry after (\d+)s/.exec(start.out.error.likely_cause)?.[1] ?? 30); + log(`too_many_crawls: retrying start in ${wait}s`); + await new Promise((r) => setTimeout(r, wait * 1000)); + } + assert.equal(start.code, 0, JSON.stringify(start.out.error ?? start.out)); + const id = start.out.crawlId as string; + assert.match(id, /\S/); + assert.equal(start.out.status, "completed"); + log(`crawl: ${start.out.status} (${start.out.stop_reason ?? "nothing left"})`); + + // 2. Every kept URL, across all pages. + const results = await cli(dir, ["crawl", "results", id]); + assert.equal(results.code, 0, JSON.stringify(results.out)); + const rows = results.out.results as Array<{ url: string; content_status?: string; content_url?: string }>; + assert.ok(rows.length >= 1, "at least one result"); + if (INCLUDE) for (const r of rows) assert.ok(r.url.includes(INCLUDE), `${r.url} matches --include`); + log(`results: ${rows.length} URL(s)${INCLUDE ? ", all matching the include pattern" : ""}`); + + // 3. One fetched page's HTML. + const fetched = rows.find((r) => r.content_status === "fetched" && r.content_url); + assert.ok(fetched, "at least one result has fetched content"); + const content = await cli(dir, ["crawl", "content", id, fetched.content_url!]); + assert.equal(content.code, 0, JSON.stringify(content.out)); + assert.match(content.out.contentType, /text\/html/); + assert.match(content.out.content, / l.trim()); + assert.equal(lines.length, rows.length); + assert.equal(download.out.count, rows.length); + log(`download: ${lines.length} NDJSON line(s), X-Crawl-Status ${download.out.crawlStatus}`); + + // 5. The new crawl is listed. + const list = await cli(dir, ["crawl", "list", "--limit", "100"]); + assert.equal(list.code, 0, JSON.stringify(list.out)); + assert.ok((list.out.crawls as Array<{ crawl_id: string }>).some((c) => c.crawl_id === id), "crawl is listed"); + log("list: the new crawl is listed"); + + // 6. Stopping an ended crawl answers as it ended. + const stop = await cli(dir, ["crawl", "stop", id]); + assert.equal(stop.code, 0, JSON.stringify(stop.out)); + assert.equal(stop.out.status, start.out.status); + log(`stop on ended crawl: status ${stop.out.status}`); + + // 7. An unknown id is CRAWL_NOT_FOUND. + const missing = await cli(dir, ["crawl", "status", "c_does_not_exist_e2e"]); + assert.equal(missing.code, 1); + assert.equal(missing.out.error.code, "CRAWL_NOT_FOUND"); + log("status on unknown id: CRAWL_NOT_FOUND"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/production-surface.test.ts b/tests/production-surface.test.ts index 4b6c420..db39b18 100644 --- a/tests/production-surface.test.ts +++ b/tests/production-surface.test.ts @@ -3,7 +3,8 @@ import assert from "node:assert/strict"; import { main } from "../src/cli/index.ts"; test("removed non-GA commands are not registered", async () => { - for (const cmd of ["crawl", "discover", "monitor"]) { + // `crawl` was here while it was non-GA; it is registered now. + for (const cmd of ["discover", "monitor"]) { const code = await main([cmd]); assert.equal(code, 1, `${cmd} should be unknown (exit 1)`); } @@ -15,4 +16,6 @@ test("GA commands remain registered", async () => { assert.equal(await main(["account", "--help"]), 0); // batch is registered again (beta) — --help exits 0 assert.equal(await main(["batch", "--help"]), 0); + // crawl is registered + assert.equal(await main(["crawl", "--help"]), 0); }); diff --git a/tests/registry.test.ts b/tests/registry.test.ts index b09c135..3ca7fb7 100644 --- a/tests/registry.test.ts +++ b/tests/registry.test.ts @@ -29,6 +29,7 @@ test("core skills include the GA set and the master decision-tree skill", () => "protected-fetch", "extract", "batch-jobs", + "crawl", "interact-browser", "cost-control", "trace-debug", @@ -36,8 +37,8 @@ test("core skills include the GA set and the master decision-tree skill", () => ]) { assert.ok(skills.includes(required), `skill ${required} present`); } - // Non-GA skills were trimmed from the registry. - for (const removed of ["discover", "crawl", "monitor"]) { + // Non-GA skills were trimmed from the registry (crawl is back). + for (const removed of ["discover", "monitor"]) { assert.ok(!skills.includes(removed), `skill ${removed} removed`); } assert.ok(existsSync(pkgPath("skills/zenrows/SKILL.md")), "master SKILL.md exists"); @@ -50,4 +51,6 @@ test("assetRunnable reflects backend capability status", () => { assert.equal(assetRunnable(batchSkill), true); // needs batch (open beta = usable) const extractSkill = loadRegistry("skill").find((s) => s.name === "extract")!; assert.equal(assetRunnable(extractSkill), true); // needs extract (open beta = usable) + const crawlSkill = loadRegistry("skill").find((s) => s.name === "crawl")!; + assert.equal(assetRunnable(crawlSkill), true); // needs crawl (available) }); From 66569d184d0b27e16e97b71c03126b4eab50b6a7 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Fri, 9 Oct 2026 12:39:24 +0200 Subject: [PATCH 2/3] fix(crawl): mark Crawl beta and align errors with the Crawl API Mark the crawl capability and skill as beta. Say Beta in the help, the README and the skills. Map Crawl errors to codes an agent can act on. 429 is now CRAWL_TOO_MANY_CRAWLS with a retry_after field. 400 and 422 are CRAWL_INVALID_REQUEST. 409 is CRAWL_REQUEST_IN_FLIGHT. A missing page is CRAWL_CONTENT_NOT_FOUND. A wait timeout is CRAWL_WAIT_TIMEOUT. Every error carries the API code as server_code, and crawl_id once the crawl exists. Keep --wait and remove --follow. Ctrl-C during a wait now says that the crawl keeps running, gives the resume command and exits 130. results returns what a running crawl kept, with partial: true. results --out --json prints the JSON envelope. stop on an ended crawl prints the status it ended with. The help and docs say that the local policy caps --max-pages and that --html uses up the page budget. --- README.md | 36 +++++++---- docs/capabilities.md | 2 +- docs/contributing.md | 5 +- registry/capabilities.json | 4 +- registry/skills.json | 4 +- skills/crawl/SKILL.md | 53 +++++++++++----- skills/trace-debug/SKILL.md | 5 +- skills/zenrows/SKILL.md | 4 +- src/cli/commands/crawl.ts | 104 ++++++++++++++++++++------------ src/core/crawl-api.ts | 116 ++++++++++++++++++++++-------------- src/core/errors.ts | 23 ++++++- tests/capabilities.test.ts | 2 +- tests/crawl-api.test.ts | 97 +++++++++++++++++++++++------- tests/crawl-command.test.ts | 87 +++++++++++++++++++++++++-- tests/e2e/crawl.e2e.test.ts | 8 +-- tests/registry.test.ts | 2 +- 16 files changed, 397 insertions(+), 155 deletions(-) diff --git a/README.md b/README.md index 0973eaf..64fc422 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ that endpoint with extraction parameters, not a separate product. | `zenrows fetch` | Fetch — retrieve a protected page | **available** — `GET https://api.zenrows.com/v1/` | | `zenrows extract` | Extract — `extract=auto` (domain-gated open beta; falls back to Autoparse) / CSS / Markdown | **beta** — same `/v1/` | | `zenrows batch` | Batch — fan out over many URLs | beta — cloud works with beta access; local validate/estimate always | -| `zenrows crawl` | Crawl — collect the URLs behind one start page, optionally with each page's HTML | **available** — `https://api.zenrows.com/v1/crawls` | +| `zenrows crawl` | Crawl (beta) — collect the URLs behind one start page, optionally with each page's HTML | beta — `https://api.zenrows.com/v1/crawls` | | `zenrows browser` | Browser Sessions REST API (same backend as MCP `browser_*`) | **available** — escalation-only; bills by bandwidth + time | | `zenrows mcp` | MCP server config (remote + local) | **available** | | Zenrows CLI | this repo | available | @@ -157,19 +157,19 @@ An Extract task costs the same as a regular one (1 credit at base tier), so keys: `html` (the raw page) and `parsed` (the structured data) — validate a sample with `zenrows extract ` before running the full batch. -## 10. Crawl +## 10. Crawl (beta) -Zenrows **Crawl** (`https://api.zenrows.com/v1/crawls`) takes one start URL and -returns the URLs it finds behind it, optionally with each page's HTML. Use it -when you know the listing or section but not the item URLs. Crawl follows the -links on each page up to `--depth` hops and stays on the start URL's domain. -Without `--html` it returns URLs only. An account without Crawl access gets +Zenrows **Crawl** is in beta. Crawl (`https://api.zenrows.com/v1/crawls`) +takes one start URL and returns the URLs it finds behind it, optionally with +each page's HTML. Use it when you know the listing or section but not the item +URLs. Crawl follows the links on each page up to `--depth` hops and stays on +the start URL's registrable domain (subdomains count). Without `--html` it returns URLs only. An account without Crawl access gets `CRAWL_NOT_ENABLED` ("Crawl is not enabled for this account", HTTP 403 `REQS008`). ```bash zenrows crawl start https://example.com/products/ --depth 1 \ - --max-items 20 --include /product/ --follow # start and wait for the end + --max-items 20 --include /product/ --wait # start and wait for the end zenrows crawl results --out urls.jsonl # every kept URL (paginated) zenrows crawl start --depth 2 --html # also fetch each kept page's HTML zenrows crawl content # one page's HTML @@ -181,10 +181,22 @@ zenrows crawl stop # stop a running crawl `--depth` (link hops) is required; `--max-items` and `--max-pages` cap the crawl (API default 10 each), and `--max-pages` bounds the cost, since each page -is one fetch. `--include` / `--exclude` are repeatable substring filters on the -URL. When the account has too many crawls running, a start gets -`CRAWL_QUOTA_EXCEEDED` and creates nothing; retry after `Retry-After`. Set -`ZENROWS_CRAWL_API_BASE` to point the CLI at another deployment. +is one fetch. The local policy `max_pages_per_run` caps `--max-pages` (default +1000). `--html` uses up the page budget: each kept page is one more fetch, and +`--max-pages` counts it. `--include` / `--exclude` are repeatable substring +filters on the URL. `results` on a running crawl returns the URLs kept so far, +with `partial: true`. `--wait` gives up after `--timeout` (default 600000 ms) +with `CRAWL_WAIT_TIMEOUT`; the crawl keeps running. Ctrl-C stops the wait, not +the crawl, and exits 130; resume with `zenrows crawl wait `. + +Errors carry the API's code as `server_code`, and `crawl_id` once the crawl +exists. When the account has reached its limit of active jobs (3 by default), +shared with its Batch jobs, a start gets `CRAWL_TOO_MANY_CRAWLS` and creates +nothing; retry after `retry_after` seconds. Other codes: `CRAWL_INVALID_REQUEST` +(400/422, do not retry as is), `CRAWL_REQUEST_IN_FLIGHT` (409, retry once the +first request ends), `CRAWL_NOT_FOUND`, `CRAWL_CONTENT_NOT_FOUND` and +`CRAWL_FAILED`. Set `ZENROWS_CRAWL_API_BASE` to point the CLI at another +deployment. ## 11. Browser Sessions diff --git a/docs/capabilities.md b/docs/capabilities.md index 2557ee7..96f241e 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -22,6 +22,6 @@ Classification is based on the public Zenrows documentation: | `protected_fetch` | Fetch `GET https://api.zenrows.com/v1/` with `mode`, `js_render`, `premium_proxy`, `proxy_country`, `wait`/`wait_for`, `js_instructions`, `response_type`, `screenshot`, `original_status`, … | available | | `extract` | Extract `GET https://api.zenrows.com/v1/` via `extract=auto` (domain-gated open beta; CLI falls back to `autoparse`), plus `autoparse`, `css_extractor`, `outputs`, `response_type=markdown\|plaintext` | beta | | `batch` | Batch `https://async.api.zenrows.com/v1` (separate host, `X-API-Key` header) — open beta. Local JSONL spec validation + credit estimation work with no key. | beta | -| `crawl` | Crawl `https://api.zenrows.com/v1/crawls` (`X-API-Key` header; same host as Fetch/Extract). Follows the links on each page from one start URL up to `depth`, on the start URL's domain; returns URLs, and with `output_format: "html"` each page's HTML. Without Crawl access the API returns 403 `REQS008` → `CRAWL_NOT_ENABLED`. | available | +| `crawl` | Crawl `https://api.zenrows.com/v1/crawls` (`X-API-Key` header; same host as Fetch/Extract). Follows the links on each page from one start URL up to `depth`, on the start URL's registrable domain (subdomains count); returns URLs, and with `output_format: "html"` each page's HTML. Without Crawl access the API returns 403 `REQS008` → `CRAWL_NOT_ENABLED`. | beta | | `browser` | Browser Sessions REST API `https://mcp.zenrows.com/browser/sessions/*` (Bearer; same backend as `@zenrows/mcp` `browser_*`); CDP via `zenrows browser connect`. Escalation-only (prefer fetch/extract); on by default, opt out via `policy.allow_browser`; bills by bandwidth + session time (15-min max) | available | | `mcp` | Hosted `https://mcp.zenrows.com/mcp` + local `npx -y @zenrows/mcp` | available | diff --git a/docs/contributing.md b/docs/contributing.md index cc3a519..437ef4d 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -65,8 +65,9 @@ ZENROWS_E2E=1 ZENROWS_E2E_CRAWL_URL=https://example.com/products/ \ ``` To target a local or staging deployment, set `ZENROWS_CRAWL_API_BASE` to its -`/v1` base. When the account has too many crawls running, the test waits -`Retry-After` and retries the start for up to 5 minutes, so run concurrent e2e +`/v1` base. When the account has reached its limit of active jobs (3 by +default, shared with its Batch jobs), the test waits `retry_after` seconds and +retries the start for up to 5 minutes, so run concurrent e2e runs on one account one at a time. ## Rules diff --git a/registry/capabilities.json b/registry/capabilities.json index 36c8a1f..b331569 100644 --- a/registry/capabilities.json +++ b/registry/capabilities.json @@ -31,11 +31,11 @@ "crawl": { "key": "crawl", "label": "Crawl", - "status": "available", + "status": "beta", "command": "zenrows crawl", "backend": "Crawl — https://api.zenrows.com/v1/crawls (X-API-Key header; same host as Fetch/Extract)", "requiresAuth": true, - "notes": "Follows the links on each page from one start URL up to depth, on the start URL's domain, and returns the URLs it keeps, optionally with each page's HTML (start/status/wait/results/content/list/stop). An account without Crawl access gets 403 REQS008 (CRAWL_NOT_ENABLED). Too many crawls running on the account gives 429 too_many_crawls (CRAWL_QUOTA_EXCEEDED); retry after Retry-After." + "notes": "Beta. Follows the links on each page from one start URL up to depth, on the start URL's registrable domain (subdomains count), and returns the URLs it keeps, optionally with each page's HTML (start/status/wait/results/content/list/stop). An account without Crawl access gets 403 REQS008 (CRAWL_NOT_ENABLED). When the account has reached its limit of active jobs (3 by default), shared with its Batch jobs, a start gets 429 too_many_crawls (CRAWL_TOO_MANY_CRAWLS); retry after retry_after seconds." }, "browser": { "key": "browser", diff --git a/registry/skills.json b/registry/skills.json index 2658341..7a01812 100644 --- a/registry/skills.json +++ b/registry/skills.json @@ -47,8 +47,8 @@ { "name": "crawl", "type": "skill", - "description": "Collect the URLs behind one start page with Crawl: start, wait, read results and page HTML, stop.", - "status": "available", + "description": "Collect the URLs behind one start page with Crawl (beta): start, wait, read results and page HTML, stop.", + "status": "beta", "requires_backend_capabilities": ["crawl"], "requires_auth": true, "version": "0.1.0", diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index 0d8136e..7008f59 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -1,20 +1,20 @@ --- name: crawl -description: Collect the URLs behind one start page with Crawl — start a crawl, wait for it, read the kept URLs and optionally each page's HTML, stop it. Use when the user has a listing or section URL but not the item URLs. +description: Collect the URLs behind one start page with Crawl (beta) — start a crawl, wait for it, read the kept URLs and optionally each page's HTML, stop it. Use when the user has a listing or section URL but not the item URLs. version: 0.1.0 requires_backend_capabilities: [crawl] --- -# Crawl +# Crawl (beta) -Give Crawl one start URL; it follows the links on that page (and, with a +Crawl is in beta. Give Crawl one start URL; it follows the links on that page (and, with a higher depth, on the pages behind it) and returns the URLs it keeps. Use it when you know where the items live (a category page, a blog index, a docs section) but not the item URLs themselves. Once you have the URLs, read pages with `zenrows fetch` / `zenrows extract`, or ask Crawl for each page's HTML with `--html`. -Crawl stays on the start URL's domain. An account without Crawl access gets +Crawl stays on the start URL's registrable domain; subdomains count. An account without Crawl access gets 403 `REQS008` → `CRAWL_NOT_ENABLED` ("Crawl is not enabled for this account"); do not retry it. @@ -22,8 +22,8 @@ do not retry it. ``` zenrows crawl start https://example.com/products/ --depth 1 \ - --max-items 20 --max-pages 25 --include /product/ --follow --json -zenrows crawl results --json # every kept URL (crawl must have ended) + --max-items 20 --max-pages 25 --include /product/ --wait --json +zenrows crawl results --json # every kept URL zenrows crawl results --out urls.jsonl # same, as JSONL ``` @@ -31,23 +31,31 @@ zenrows crawl results --out urls.jsonl # same, as JSONL page's links; `2` = also the links on each of those pages. - `--max-items` (kept URLs) and `--max-pages` (fetched pages) cap the crawl; both default to 10. `--max-pages` bounds the cost: each page is one fetch. + The local policy `max_pages_per_run` caps `--max-pages` (default 1000); + above it the CLI refuses with `POLICY_LIMIT_EXCEEDED` before any request. - `--include` / `--exclude` are substring matches on the normalized URL and repeat: `--include /product/ --exclude ?add-to-cart`. -- `start` returns at once. `--follow` (or `zenrows crawl wait `) polls - until the crawl ends; on `--timeout` the crawl keeps running (it is not - stopped) and you get `CRAWL_TIMEOUT`. +- `start` returns at once. `--wait` (or `zenrows crawl wait `) polls + until the crawl ends (default timeout 600000 ms). On `--timeout` the crawl + keeps running (it is not stopped) and you get `CRAWL_WAIT_TIMEOUT` with + `crawl_id`. Ctrl-C also stops the wait, not the crawl, and exits 130; + resume with `zenrows crawl wait `. +- `results` on a running crawl returns the URLs kept so far, with + `partial: true`. Read again once the crawl ends. ## Page HTML ``` -zenrows crawl start --depth 1 --html --follow +zenrows crawl start --depth 1 --html --max-pages 20 --wait zenrows crawl results --json # fetched results carry content_url zenrows crawl content # one page's HTML (or pass the content_url) zenrows crawl results --download --out pages.jsonl # every URL + HTML, NDJSON ``` +`--html` uses up the page budget: each kept page is one more fetch, and +`--max-pages` counts it. Raise `--max-pages` to keep the same number of URLs. A result's `content_status` is `pending`, `fetched` or `failed`; only -`fetched` results have content. +`fetched` results have content (else `CRAWL_CONTENT_NOT_FOUND`). ## While it runs, and after @@ -61,12 +69,29 @@ zenrows crawl stop # stop it; kept URLs stay readab A crawl ends `completed` (nothing left, or a cap hit: `stop_reason` `max_items` / `max_pages`), `stopped` (you stopped it) or `failed` (`error.code` and `error.detail` say why). Only `failed` exits non-zero (`CRAWL_FAILED`). +`stop` on a crawl that already ended prints the status it ended with. ## Limits -When the account has too many crawls running, a start gets 429 -`too_many_crawls` → `CRAWL_QUOTA_EXCEEDED`; nothing was created. Retry after -`Retry-After` (about 30s), or stop one of your crawls first. +When the account has reached its limit of active jobs (3 by default), shared +with its Batch jobs, a start gets 429 `too_many_crawls` → +`CRAWL_TOO_MANY_CRAWLS`; nothing was created. Retry after `retry_after` +seconds (about 30), or stop one of your crawls or Batch jobs first. + +## Errors + +Every Crawl error carries the API's code as `server_code` (when the API sent +one), and `crawl_id` once the crawl exists. + +| Code | Meaning | Retry? | +| --- | --- | --- | +| `CRAWL_NOT_ENABLED` | 403 `REQS008`: Crawl is not enabled for this account | no | +| `CRAWL_INVALID_REQUEST` | 400 or 422: fix the request. For `idempotency_key_reused`, send a new key or no key | no, not as is | +| `CRAWL_REQUEST_IN_FLIGHT` | 409: the same Idempotency-Key is still in flight | yes, once the first request ends | +| `CRAWL_NOT_FOUND` / `CRAWL_CONTENT_NOT_FOUND` | 404: wrong id, or the page has no content | no | +| `CRAWL_TOO_MANY_CRAWLS` | 429: the account's active-job limit | yes, after `retry_after` | +| `CRAWL_WAIT_TIMEOUT` | the wait ran out; the crawl keeps running | wait again | +| `CRAWL_FAILED` | the crawl ended `failed`, or another error | 5xx: yes, with backoff | Rules: start small (`--depth 1`, low caps) and check the results before a deep crawl. Mind the cost ([[cost-control]]). Read errors with diff --git a/skills/trace-debug/SKILL.md b/skills/trace-debug/SKILL.md index e786019..f0e35a2 100644 --- a/skills/trace-debug/SKILL.md +++ b/skills/trace-debug/SKILL.md @@ -47,8 +47,9 @@ zenrows trace export # JSON for sharing `failure_detail` in `zenrows batch status --json` say why. - `CRAWL_NOT_ENABLED` → the API answered 403 `REQS008`: Crawl is not enabled for this account. Do not retry; ask Zenrows support for access. -- `CRAWL_QUOTA_EXCEEDED` → 429 `too_many_crawls`: the account has too many - crawls running. Nothing was created; retry after `Retry-After` (see +- `CRAWL_TOO_MANY_CRAWLS` → 429 `too_many_crawls`: the account has reached + its limit of active jobs (3 by default), shared with its Batch jobs. + Nothing was created; retry after `retry_after` seconds (see `zenrows crawl list`). - `CRAWL_FAILED` → the crawl ended `failed`; `error.code` in `zenrows crawl status --json` says why, with `error.detail`. See diff --git a/skills/zenrows/SKILL.md b/skills/zenrows/SKILL.md index 58f87df..1f885d4 100644 --- a/skills/zenrows/SKILL.md +++ b/skills/zenrows/SKILL.md @@ -35,7 +35,7 @@ If the user has a known URL and wants structured data: → Use Extract. (zenrows extract | --autoparse | --css) If the user has a start page (listing, index, section) but not the item URLs: - → Use Crawl. (zenrows crawl start --depth 1 --follow, + → Use Crawl. (zenrows crawl start --depth 1 --wait, then `zenrows crawl results `) If the user has many URLs (bulk): @@ -74,7 +74,7 @@ Run `zenrows status` for the live capability matrix. As of this toolkit: | Protected Fetch | `zenrows fetch` | available (`GET /v1/`) | | Extract (extract=auto / Autoparse/CSS/Markdown) | `zenrows extract` | beta (same `/v1/`; extract=auto falls back to autoparse) | | Batch | `zenrows batch` | beta — runs cloud jobs (create/status/results); cheapest path at scale. Estimate specs locally with no key | -| Crawl | `zenrows crawl` | available — collect the URLs behind one start page, optionally with each page's HTML | +| Crawl | `zenrows crawl` | beta — collect the URLs behind one start page, optionally with each page's HTML | | Browser | `zenrows browser` | available (Browser Sessions REST API / MCP) | | MCP | `zenrows mcp` | available (remote + local server) | diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index a21775b..5579c75 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -1,9 +1,9 @@ /** - * `zenrows crawl` — Crawl API. + * `zenrows crawl` — Crawl API (status: beta). * * Give Crawl one start URL and read back the URLs it finds behind it, optionally * with each page's HTML. A crawl is an async job: `start` returns at once - * (or polls with `--follow`), then `status` / `wait` / `results` / `content` + * (or polls with `--wait`), then `status` / `wait` / `results` / `content` * read it and `stop` ends it. An account without Crawl access gets 403 REQS008 * → CRAWL_NOT_ENABLED. */ @@ -24,6 +24,7 @@ import { stopCrawl, waitForCrawl, type Crawl, + type CrawlResult, type CrawlStop, type CreateCrawlParams, } from "../../core/crawl-api.ts"; @@ -39,25 +40,27 @@ const MAX_LIMIT = 100_000; export const crawl: Command = { name: "crawl", - summary: "Crawl a site from one start URL and collect the URLs behind it.", + summary: "Crawl a site from one start URL and collect the URLs behind it (beta).", usage: "zenrows crawl --depth N|status |results |content |list|stop |wait >", help: [ - "Cloud (needs a key with Crawl access):", + "Crawl is in beta. Cloud (needs a key with Crawl access):", " start --depth [flags] start a crawl from one URL (returns at once)", " --depth link hops to follow from the start URL (1-100000, required)", " --max-items stop after keeping n URLs (API default 10)", - " --max-pages stop after fetching n pages (API default 10; bounds cost)", + " --max-pages stop after fetching n pages (API default 10; bounds cost);", + " the local policy max_pages_per_run caps it (default 1000)", " --include keep only URLs containing this substring (repeatable)", " --exclude drop URLs containing this substring (repeatable)", - " --html also fetch each kept URL's page HTML (read with `content`)", - " --follow poll until the crawl ends (alias: --wait)", - " --timeout with --follow: give up waiting after ms (default 600000)", + " --html also fetch each kept URL's page HTML (read with `content`);", + " each kept page is one more fetch counted by --max-pages", + " --wait poll until the crawl ends (Ctrl-C stops the wait, not the crawl)", + " --timeout with --wait: give up waiting after ms (default 600000)", " --idempotency-key make a retried start create no second crawl", " --no-signup do not auto-create a Free plan account if no key exists", " status show status, coverage, stop reason / error", " wait [--timeout ] poll until the crawl ends (it is not stopped on timeout)", - " results all kept URLs (crawl must have ended); paginated", + " results all kept URLs; on a running crawl, those kept so far (partial)", " --out write results as JSONL instead of printing", " --cursor [--limit ] read one page only (works while running; prints next_cursor)", " --download fetch the NDJSON export (with page HTML for --html crawls)", @@ -65,7 +68,7 @@ export const crawl: Command = { " content print one kept URL's page (content_id or content_url)", " --out write it to a file instead", " list [--limit ] [--cursor c] list your crawls, newest first", - " stop stop a running crawl (idempotent)", + " stop stop a running crawl (an ended crawl answers as it ended)", " --json print structured output", ].join("\n"), async run(argv: string[], ctx: RunContext): Promise { @@ -111,15 +114,14 @@ async function startCmd(rest: string[], ctx: RunContext): Promise { include: { type: "string", multiple: true }, exclude: { type: "string", multiple: true }, html: { type: "boolean" }, - follow: { type: "boolean" }, - wait: { type: "boolean" }, // alias for --follow + wait: { type: "boolean" }, timeout: { type: "string" }, "idempotency-key": { type: "string" }, "no-signup": { type: "boolean" }, json: { type: "boolean" }, }); const json = ctx.json || values.json === true; - const follow = values.follow === true || values.wait === true; + const wait = values.wait === true; const url = positionals[0]; if (!url) { throw new ToolkitError({ @@ -173,7 +175,7 @@ async function startCmd(rest: string[], ctx: RunContext): Promise { log.step(`Starting crawl of ${url} (depth ${depth})…`); try { const created = await createCrawl(params, { apiKey, idempotencyKey }); - const finished: Crawl = follow ? await waitForCrawl(created.crawl_id, { apiKey, timeoutMs }) : created; + const finished: Crawl = wait ? await waitUntilEnd(created.crawl_id, { apiKey, timeoutMs }) : created; const failure = crawlFailure(finished); const runDir = writeRun({ runId, @@ -190,7 +192,7 @@ async function startCmd(rest: string[], ctx: RunContext): Promise { const code = printCrawl(finished, json, `Started crawl ${created.crawl_id}`); if (!json) { if (runDir) log.dim(` artifact: ${runDir}`); - if (!follow) log.dim(` next: zenrows crawl wait ${created.crawl_id}`); + if (!wait) log.dim(` next: zenrows crawl wait ${created.crawl_id}`); } return code; } catch (err) { @@ -224,7 +226,7 @@ async function waitCmd(rest: string[], ctx: RunContext): Promise { const timeoutMs = normalizeTimeout(values.timeout); assertUsable("crawl"); const apiKey = requireApiKey(); - const c = await waitForCrawl(id, { apiKey, timeoutMs }); + const c = await waitUntilEnd(id, { apiKey, timeoutMs }); return printCrawl(c, ctx.json || values.json === true, `Crawl ${id} finished`); } @@ -262,31 +264,38 @@ async function resultsCmd(rest: string[], ctx: RunContext): Promise { // One page, as the API returns it — the way to follow a crawl while it runs. if (cursor !== undefined || limit !== undefined) { const page = await getCrawl(id, { apiKey, cursor, limit }); - const body = { ok: true, crawlId: id, status: page.status, count: page.results.length, results: page.results, next_cursor: page.next_cursor }; - if (outFile) { - writeOut(outFile, toJsonl(page.results)); - log.success(`Wrote ${page.results.length} result(s) → ${outFile}`); - log.info(`next_cursor: ${page.next_cursor ?? "null (last page)"}`); - } else if (json) { - log.out(JSON.stringify(body, null, 2)); - } else { - log.info(`${page.results.length} result(s) for crawl ${id} (status: ${page.status}):`); - log.out(toJsonl(page.results).trimEnd()); - log.info(`next_cursor: ${page.next_cursor ?? "null (last page)"}`); - } - return 0; + return printResults(id, page.status, page.results, { outFile, json, next_cursor: page.next_cursor ?? null }); } - const { crawl: c, results } = await listAllResults(id, { apiKey }); - if (outFile) { - writeOut(outFile, toJsonl(results)); - log.success(`Wrote ${results.length} result(s) → ${outFile}`); - } else if (json) { - log.out(JSON.stringify({ ok: true, crawlId: id, status: c.status, count: results.length, results }, null, 2)); + const { crawl: c, results, partial } = await listAllResults(id, { apiKey }); + return printResults(id, c.status, results, { outFile, json, partial }); +} + +/** Write or print results. Under --json, always print the envelope (`file` replaces `results` with --out). */ +function printResults( + id: string, + status: string, + results: CrawlResult[], + o: { outFile?: string; json: boolean; partial?: boolean; next_cursor?: string | null }, +): number { + const extra = { + ...(o.partial !== undefined ? { partial: o.partial } : {}), + ...(o.next_cursor !== undefined ? { next_cursor: o.next_cursor } : {}), + }; + if (o.outFile) writeOut(o.outFile, toJsonl(results)); + if (o.json) { + const rows = o.outFile ? { file: o.outFile } : { results }; + log.out(JSON.stringify({ ok: true, crawlId: id, status, count: results.length, ...rows, ...extra }, null, 2)); + return 0; + } + if (o.outFile) { + log.success(`Wrote ${results.length} result(s) → ${o.outFile}`); } else { - log.info(`${results.length} result(s) for crawl ${id} (status: ${c.status}):`); + log.info(`${results.length} result(s) for crawl ${id} (status: ${status}):`); log.out(toJsonl(results).trimEnd()); } + if (o.partial) log.warn(" The crawl is still running: these results are partial. Read them again once it ends."); + if (o.next_cursor !== undefined) log.info(`next_cursor: ${o.next_cursor ?? "null (last page)"}`); return 0; } @@ -356,7 +365,26 @@ async function stopCmd(rest: string[], ctx: RunContext): Promise { assertUsable("crawl"); const apiKey = requireApiKey(); const s = await stopCrawl(id, { apiKey }); - return printCrawl(s, ctx.json || values.json === true, `Stopped crawl ${id}`); + const headline = s.status === "stopped" ? `Stopped crawl ${id}` : `Nothing to stop: crawl ${id} had already ended`; + return printCrawl(s, ctx.json || values.json === true, headline); +} + +/** + * Wait for a crawl to end. Ctrl-C ends the wait, not the crawl: say so, give + * the command that resumes the wait, and exit 130 as shells do on SIGINT. + */ +async function waitUntilEnd(id: string, opts: { apiKey: string; timeoutMs?: number }): Promise { + const onSigint = () => { + log.warn(`Stopped waiting. Crawl ${id} is still running; it was not stopped.`); + log.dim(` resume: zenrows crawl wait ${id}`); + process.exit(130); + }; + process.once("SIGINT", onSigint); + try { + return await waitForCrawl(id, opts); + } finally { + process.off("SIGINT", onSigint); + } } /** @@ -379,6 +407,8 @@ export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null { likely_cause: why || "The crawl ended in status failed without a reason.", next_action: (code && next[code]) || "Read the error, fix the cause, then start a new crawl.", suggested_commands: [`zenrows crawl status ${c.crawl_id}`], + server_code: code, + crawl_id: c.crawl_id, }); } diff --git a/src/core/crawl-api.ts b/src/core/crawl-api.ts index f434f51..f98b248 100644 --- a/src/core/crawl-api.ts +++ b/src/core/crawl-api.ts @@ -138,6 +138,8 @@ interface RequestOpts { accept?: string; fetchImpl?: typeof fetch; timeoutMs?: number; + /** Set on every call about one crawl, so its errors carry `crawl_id`. */ + crawlId?: string; } interface RawResponse { @@ -188,13 +190,14 @@ async function crawlFetch(method: string, path: string, opts: RequestOpts): Prom next_action: "Check connectivity and retry. Override the host with ZENROWS_CRAWL_API_BASE if you are testing against staging.", suggested_commands: ["zenrows status"], + crawl_id: opts.crawlId, }); } finally { clearTimeout(timeout); } if (res.status < 200 || res.status >= 300) { - throw problemToError(res.status, text, method, path, res.headers.get("retry-after")); + throw problemToError(res.status, text, `${method} ${path}`, res.headers.get("retry-after"), opts.crawlId); } return { status: res.status, headers: res.headers, text }; } @@ -210,17 +213,21 @@ async function crawlJson(method: string, path: string, opts: RequestOpts): Pr message: "The Crawl API response was not valid JSON.", likely_cause: text.slice(0, 240) || "Empty response body.", next_action: "Retry, or check the crawl with `zenrows crawl status `.", + crawl_id: opts.crawlId, }); } } -/** Map a problem+json body + HTTP status to a normalized ToolkitError. */ +/** + * Map a problem+json body + HTTP status to a normalized ToolkitError that + * carries the API's `code` as `server_code`, and `crawl_id` when known. + */ function problemToError( status: number, body: string, - method: string, - path: string, + request: string, retryAfter: string | null, + crawlId?: string, ): ToolkitError { let problem: ProblemJson = {}; try { @@ -228,9 +235,22 @@ function problemToError( } catch { // non-JSON error body — fall through with an empty problem } - const serverCode = problem.code ?? ""; + const serverCode = problem.code || undefined; + const seconds = status === 429 && retryAfter && /^\d+$/.test(retryAfter.trim()) ? Number(retryAfter) : undefined; + const err = classifyProblem(status, problem, body, request, serverCode, seconds); + return new ToolkitError({ ...err.toJSON(), server_code: serverCode, crawl_id: crawlId, retry_after: seconds }); +} + +function classifyProblem( + status: number, + problem: ProblemJson, + body: string, + request: string, + serverCode: string | undefined, + retryAfter: number | undefined, +): ToolkitError { const detail = problem.detail || problem.title || body.slice(0, 240) || `HTTP ${status}`; - const cause = `HTTP ${status}${serverCode ? ` (${serverCode})` : ""} for ${method} ${path}: ${detail}`; + const cause = `HTTP ${status}${serverCode ? ` (${serverCode})` : ""} for ${request}: ${detail}`; if (status === 403 && serverCode === CRAWL_NOT_ENABLED_CODE) { return new ToolkitError({ @@ -251,49 +271,59 @@ function problemToError( suggested_commands: ["zenrows login --api-key "], }); } + if (status === 404 && serverCode === "content_not_found") { + return new ToolkitError({ + code: "CRAWL_CONTENT_NOT_FOUND", + message: "Crawl content not found.", + likely_cause: `${cause}. The page was not fetched (yet), its fetch failed, or the crawl ran without --html.`, + next_action: "Read the result's content_status with `zenrows crawl results `; only `fetched` results have content.", + suggested_commands: ["zenrows crawl list"], + }); + } if (status === 404) { - const content = serverCode === "content_not_found"; return new ToolkitError({ code: "CRAWL_NOT_FOUND", - message: content ? "Crawl content not found." : "Crawl not found.", - likely_cause: content - ? `${cause}. The page was not fetched (yet), its fetch failed, or the crawl ran without --html.` - : `${cause}. The id may be wrong or the crawl is not owned by this account.`, - next_action: content - ? "Read the result's content_status with `zenrows crawl results `; only `fetched` results have content." - : "Check the crawl id, or list your crawls with `zenrows crawl list`.", + message: "Crawl not found.", + likely_cause: `${cause}. The id may be wrong or the crawl is not owned by this account.`, + next_action: "Check the crawl id, or list your crawls with `zenrows crawl list`.", suggested_commands: ["zenrows crawl list"], }); } if (status === 429) { - const wait = retryAfter && /^\d+$/.test(retryAfter.trim()) ? ` Retry after ${retryAfter.trim()}s.` : ""; return new ToolkitError({ - code: "CRAWL_QUOTA_EXCEEDED", - message: "Too many crawls running.", - likely_cause: `${cause}. The account has too many crawls running.${wait}`, - next_action: - "Retry after Retry-After, or stop one of your crawls with `zenrows crawl stop ` first. Nothing was created.", + code: "CRAWL_TOO_MANY_CRAWLS", + message: "The account has reached its limit of active jobs.", + likely_cause: `${cause}. The account has reached its limit of active jobs (3 by default), shared with its Batch jobs.`, + next_action: `Nothing was created. ${retryAfter !== undefined ? `Retry after ${retryAfter} seconds` : "Retry later"}, or stop one of your crawls (\`zenrows crawl stop \`) or Batch jobs first.`, suggested_commands: ["zenrows crawl list"], }); } if (status === 402 && isKeyCapReached(serverCode)) { - return keyCapReached(`${method} ${path}`, { status: 402, detail: problem.detail || problem.title || undefined }); + return keyCapReached(request, { status: 402, detail: problem.detail || problem.title || undefined }); } if (status === 402) { const acct = readAccount(); - return quotaExhausted(`${method} ${path}`, acct?.unclaimed ? acct.claimUrl : undefined, { + return quotaExhausted(request, acct?.unclaimed ? acct.claimUrl : undefined, { status: 402, detail: problem.detail || problem.title || undefined, }); } - if (status === 400 || status === 409 || status === 422) { + if (status === 409) { return new ToolkitError({ - code: "CRAWL_INVALID", + code: "CRAWL_REQUEST_IN_FLIGHT", + message: "A request with the same Idempotency-Key is still in flight.", + likely_cause: `${cause}.`, + next_action: "Retry the same request once the first one has finished.", + }); + } + if (status === 400 || status === 422) { + return new ToolkitError({ + code: "CRAWL_INVALID_REQUEST", message: `Crawl rejected the request (HTTP ${status}).`, likely_cause: `${cause}.`, next_action: - status === 409 - ? "A request with the same Idempotency-Key is still in flight; retry once it has finished." + serverCode === "idempotency_key_reused" + ? "This Idempotency-Key was already used for a different request. Send a new key, or no key. Do not retry as is." : "Fix the reported parameter and retry. See `zenrows crawl --help` for the accepted flags and ranges.", }); } @@ -333,6 +363,7 @@ export function createCrawl(params: CreateCrawlParams, opts: CallOpts & { idempo export function getCrawl(id: string, opts: CallOpts & { cursor?: string; limit?: number }): Promise { return crawlJson("GET", `/crawls/${encodeURIComponent(id)}`, { apiKey: opts.apiKey, + crawlId: id, query: { cursor: opts.cursor, limit: opts.limit }, fetchImpl: opts.fetchImpl, timeoutMs: opts.timeoutMs, @@ -353,6 +384,7 @@ export function listCrawls(opts: CallOpts & { cursor?: string; limit?: number }) export function stopCrawl(id: string, opts: CallOpts): Promise { return crawlJson("POST", `/crawls/${encodeURIComponent(id)}/stop`, { apiKey: opts.apiKey, + crawlId: id, fetchImpl: opts.fetchImpl, timeoutMs: opts.timeoutMs, }); @@ -366,6 +398,7 @@ export async function getCrawlContent( ): Promise<{ contentType: string; body: string }> { const res = await crawlFetch("GET", `/crawls/${encodeURIComponent(id)}/contents/${encodeURIComponent(contentId)}`, { apiKey: opts.apiKey, + crawlId: id, accept: "text/html, application/json;q=0.9, application/problem+json;q=0.8", fetchImpl: opts.fetchImpl, timeoutMs: opts.timeoutMs, @@ -380,6 +413,7 @@ export async function getCrawlContent( export async function downloadCrawl(id: string, opts: CallOpts): Promise<{ crawlStatus?: string; ndjson: string }> { const res = await crawlFetch("GET", `/crawls/${encodeURIComponent(id)}/download`, { apiKey: opts.apiKey, + crawlId: id, accept: "application/x-ndjson, application/problem+json;q=0.8", fetchImpl: opts.fetchImpl, timeoutMs: opts.timeoutMs ?? 300_000, @@ -398,39 +432,30 @@ export function contentIdOf(result: CrawlResult): string | undefined { } /** - * Read every result, following `next_cursor` until it is null. Only call this - * on an ended crawl: while a crawl runs `next_cursor` is never null, so this - * refuses a `running` crawl instead of looping forever (wait first). + * Read every kept result, following `next_cursor`. While a crawl runs its + * `next_cursor` is never null, so the read also ends at the first empty page; + * `partial` is true when the crawl was still running. */ export async function listAllResults( id: string, opts: CallOpts & { pageSize?: number }, -): Promise<{ crawl: CrawlWithResults; results: CrawlResult[] }> { +): Promise<{ crawl: CrawlWithResults; results: CrawlResult[]; partial: boolean }> { const results: CrawlResult[] = []; let cursor: string | undefined; - let first: CrawlWithResults | undefined; for (;;) { const page = await getCrawl(id, { ...opts, cursor, limit: opts.pageSize }); - if (!isTerminal(page.status)) { - throw new ToolkitError({ - code: "INVALID_USAGE", - message: `Crawl ${id} is still running.`, - likely_cause: "Reading all results needs an ended crawl: while it runs, more results keep arriving.", - next_action: "Wait for it first (`zenrows crawl wait `), or stop it with `zenrows crawl stop `.", - suggested_commands: [`zenrows crawl wait ${id}`, `zenrows crawl status ${id}`], - }); + const rows = page.results ?? []; + results.push(...rows); + if (!rows.length || !page.next_cursor || page.next_cursor === cursor) { + return { crawl: page, results, partial: !isTerminal(page.status) }; } - first ??= page; - results.push(...(page.results ?? [])); - if (page.next_cursor === null || page.next_cursor === undefined || page.next_cursor === cursor) break; cursor = page.next_cursor; } - return { crawl: first!, results }; } /** * Poll `GET /crawls/{id}?limit=1` until the status leaves `running`, backing off - * 2s → ×1.5 → capped at 15s. On timeout throws CRAWL_TIMEOUT + * 2s → ×1.5 → capped at 15s. On timeout throws CRAWL_WAIT_TIMEOUT * and leaves the crawl running. `sleepImpl` is injectable for tests. */ export async function waitForCrawl( @@ -446,11 +471,12 @@ export async function waitForCrawl( if (isTerminal(crawl.status)) return crawl; if (Date.now() + delay > deadline) { throw new ToolkitError({ - code: "CRAWL_TIMEOUT", + code: "CRAWL_WAIT_TIMEOUT", message: `Timed out waiting for crawl ${id} to finish.`, likely_cause: `The crawl was still running after ${Math.round(timeoutMs / 1000)}s. It keeps running; it was not stopped.`, next_action: "Re-check progress with `zenrows crawl status `, wait again with a larger --timeout, or stop it.", suggested_commands: [`zenrows crawl status ${id}`, `zenrows crawl wait ${id}`, `zenrows crawl stop ${id}`], + crawl_id: id, }); } await sleep(delay); diff --git a/src/core/errors.ts b/src/core/errors.ts index 95d3a22..0c2c60f 100644 --- a/src/core/errors.ts +++ b/src/core/errors.ts @@ -36,11 +36,13 @@ export type ErrorCode = | "BATCH_NOT_FOUND" | "BATCH_FAILED" | "CRAWL_NOT_ENABLED" - | "CRAWL_QUOTA_EXCEEDED" + | "CRAWL_TOO_MANY_CRAWLS" | "CRAWL_NOT_FOUND" - | "CRAWL_INVALID" + | "CRAWL_CONTENT_NOT_FOUND" + | "CRAWL_INVALID_REQUEST" + | "CRAWL_REQUEST_IN_FLIGHT" | "CRAWL_FAILED" - | "CRAWL_TIMEOUT" + | "CRAWL_WAIT_TIMEOUT" | "PLUGIN_UNSUPPORTED" | "MCP_CLIENT_UNSUPPORTED" | "ASSET_NOT_FOUND" @@ -54,6 +56,12 @@ export interface ToolkitErrorShape { likely_cause: string; next_action: string; suggested_commands?: string[]; + /** The API's own `code`, when the error came from an API answer that had one. */ + server_code?: string; + /** The crawl the error is about, once it exists. */ + crawl_id?: string; + /** Seconds to wait before a retry (the `Retry-After` header). */ + retry_after?: number; } export class ToolkitError extends Error { @@ -61,6 +69,9 @@ export class ToolkitError extends Error { readonly likely_cause: string; readonly next_action: string; readonly suggested_commands: string[]; + readonly server_code?: string; + readonly crawl_id?: string; + readonly retry_after?: number; constructor(shape: ToolkitErrorShape) { super(shape.message); @@ -69,6 +80,9 @@ export class ToolkitError extends Error { this.likely_cause = shape.likely_cause; this.next_action = shape.next_action; this.suggested_commands = shape.suggested_commands ?? []; + this.server_code = shape.server_code; + this.crawl_id = shape.crawl_id; + this.retry_after = shape.retry_after; } toJSON(): ToolkitErrorShape { @@ -78,6 +92,9 @@ export class ToolkitError extends Error { likely_cause: this.likely_cause, next_action: this.next_action, suggested_commands: this.suggested_commands, + ...(this.server_code ? { server_code: this.server_code } : {}), + ...(this.crawl_id ? { crawl_id: this.crawl_id } : {}), + ...(this.retry_after !== undefined ? { retry_after: this.retry_after } : {}), }; } } diff --git a/tests/capabilities.test.ts b/tests/capabilities.test.ts index 73c7da6..3485169 100644 --- a/tests/capabilities.test.ts +++ b/tests/capabilities.test.ts @@ -11,7 +11,7 @@ test("capability matrix loads with the honest classifications", () => { assert.equal(caps.mcp?.status, "available"); // Batch is a real product in open beta. assert.equal(caps.batch?.status, "beta"); - assert.equal(caps.crawl?.status, "available"); + assert.equal(caps.crawl?.status, "beta"); }); test("isUsable true for available + open beta, false for absent capabilities", () => { diff --git a/tests/crawl-api.test.ts b/tests/crawl-api.test.ts index 2dc80b3..0ffab09 100644 --- a/tests/crawl-api.test.ts +++ b/tests/crawl-api.test.ts @@ -154,7 +154,19 @@ test("403 REQS008 maps to CRAWL_NOT_ENABLED with the API's wording", async () => await assert.rejects( () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: impl }), (e: unknown) => - e instanceof ToolkitError && e.code === "CRAWL_NOT_ENABLED" && e.message === "Crawl is not enabled for this account.", + e instanceof ToolkitError && + e.code === "CRAWL_NOT_ENABLED" && + e.message === "Crawl is not enabled for this account." && + e.server_code === "REQS008" && + e.crawl_id === undefined, + ); +}); + +test("any other 403 maps to CRAWL_FAILED", async () => { + const { impl } = problem(403, "REQS001"); + await assert.rejects( + () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: impl }), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_FAILED" && e.server_code === "REQS001", ); }); @@ -162,37 +174,71 @@ test("404 crawl_not_found maps to CRAWL_NOT_FOUND", async () => { const { impl } = problem(404, "crawl_not_found"); await assert.rejects( () => getCrawl("c_nope", { apiKey: "k", fetchImpl: impl }), - (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_NOT_FOUND" && /crawl_not_found/.test(e.likely_cause), + (e: unknown) => + e instanceof ToolkitError && + e.code === "CRAWL_NOT_FOUND" && + e.server_code === "crawl_not_found" && + e.crawl_id === "c_nope", ); }); -test("404 content_not_found maps to CRAWL_NOT_FOUND with content guidance", async () => { +test("404 content_not_found maps to CRAWL_CONTENT_NOT_FOUND with content guidance", async () => { const { impl } = problem(404, "content_not_found"); await assert.rejects( () => getCrawlContent("c_1", "ct_x", { apiKey: "k", fetchImpl: impl }), - (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_NOT_FOUND" && /content_status/.test(e.next_action), + (e: unknown) => + e instanceof ToolkitError && e.code === "CRAWL_CONTENT_NOT_FOUND" && /content_status/.test(e.next_action) && e.crawl_id === "c_1", ); }); -test("422 invalid_parameter and 400 unknown_parameter map to CRAWL_INVALID", async () => { +test("400 and 422 map to CRAWL_INVALID_REQUEST with the server code", async () => { for (const [status, code] of [ [422, "invalid_parameter"], [400, "unknown_parameter"], - [409, "idempotency_request_in_flight"], ] as const) { const { impl } = problem(status, code, "'depth' is out of range."); await assert.rejects( () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: impl }), - (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_INVALID" && e.likely_cause.includes(code), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_INVALID_REQUEST" && e.server_code === code, ); } }); -test("429 too_many_crawls maps to CRAWL_QUOTA_EXCEEDED and carries Retry-After", async () => { - const { impl } = problem(429, "too_many_crawls", "Too many crawls.", { "retry-after": "30" }); +test("422 idempotency_key_reused says to send a new key, not to retry as is", async () => { + const { impl } = problem(422, "idempotency_key_reused"); await assert.rejects( - () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: impl }), - (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_QUOTA_EXCEEDED" && /Retry after 30s/.test(e.likely_cause), + () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: impl, idempotencyKey: "k1" }), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_INVALID_REQUEST" && /new key, or no key/.test(e.next_action), + ); +}); + +test("409 maps to CRAWL_REQUEST_IN_FLIGHT", async () => { + const { impl } = problem(409, "idempotency_request_in_flight"); + await assert.rejects( + () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: impl, idempotencyKey: "k1" }), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_REQUEST_IN_FLIGHT" && /finished/.test(e.next_action), + ); +}); + +test("429 too_many_crawls maps to CRAWL_TOO_MANY_CRAWLS with retry_after and no retry", async () => { + const { impl, calls } = problem(429, "too_many_crawls", "Too many crawls.", { "retry-after": "30" }); + await assert.rejects( + () => createCrawl({ url: "https://example.com/products/", depth: 1 }, { apiKey: "k", fetchImpl: impl, idempotencyKey: "k1" }), + (e: unknown) => + e instanceof ToolkitError && + e.code === "CRAWL_TOO_MANY_CRAWLS" && + e.retry_after === 30 && + e.server_code === "too_many_crawls" && + /shared with its Batch jobs/.test(e.likely_cause), + ); + assert.equal(calls.length, 1); +}); + +test("an error body without a code leaves server_code unset", async () => { + const { impl } = stubFetch(500, "upstream exploded"); + await assert.rejects( + () => getCrawl("c_1", { apiKey: "k", fetchImpl: impl }), + (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_FAILED" && !("server_code" in e.toJSON()), ); }); @@ -207,7 +253,10 @@ test("a transport failure maps to BACKEND_UNAVAILABLE", async () => { const impl = (async () => { throw new TypeError("fetch failed"); }) as unknown as typeof fetch; - await assert.rejects(() => getCrawl("c", { apiKey: "k", fetchImpl: impl }), (e: unknown) => e instanceof ToolkitError && e.code === "BACKEND_UNAVAILABLE"); + await assert.rejects( + () => getCrawl("c", { apiKey: "k", fetchImpl: impl }), + (e: unknown) => e instanceof ToolkitError && e.code === "BACKEND_UNAVAILABLE" && e.crawl_id === "c", + ); }); /** A stub that serves `pages` in order, recording the cursor each call sent. */ @@ -231,19 +280,22 @@ test("listAllResults follows next_cursor and stops on null", async () => { { status: "completed", results: [{ url: "https://example.com/product/2" }], next_cursor: "c2" }, { status: "completed", results: [{ url: "https://example.com/product/3" }], next_cursor: null }, ]); - const { results, crawl } = await listAllResults("c_1", { apiKey: "k", fetchImpl: impl }); + const { results, crawl, partial } = await listAllResults("c_1", { apiKey: "k", fetchImpl: impl }); + assert.equal(partial, false); assert.deepEqual(results.map((r) => r.url), ["https://example.com/product/1", "https://example.com/product/2", "https://example.com/product/3"]); assert.deepEqual(cursors, [null, "c1", "c2"]); assert.equal(crawl.status, "completed"); }); -test("listAllResults refuses a running crawl instead of polling forever", async () => { - const { impl, calls } = pagedFetch([{ status: "running", results: [], next_cursor: "c1" }]); - await assert.rejects( - () => listAllResults("c_1", { apiKey: "k", fetchImpl: impl }), - (e: unknown) => e instanceof ToolkitError && e.code === "INVALID_USAGE" && /still running/.test(e.message), - ); - assert.equal(calls(), 1); +test("listAllResults returns what a running crawl kept, ending at the first empty page", async () => { + const { impl, calls } = pagedFetch([ + { status: "running", results: [{ url: "https://example.com/product/1" }], next_cursor: "c1" }, + { status: "running", results: [], next_cursor: "c1" }, + ]); + const { results, partial } = await listAllResults("c_1", { apiKey: "k", fetchImpl: impl }); + assert.deepEqual(results.map((r) => r.url), ["https://example.com/product/1"]); + assert.equal(partial, true); + assert.equal(calls(), 2); }); test("waitForCrawl polls with limit=1 until the status leaves running, backing off", async () => { @@ -269,7 +321,7 @@ test("waitForCrawl treats failed and stopped as terminal", async () => { } }); -test("waitForCrawl times out with CRAWL_TIMEOUT and does not stop the crawl", async () => { +test("waitForCrawl times out with CRAWL_WAIT_TIMEOUT, carries crawl_id and does not stop the crawl", async () => { const methods: string[] = []; const impl = (async (_url: string, init?: RequestInit) => { methods.push(init?.method ?? "GET"); @@ -280,7 +332,8 @@ test("waitForCrawl times out with CRAWL_TIMEOUT and does not stop the crawl", as }) as unknown as typeof fetch; await assert.rejects( () => waitForCrawl("c_1", { apiKey: "k", fetchImpl: impl, timeoutMs: 1, sleepImpl: async () => {} }), - (e: unknown) => e instanceof ToolkitError && e.code === "CRAWL_TIMEOUT" && /not stopped/.test(e.likely_cause), + (e: unknown) => + e instanceof ToolkitError && e.code === "CRAWL_WAIT_TIMEOUT" && e.crawl_id === "c_1" && /not stopped/.test(e.likely_cause), ); assert.ok(methods.every((m) => m === "GET"), "no stop call on timeout"); }); diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index 768fb98..631bf08 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -1,4 +1,4 @@ -import { test } from "node:test"; +import { mock, test } from "node:test"; import assert from "node:assert/strict"; import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; @@ -84,7 +84,7 @@ test("crawl start rejects flags it does not declare, before any network call", a await withCrawlWorkspace( () => json(crawlBody, 202), async (calls) => { - for (const flag of ["--pagination", "--discovery", "--json-output"]) { + for (const flag of ["--pagination", "--discovery", "--json-output", "--follow"]) { const { code, out } = await run(["start", "https://example.com/products/", "--depth", "1", flag]); assert.equal(code, 1, flag); assert.equal(out.error.code, "UNKNOWN_FLAG", flag); @@ -163,7 +163,7 @@ test("crawl start honors the domain policy and the page cap before any network c ); }); -test("crawl start --follow waits for the end and exits 1 with CRAWL_FAILED on a failed crawl", async () => { +test("crawl start --wait waits for the end and exits 1 with CRAWL_FAILED on a failed crawl", async () => { await withCrawlWorkspace( (url, init) => init?.method === "POST" @@ -176,12 +176,14 @@ test("crawl start --follow waits for the end and exits 1 with CRAWL_FAILED on a next_cursor: null, }), async (calls) => { - const { code, out } = await run(["start", "https://example.com/products/", "--depth", "1", "--follow"]); + const { code, out } = await run(["start", "https://example.com/products/", "--depth", "1", "--wait"]); assert.equal(code, 1); assert.equal(out.ok, false); assert.equal(out.status, "failed"); assert.equal(out.error.code, "CRAWL_FAILED"); assert.match(out.error.likely_cause, /seed_unreachable/); + assert.equal(out.error.server_code, "seed_unreachable"); + assert.equal(out.error.crawl_id, "c_1"); assert.equal(calls[1]!.url.searchParams.get("limit"), "1"); assert.equal(out.crawl.results, undefined, "status output carries no partial result page"); }, @@ -246,13 +248,33 @@ test("crawl results reads every page of an ended crawl and writes JSONL with --o assert.equal(out.count, 2); assert.deepEqual(out.results.map((r: { url: string }) => r.url), ["https://example.com/product/1", "https://example.com/product/2"]); + assert.equal(out.partial, false); + const file = join(process.cwd(), "urls.jsonl"); - await captureOut(() => crawl.run(["results", "c_1", "--out", file], ctx)); + const written = await run(["results", "c_1", "--out", file]); + assert.deepEqual(written.out, { ok: true, crawlId: "c_1", status: "completed", count: 2, file, partial: false }); assert.equal(readFileSync(file, "utf8"), '{"url":"https://example.com/product/1"}\n{"url":"https://example.com/product/2"}\n'); }, ); }); +test("crawl results on a running crawl returns what is kept with partial: true", async () => { + const pages: Record = { + "": { ...crawlBody, results: [{ url: "https://example.com/product/1" }], next_cursor: "p2" }, + p2: { ...crawlBody, results: [], next_cursor: "p2" }, + }; + await withCrawlWorkspace( + (url) => json(pages[url.searchParams.get("cursor") ?? ""]), + async () => { + const { code, out } = await run(["results", "c_1"]); + assert.equal(code, 0); + assert.equal(out.status, "running"); + assert.equal(out.partial, true); + assert.equal(out.count, 1); + }, + ); +}); + test("crawl results --cursor reads one page and returns next_cursor while running", async () => { await withCrawlWorkspace( () => json({ ...crawlBody, results: [{ url: "https://example.com/product/3" }], next_cursor: "p4" }), @@ -317,6 +339,61 @@ test("crawl list and stop call the right endpoints", async () => { ); }); +/** Capture stderr (human output) for the duration of `fn`. */ +async function captureErr(fn: () => unknown): Promise { + const orig = process.stderr.write.bind(process.stderr); + let buf = ""; + process.stderr.write = ((s: string | Uint8Array) => { + buf += typeof s === "string" ? s : Buffer.from(s).toString(); + return true; + }) as typeof process.stderr.write; + try { + await fn(); + } finally { + process.stderr.write = orig; + } + return buf; +} + +test("crawl stop on an ended crawl prints its real status, not Stopped", async () => { + await withCrawlWorkspace( + () => json({ crawl_id: "c_1", status: "completed", stop_reason: "max_items" }), + async () => { + let code = -1; + const err = await captureErr(async () => { + code = await crawl.run(["stop", "c_1"], { json: false, yes: false }); + }); + assert.equal(code, 0); + assert.match(err, /already ended · status: completed/); + assert.doesNotMatch(err, /Stopped crawl/); + }, + ); +}); + +test("Ctrl-C during crawl wait says the crawl keeps running and exits 130", async () => { + let interrupted = false; + const exit = mock.method(process, "exit", () => { + interrupted = true; + }); + try { + await withCrawlWorkspace( + () => { + if (!interrupted) setImmediate(() => process.emit("SIGINT")); + return json({ ...crawlBody, status: interrupted ? "completed" : "running", results: [], next_cursor: null }); + }, + async () => { + const err = await captureErr(() => crawl.run(["wait", "c_1"], { json: false, yes: false })); + assert.deepEqual(exit.mock.calls[0]?.arguments, [130]); + assert.match(err, /c_1 is still running/); + assert.match(err, /zenrows crawl wait c_1/); + assert.equal(process.listenerCount("SIGINT"), 0, "the wait removes its SIGINT handler"); + }, + ); + } finally { + exit.mock.restore(); + } +}); + test("unknown crawl subcommand is INVALID_USAGE", async () => { const { code, out } = await run(["discover"]); assert.equal(code, 1); diff --git a/tests/e2e/crawl.e2e.test.ts b/tests/e2e/crawl.e2e.test.ts index 14d5a72..dddd43c 100644 --- a/tests/e2e/crawl.e2e.test.ts +++ b/tests/e2e/crawl.e2e.test.ts @@ -71,7 +71,7 @@ test("zenrows crawl end to end against a live API", { skip, timeout: 20 * 60_000 const dir = mkdtempSync(join(tmpdir(), "zr-crawl-e2e-")); try { // 1. Start a small crawl and wait for it. Other runs may hold the account's - // active-crawl slots: on CRAWL_QUOTA_EXCEEDED wait Retry-After and try again. + // active-job slots: on CRAWL_TOO_MANY_CRAWLS wait retry_after and try again. const deadline = Date.now() + QUOTA_RETRY_MS; let start: CliResult; for (;;) { @@ -79,10 +79,10 @@ test("zenrows crawl end to end against a live API", { skip, timeout: 20 * 60_000 "crawl", "start", START_URL!, "--depth", "1", "--max-items", "3", "--max-pages", "5", ...(INCLUDE ? ["--include", INCLUDE] : []), - "--html", "--follow", "--timeout", "600000", + "--html", "--wait", "--timeout", "600000", ]); - if (start.out.error?.code !== "CRAWL_QUOTA_EXCEEDED" || Date.now() > deadline) break; - const wait = Number(/Retry after (\d+)s/.exec(start.out.error.likely_cause)?.[1] ?? 30); + if (start.out.error?.code !== "CRAWL_TOO_MANY_CRAWLS" || Date.now() > deadline) break; + const wait = Number(start.out.error.retry_after ?? 30); log(`too_many_crawls: retrying start in ${wait}s`); await new Promise((r) => setTimeout(r, wait * 1000)); } diff --git a/tests/registry.test.ts b/tests/registry.test.ts index 3ca7fb7..145b479 100644 --- a/tests/registry.test.ts +++ b/tests/registry.test.ts @@ -52,5 +52,5 @@ test("assetRunnable reflects backend capability status", () => { const extractSkill = loadRegistry("skill").find((s) => s.name === "extract")!; assert.equal(assetRunnable(extractSkill), true); // needs extract (open beta = usable) const crawlSkill = loadRegistry("skill").find((s) => s.name === "crawl")!; - assert.equal(assetRunnable(crawlSkill), true); // needs crawl (available) + assert.equal(assetRunnable(crawlSkill), true); // needs crawl (open beta = usable) }); From 8636931d2d412291d5d3be1c9328f432e018eeb5 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Fri, 9 Oct 2026 16:47:13 +0200 Subject: [PATCH 3/3] fix(crawl): wait with --follow, as batch does fetch and extract use --wait for a different meaning, so a boolean --wait on crawl start can confuse people. crawl start now takes --follow only. The crawl wait subcommand stays. --- README.md | 4 ++-- skills/crawl/SKILL.md | 6 +++--- skills/zenrows/SKILL.md | 2 +- src/cli/commands/crawl.ts | 10 +++++----- tests/crawl-command.test.ts | 6 +++--- tests/e2e/crawl.e2e.test.ts | 2 +- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 64fc422..e748cbc 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ the start URL's registrable domain (subdomains count). Without `--html` it retur ```bash zenrows crawl start https://example.com/products/ --depth 1 \ - --max-items 20 --include /product/ --wait # start and wait for the end + --max-items 20 --include /product/ --follow # start and wait for the end zenrows crawl results --out urls.jsonl # every kept URL (paginated) zenrows crawl start --depth 2 --html # also fetch each kept page's HTML zenrows crawl content # one page's HTML @@ -185,7 +185,7 @@ is one fetch. The local policy `max_pages_per_run` caps `--max-pages` (default 1000). `--html` uses up the page budget: each kept page is one more fetch, and `--max-pages` counts it. `--include` / `--exclude` are repeatable substring filters on the URL. `results` on a running crawl returns the URLs kept so far, -with `partial: true`. `--wait` gives up after `--timeout` (default 600000 ms) +with `partial: true`. `--follow` gives up after `--timeout` (default 600000 ms) with `CRAWL_WAIT_TIMEOUT`; the crawl keeps running. Ctrl-C stops the wait, not the crawl, and exits 130; resume with `zenrows crawl wait `. diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index 7008f59..e6aac46 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -22,7 +22,7 @@ do not retry it. ``` zenrows crawl start https://example.com/products/ --depth 1 \ - --max-items 20 --max-pages 25 --include /product/ --wait --json + --max-items 20 --max-pages 25 --include /product/ --follow --json zenrows crawl results --json # every kept URL zenrows crawl results --out urls.jsonl # same, as JSONL ``` @@ -35,7 +35,7 @@ zenrows crawl results --out urls.jsonl # same, as JSONL above it the CLI refuses with `POLICY_LIMIT_EXCEEDED` before any request. - `--include` / `--exclude` are substring matches on the normalized URL and repeat: `--include /product/ --exclude ?add-to-cart`. -- `start` returns at once. `--wait` (or `zenrows crawl wait `) polls +- `start` returns at once. `--follow` (or `zenrows crawl wait `) polls until the crawl ends (default timeout 600000 ms). On `--timeout` the crawl keeps running (it is not stopped) and you get `CRAWL_WAIT_TIMEOUT` with `crawl_id`. Ctrl-C also stops the wait, not the crawl, and exits 130; @@ -46,7 +46,7 @@ zenrows crawl results --out urls.jsonl # same, as JSONL ## Page HTML ``` -zenrows crawl start --depth 1 --html --max-pages 20 --wait +zenrows crawl start --depth 1 --html --max-pages 20 --follow zenrows crawl results --json # fetched results carry content_url zenrows crawl content # one page's HTML (or pass the content_url) zenrows crawl results --download --out pages.jsonl # every URL + HTML, NDJSON diff --git a/skills/zenrows/SKILL.md b/skills/zenrows/SKILL.md index 1f885d4..595c9c9 100644 --- a/skills/zenrows/SKILL.md +++ b/skills/zenrows/SKILL.md @@ -35,7 +35,7 @@ If the user has a known URL and wants structured data: → Use Extract. (zenrows extract | --autoparse | --css) If the user has a start page (listing, index, section) but not the item URLs: - → Use Crawl. (zenrows crawl start --depth 1 --wait, + → Use Crawl. (zenrows crawl start --depth 1 --follow, then `zenrows crawl results `) If the user has many URLs (bulk): diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index 5579c75..1c3a7cc 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -3,7 +3,7 @@ * * Give Crawl one start URL and read back the URLs it finds behind it, optionally * with each page's HTML. A crawl is an async job: `start` returns at once - * (or polls with `--wait`), then `status` / `wait` / `results` / `content` + * (or polls with `--follow`), then `status` / `wait` / `results` / `content` * read it and `stop` ends it. An account without Crawl access gets 403 REQS008 * → CRAWL_NOT_ENABLED. */ @@ -54,8 +54,8 @@ export const crawl: Command = { " --exclude drop URLs containing this substring (repeatable)", " --html also fetch each kept URL's page HTML (read with `content`);", " each kept page is one more fetch counted by --max-pages", - " --wait poll until the crawl ends (Ctrl-C stops the wait, not the crawl)", - " --timeout with --wait: give up waiting after ms (default 600000)", + " --follow poll until the crawl ends (Ctrl-C stops the wait, not the crawl)", + " --timeout with --follow: give up waiting after ms (default 600000)", " --idempotency-key make a retried start create no second crawl", " --no-signup do not auto-create a Free plan account if no key exists", " status show status, coverage, stop reason / error", @@ -114,14 +114,14 @@ async function startCmd(rest: string[], ctx: RunContext): Promise { include: { type: "string", multiple: true }, exclude: { type: "string", multiple: true }, html: { type: "boolean" }, - wait: { type: "boolean" }, + follow: { type: "boolean" }, timeout: { type: "string" }, "idempotency-key": { type: "string" }, "no-signup": { type: "boolean" }, json: { type: "boolean" }, }); const json = ctx.json || values.json === true; - const wait = values.wait === true; + const wait = values.follow === true; const url = positionals[0]; if (!url) { throw new ToolkitError({ diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index 631bf08..cff1d62 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -84,7 +84,7 @@ test("crawl start rejects flags it does not declare, before any network call", a await withCrawlWorkspace( () => json(crawlBody, 202), async (calls) => { - for (const flag of ["--pagination", "--discovery", "--json-output", "--follow"]) { + for (const flag of ["--pagination", "--discovery", "--json-output", "--wait"]) { const { code, out } = await run(["start", "https://example.com/products/", "--depth", "1", flag]); assert.equal(code, 1, flag); assert.equal(out.error.code, "UNKNOWN_FLAG", flag); @@ -163,7 +163,7 @@ test("crawl start honors the domain policy and the page cap before any network c ); }); -test("crawl start --wait waits for the end and exits 1 with CRAWL_FAILED on a failed crawl", async () => { +test("crawl start --follow waits for the end and exits 1 with CRAWL_FAILED on a failed crawl", async () => { await withCrawlWorkspace( (url, init) => init?.method === "POST" @@ -176,7 +176,7 @@ test("crawl start --wait waits for the end and exits 1 with CRAWL_FAILED on a fa next_cursor: null, }), async (calls) => { - const { code, out } = await run(["start", "https://example.com/products/", "--depth", "1", "--wait"]); + const { code, out } = await run(["start", "https://example.com/products/", "--depth", "1", "--follow"]); assert.equal(code, 1); assert.equal(out.ok, false); assert.equal(out.status, "failed"); diff --git a/tests/e2e/crawl.e2e.test.ts b/tests/e2e/crawl.e2e.test.ts index dddd43c..c5fe6a5 100644 --- a/tests/e2e/crawl.e2e.test.ts +++ b/tests/e2e/crawl.e2e.test.ts @@ -79,7 +79,7 @@ test("zenrows crawl end to end against a live API", { skip, timeout: 20 * 60_000 "crawl", "start", START_URL!, "--depth", "1", "--max-items", "3", "--max-pages", "5", ...(INCLUDE ? ["--include", INCLUDE] : []), - "--html", "--wait", "--timeout", "600000", + "--html", "--follow", "--timeout", "600000", ]); if (start.out.error?.code !== "CRAWL_TOO_MANY_CRAWLS" || Date.now() > deadline) break; const wait = Number(start.out.error.retry_after ?? 30);