From 9d6b73626c88f7075b7df6318b43b33cf7ca6a60 Mon Sep 17 00:00:00 2001 From: Ander Date: Wed, 7 Oct 2026 12:58:16 +0200 Subject: [PATCH 1/2] feat(batch,extract): default batch to Adaptive Stealth Mode; fall back on REQS007 `zenrows batch create` sent tasks without `mode`, so protected sites failed with REQS002 unless the user knew to force js_render or premium_proxy. It now adds mode=auto to every task that doesn't force those flags or set `mode` itself, matching `fetch` and `extract`. It's applied per task because the API rejects a task that combines a job-level mode=auto with its own js_render or premium_proxy. `--manual` turns it off, and `--proxy-country` now works without `--premium-proxy`. `batch estimate` applies the same rule and counts auto tasks at the 25x upper bound. `zenrows extract` now also falls back to Autoparse on REQS007 (domain not prepared for Extract yet), not only AUTH010. Co-Authored-By: Claude Opus 5.5 --- agent-plugin/skills/extract/SKILL.md | 14 ++++++--- skills/batch-jobs/SKILL.md | 21 +++++++++---- src/adapters/batch.ts | 22 +++++++++++-- src/adapters/extract.ts | 5 +-- src/cli/commands/batch.ts | 39 ++++++++++++++--------- src/cli/commands/extract.ts | 6 ++-- src/core/http.ts | 12 +++++++ templates/batch-jsonl-pipeline/README.md | 6 ++-- tests/batch-adapter.test.ts | 30 +++++++++++++++++- tests/batch-command.test.ts | 40 ++++++++++++++++++++++++ tests/extract-outputs.test.ts | 18 +++++++++++ tests/http.test.ts | 21 +++++++++++++ 12 files changed, 200 insertions(+), 34 deletions(-) diff --git a/agent-plugin/skills/extract/SKILL.md b/agent-plugin/skills/extract/SKILL.md index c4567b5..dab9be1 100644 --- a/agent-plugin/skills/extract/SKILL.md +++ b/agent-plugin/skills/extract/SKILL.md @@ -17,12 +17,18 @@ Use **`extract`** when you need structured JSON fields rather than a full page: extract({ url }) # mode=auto (default) extract({ url, mode: "autoparse" }) # general Autoparse (any domain) extract({ url, mode: "css", css_extractor: '{"title":"h1","price":".price"}' }) -extract({ url, js_render: true, premium_proxy: true }) # stealth flags when needed +extract({ url, proxy_country: "US" }) # geo-targeted ``` `mode=auto` (`extract=auto`) is **open beta**: richest on enabled domains, -currently free; billing may apply later. If the domain is not enabled (AUTH010), -the tool retries once with Autoparse by default (`fallback_autoparse`). +currently free; billing may apply later. If the domain is not enabled (AUTH010) +or not prepared yet (REQS007), the tool retries once with Autoparse by default +(`fallback_autoparse`). + +Every mode uses Adaptive Stealth Mode by default: pass just the URL, even for +protected or dynamic sites. Set `js_render` or `premium_proxy` only to force a +fixed configuration; that turns Adaptive Stealth Mode off. Check `empty` in the +result: true means no field came back with a value. ## Fallback: `scrape` extraction params @@ -40,7 +46,7 @@ scrape({ url, outputs: "emails,links" }) # or "*" for all built-ins ## When to use which - **`mode=auto`**: site-tailored fields on beta-enabled domains. -- **`mode=autoparse`**: any website; also the automatic AUTH010 fallback. +- **`mode=autoparse`**: any website; also the automatic AUTH010 / REQS007 fallback. - **`mode=css`**: known fields/selectors; deterministic. - **Markdown via `scrape`**: feed clean content to an LLM yourself. diff --git a/skills/batch-jobs/SKILL.md b/skills/batch-jobs/SKILL.md index dba04f0..8f5acfc 100644 --- a/skills/batch-jobs/SKILL.md +++ b/skills/batch-jobs/SKILL.md @@ -35,14 +35,15 @@ structured data instead of raw HTML. {"url": "https://example.com/products", "extract": "auto", "external_id": "p1"} ``` -An Extract task costs the same as a regular one (1 credit at base tier), so -`batch estimate` prices it correctly. The result carries two keys: `html` (the +An Extract task costs the same as a regular one, so `batch estimate` prices it +correctly. The result carries two keys: `html` (the raw page) and `parsed` (the structured data). Validate on a single URL with `zenrows extract ` before running the full batch. ## Cloud (needs a key + beta access) ``` -zenrows batch create jobs.jsonl [--js-render] [--premium-proxy] [--proxy-country cc] [--wait] +zenrows batch create jobs.jsonl [--proxy-country cc] [--wait] +zenrows batch create jobs.jsonl --manual [--js-render] [--premium-proxy] zenrows batch status # run status + stats zenrows batch results [--status successful|failed|all] [--out results.jsonl] zenrows batch wait [--timeout ms] @@ -50,9 +51,17 @@ zenrows batch cancel # stop an in-flight run zenrows batch retry-failed # rerun only the failed tasks (new run) ``` -`create` validates the spec first, then submits. `--proxy-country` needs -`--premium-proxy` (or `mode=auto`) and is rejected before any request. Job-level -flags apply to every task; per-task keys in the JSONL override them. +Every task runs in Adaptive Stealth Mode (`mode=auto`) by default, like `fetch` +and `extract`: Zenrows enables JS rendering and premium proxies only on the pages +that need them, and bills only the configuration that succeeds. A task that sets +`js_render`, `premium_proxy` or `mode` itself keeps its own settings, and +`--js-render` / `--premium-proxy` force that configuration on every task instead. +`--manual` turns Adaptive Stealth Mode off. `batch estimate` counts auto tasks at +the 25x upper bound, so it's the most the run can cost, not the expected cost. + +`create` validates the spec first, then submits. Outside Adaptive Stealth Mode, +`--proxy-country` needs `--premium-proxy` and is rejected before any request. +Job-level flags apply to every task; per-task keys in the JSONL override them. A run ends in one of `completed`, `failed`, `stopped`, or `deleted`. `status`, `wait`, and `create --wait` exit non-zero (`ok: false` under `--json`) only for diff --git a/src/adapters/batch.ts b/src/adapters/batch.ts index 78b0042..cf5b234 100644 --- a/src/adapters/batch.ts +++ b/src/adapters/batch.ts @@ -91,6 +91,24 @@ export function estimateCredits(jobs: BatchJob[]): { credits: number; perJob: nu return { credits: perJob.reduce((a, b) => a + b, 0), perJob }; } +/** + * Adaptive Stealth Mode (mode=auto) by default, like `fetch` and `extract`: returns + * the jobs with `mode: "auto"` on every line that doesn't force `js_render` / + * `premium_proxy` or set `mode` itself, and none at all when the job-level params + * force either flag. Applied per task rather than job-level because the API rejects + * a task that combines a job-level mode=auto with its own js_render or + * premium_proxy (REQS004). + */ +export function withAdaptiveStealth(jobs: BatchJob[], jobParams: Record = {}): BatchJob[] { + const forced = (p: Record) => isTruthyFlag(p.js_render) || isTruthyFlag(p.premium_proxy); + if (forced(jobParams) || jobParams.mode != null) return jobs; + return jobs.map((job) => { + const own = { ...(job.zenrows_params ?? {}), ...job }; + if (forced(own) || own.mode != null) return job; + return { ...job, mode: "auto" }; + }); +} + /** Body for `POST /jobs` (closed, all-URLs-up-front job). */ export interface JobBody { type: "regular"; @@ -117,8 +135,8 @@ export interface JobBody { * `protected-fetch.ts::validateAutoManual`. */ export function toJobBody(jobs: BatchJob[], jobParams: Record = {}): JobBody { - assertProxyCountryPremium(jobParams, "job-level params"); - + // No separate job-level check: a job-level proxy_country is valid for each task + // in Adaptive Stealth Mode, and the per-task check below sees job ⊕ task params. const tasks = jobs.map((job, idx) => { const { url, external_id, metadata, zenrows_params } = job; const flat: Record = {}; diff --git a/src/adapters/extract.ts b/src/adapters/extract.ts index a20485a..0c4c7f1 100644 --- a/src/adapters/extract.ts +++ b/src/adapters/extract.ts @@ -8,7 +8,8 @@ * - outputs= → built-in output filters → JSON * - response_type=markdown|plaintext * - * Default method is `extract`. On AUTH010 (domain not in Extract beta) we + * Default method is `extract`. On AUTH010 / REQS007 (domain not enabled or not + * prepared for Extract) we * automatically retry once with Autoparse unless the caller opted into * `--autoparse` (or another explicit method) or disabled the fallback. */ @@ -28,7 +29,7 @@ export interface ExtractOptions extends Omit|status |results |cancel |wait |retry-failed >", help: [ "Local (no key):", - " estimate validate the spec + estimate credits", + " estimate [--manual] validate the spec + estimate credits (upper bound)", "Cloud (needs a key + Batch beta access):", " create [flags] validate, then submit the job", - " --js-render job-level: render JavaScript", - " --premium-proxy job-level: use residential IPs", - " --proxy-country job-level: geo-target (needs --premium-proxy)", + " (default) Adaptive Stealth Mode (mode=auto) on every task that", + " doesn't force js_render / premium_proxy itself", + " --manual no Adaptive Stealth Mode; plain requests unless flags below", + " --js-render job-level: force JavaScript rendering (turns auto off)", + " --premium-proxy job-level: force residential IPs (turns auto off)", + " --proxy-country job-level: geo-target (auto mode, or with --premium-proxy)", " --output job-level response_type (markdown|plaintext|pdf|html)", " --follow poll until the run finishes (alias: --wait)", " --no-signup do not auto-create a Free plan account if no key exists", @@ -77,18 +81,23 @@ export const batch: Command = { }, }; +/** Jobs as they will be sent: Adaptive Stealth Mode applied unless manual or disabled in config. */ +function effectiveJobs(jobs: BatchJob[], jobParams: Record, manual: boolean): BatchJob[] { + return manual || loadConfig().defaultMode !== "auto" ? jobs : withAdaptiveStealth(jobs, jobParams); +} + function estimateCmd(rest: string[], ctx: RunContext): number { - const { positionals } = parse(rest, {}); + const { values, positionals } = parse(rest, { manual: { type: "boolean" } }); const file = positionals[0]; if (!file) throw needFile(); const v = validateJsonl(file); - const est = estimateCredits(v.jobs); + const est = estimateCredits(effectiveJobs(v.jobs, {}, values.manual === true)); if (ctx.json) { log.out(JSON.stringify({ ok: v.errors.length === 0, ...v, estimatedCredits: est.credits }, null, 2)); } else { log.info(c(ANSI.bold, `Job spec: ${file}`)); log.info(`valid jobs: ${v.validJobs}/${v.totalLines}`); - log.info(`estimated credits (1x basic / 5x js / 10x proxy / 25x both): ${est.credits}`); + log.info(`estimated credits, upper bound (1x basic / 5x js / 10x proxy / 25x both or auto): ${est.credits}`); if (v.errors.length) { log.warn(`${v.errors.length} invalid line(s):`); v.errors.slice(0, 10).forEach((e) => log.dim(` line ${e.line}: ${e.reason}`)); @@ -104,6 +113,7 @@ async function createCmd(rest: string[], ctx: RunContext): Promise { "js-render": { type: "boolean" }, "premium-proxy": { type: "boolean" }, "proxy-country": { type: "string" }, + manual: { type: "boolean" }, output: { type: "string" }, follow: { type: "boolean" }, wait: { type: "boolean" }, // back-compat alias for --follow @@ -137,11 +147,6 @@ async function createCmd(rest: string[], ctx: RunContext): Promise { // 1. every task URL must pass the allow/deny domain policy, // 2. the run must fit the batch-only page/credit caps (a batch is the one // primitive that fans out into many requests, so the caps bind here). - const policy = loadPolicy(); - for (const job of v.jobs) assertDomainAllowed(job.url, policy); - const est = estimateCredits(v.jobs); - assertWithinLimits({ pages: v.validJobs, credits: est.credits }, policy, "batch"); - const jobParams: Record = {}; if (values["js-render"] === true) jobParams.js_render = true; if (values["premium-proxy"] === true) jobParams.premium_proxy = true; @@ -149,9 +154,15 @@ async function createCmd(rest: string[], ctx: RunContext): Promise { if (proxyCountry) jobParams.proxy_country = proxyCountry; const responseType = normalizeOutput(asString(values.output)); if (responseType) jobParams.response_type = responseType; + const jobs = effectiveJobs(v.jobs, jobParams, values.manual === true); + + const policy = loadPolicy(); + for (const job of jobs) assertDomainAllowed(job.url, policy); + const est = estimateCredits(jobs); + assertWithinLimits({ pages: v.validJobs, credits: est.credits }, policy, "batch"); // toJobBody validates proxy_country/premium_proxy BEFORE any HTTP call. - const body = toJobBody(v.jobs, jobParams); + const body = toJobBody(jobs, jobParams); const apiKey = await ensureApiKey( values["no-signup"] ? { ...policy, auto_signup: false } : policy, { diff --git a/src/cli/commands/extract.ts b/src/cli/commands/extract.ts index 4639bd4..444829d 100644 --- a/src/cli/commands/extract.ts +++ b/src/cli/commands/extract.ts @@ -2,7 +2,7 @@ * `zenrows extract [flags]` → Extract (structured extraction on `/v1/`). * * Default: `extract=auto` (site-tailored Extract, open beta). If the domain is - * not enabled (AUTH010), falls back once to Autoparse. Opt into Autoparse + * not enabled or prepared (AUTH010 / REQS007), falls back once to Autoparse. Opt into Autoparse * directly with `--autoparse`. Other methods: `--css`, `--outputs`, `--output`. */ import { ensureApiKey } from "../../core/ensure-key.ts"; @@ -24,7 +24,7 @@ export const extract: Command = { usage: "zenrows extract [--autoparse | --css | --outputs | --output md|text] [flags]", help: [ "Methods:", - " (default) extract=auto — site-tailored Extract (open beta); falls back to Autoparse if the domain is not enabled", + " (default) extract=auto — site-tailored Extract (open beta); falls back to Autoparse if the domain is not enabled or prepared", " --autoparse general-purpose Autoparse (any domain, no Extract fallback)", " --css CSS selector map, e.g. '{\"title\":\"h1\",\"price\":\".price\"}'", " --outputs built-in output filters → JSON: emails, phone_numbers, headings,", @@ -130,7 +130,7 @@ export const extract: Command = { delete (safeParams as Record).apikey; if (outcome.fellBackToAutoparse) { - log.info("Extract not enabled for this domain — fell back to Autoparse."); + log.info("Extract not available for this domain yet — fell back to Autoparse."); } const runDir = writeRun( diff --git a/src/core/http.ts b/src/core/http.ts index 92ba89e..1cc98eb 100644 --- a/src/core/http.ts +++ b/src/core/http.ts @@ -139,6 +139,18 @@ export async function scrape( redactedUrl: redacted, }; + // REQS007 on extract=auto: the domain hasn't been prepared for Extract yet. + // Same recovery as AUTH010 (the extract adapter falls back to autoparse), so it + // maps to the same code. Checked before the 403 auth branch below. + if (res.status === 403 && params.extract !== undefined && zrErrorCode(body) === "REQS007") { + throw new ToolkitError({ + code: "EXTRACT_DOMAIN_NOT_ENABLED", + message: "Extract has not been prepared for this domain yet.", + likely_cause: zrErrorDetail(body) ?? "This domain is not prepared for Extract.", + next_action: "Retry with --autoparse for general-purpose extraction on any site.", + suggested_commands: [`zenrows extract ${params.url} --autoparse`], + }); + } if (res.status === 401 || res.status === 403) { // Distinguish Zenrows auth errors from target-site 403s. Zenrows auth // failures surface AUTH00x codes in the body. diff --git a/templates/batch-jsonl-pipeline/README.md b/templates/batch-jsonl-pipeline/README.md index 15f2d0b..c923fad 100644 --- a/templates/batch-jsonl-pipeline/README.md +++ b/templates/batch-jsonl-pipeline/README.md @@ -29,7 +29,9 @@ zenrows batch retry-failed # rerun only the failed zenrows batch cancel # stop an in-flight run ``` -Job-level scrape options come from flags (`--js-render`, `--premium-proxy`, -`--proxy-country`, `--output`); per-line keys override them for that URL. +Tasks use Adaptive Stealth Mode (`mode=auto`) by default. Job-level scrape +options come from flags (`--js-render`, `--premium-proxy`, `--proxy-country`, +`--output`, or `--manual` to turn Adaptive Stealth Mode off); per-line keys +override them for that URL. Without beta access yet, validate/estimate locally or fan out with `zenrows fetch` per URL on a small sample first. diff --git a/tests/batch-adapter.test.ts b/tests/batch-adapter.test.ts index 4339811..5c26806 100644 --- a/tests/batch-adapter.test.ts +++ b/tests/batch-adapter.test.ts @@ -3,7 +3,7 @@ import assert from "node:assert/strict"; import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { estimateCredits, toJobBody, validateJsonl, type BatchJob } from "../src/adapters/batch.ts"; +import { estimateCredits, toJobBody, validateJsonl, withAdaptiveStealth, type BatchJob } from "../src/adapters/batch.ts"; import { ToolkitError } from "../src/core/errors.ts"; function specFile(lines: string[]): { file: string; cleanup: () => void } { @@ -83,3 +83,31 @@ test("toJobBody allows proxy_country in mode=auto without premium_proxy", () => const body = toJobBody(jobs, {}); assert.deepEqual(body.tasks[0]!.zenrows_params, { proxy_country: "us", mode: "auto" }); }); + +test("withAdaptiveStealth adds mode=auto to lines that don't force flags or set mode", () => { + const jobs: BatchJob[] = [ + { url: "https://a.com" }, + { url: "https://b.com", js_render: true }, + { url: "https://c.com", zenrows_params: { premium_proxy: "true" } }, + { url: "https://d.com", mode: "auto" }, + { url: "https://e.com", autoparse: true, js_render: false }, + ]; + assert.deepEqual( + withAdaptiveStealth(jobs).map((j) => j.mode), + ["auto", undefined, undefined, "auto", "auto"], + ); + assert.equal(jobs[0]!.mode, undefined, "input is not mutated"); +}); + +test("withAdaptiveStealth leaves every line alone when job-level params force a flag", () => { + const jobs: BatchJob[] = [{ url: "https://a.com" }]; + assert.equal(withAdaptiveStealth(jobs, { js_render: true })[0]!.mode, undefined); + assert.equal(withAdaptiveStealth(jobs, { premium_proxy: true })[0]!.mode, undefined); +}); + +test("toJobBody accepts proxy_country without premium_proxy once Adaptive Stealth Mode is applied", () => { + const jobs = withAdaptiveStealth([{ url: "https://a.com" }], { proxy_country: "us" }); + const body = toJobBody(jobs, { proxy_country: "us" }); + assert.equal(body.tasks[0]!.zenrows_params?.mode, "auto"); + assert.equal(body.zenrows_params?.proxy_country, "us"); +}); diff --git a/tests/batch-command.test.ts b/tests/batch-command.test.ts index 1be8e4e..4375312 100644 --- a/tests/batch-command.test.ts +++ b/tests/batch-command.test.ts @@ -256,3 +256,43 @@ test("batch create --wait whose run the cap stops exits 1 with ok:false", async assert.equal(j.failure_reason, "api_key_cap_reached"); }); }); + +test("batch create sends mode=auto per task by default; --manual and --js-render opt out", async () => { + const bodies: Array<{ tasks: Array<{ zenrows_params?: Record }>; zenrows_params?: Record }> = []; + await withBatchWorkspace({}, async () => { + const orig = globalThis.fetch; + globalThis.fetch = (async (_u: unknown, init?: RequestInit) => { + if (init?.body) bodies.push(JSON.parse(String(init.body))); + return new Response(JSON.stringify({ job_id: "j1", latest_run: { status: "queued" } }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }) as unknown as typeof fetch; + try { + const file = writeSpec(["https://ok.example/a", "https://ok.example/b"]); + await captureOut(() => batch.run(["create", file], ctx)); + await captureOut(() => batch.run(["create", file, "--manual"], ctx)); + await captureOut(() => batch.run(["create", file, "--js-render"], ctx)); + await captureOut(() => batch.run(["create", file, "--proxy-country", "us"], ctx)); + } finally { + globalThis.fetch = orig; + } + }); + assert.equal(bodies.length, 4); + assert.deepEqual(bodies[0]!.tasks.map((t) => t.zenrows_params?.mode), ["auto", "auto"]); + assert.deepEqual(bodies[1]!.tasks.map((t) => t.zenrows_params?.mode), [undefined, undefined]); + assert.deepEqual(bodies[2]!.tasks.map((t) => t.zenrows_params?.mode), [undefined, undefined]); + assert.equal(bodies[2]!.zenrows_params?.js_render, "true"); + assert.deepEqual(bodies[3]!.tasks.map((t) => t.zenrows_params?.mode), ["auto", "auto"]); + assert.equal(bodies[3]!.zenrows_params?.proxy_country, "us"); +}); + +test("batch estimate counts Adaptive Stealth Mode as the upper bound, --manual as basic", async () => { + await withBatchWorkspace({}, async () => { + const file = writeSpec(["https://ok.example/a", "https://ok.example/b"]); + const auto = JSON.parse(await captureOut(() => batch.run(["estimate", file], ctx))) as { estimatedCredits: number }; + const manual = JSON.parse(await captureOut(() => batch.run(["estimate", file, "--manual"], ctx))) as { estimatedCredits: number }; + assert.equal(auto.estimatedCredits, 50); + assert.equal(manual.estimatedCredits, 2); + }); +}); diff --git a/tests/extract-outputs.test.ts b/tests/extract-outputs.test.ts index 12f7692..3ddb694 100644 --- a/tests/extract-outputs.test.ts +++ b/tests/extract-outputs.test.ts @@ -203,6 +203,24 @@ test("runExtract falls back to autoparse on AUTH010 for extract=auto", async () }); }); +test("runExtract falls back to autoparse on REQS007 (domain not prepared) for extract=auto", async () => { + const REQS007 = JSON.stringify({ code: "REQS007", detail: "Extract has not been prepared for x.com yet.", status: 403 }); + const calls: string[] = []; + await withFetchHandler((url) => { + calls.push(url); + if (url.includes("extract=auto")) { + return new Response(REQS007, { status: 403, headers: { "content-type": "application/json" } }); + } + return new Response('{"title":"fallback"}', { status: 200, headers: { "content-type": "application/json" } }); + }, async () => { + const outcome = await runExtract({ url: "https://x.com" }, cfg, defaultPolicy(), "testkey"); + assert.equal(outcome.method, "autoparse"); + assert.equal(outcome.fellBackToAutoparse, true); + assert.equal(calls.length, 2); + assert.match(calls[1]!, /autoparse=true/); + }); +}); + test("runExtract does not fall back when fallbackAutoparse is false", async () => { await withFetch(AUTH010, async () => { await assert.rejects( diff --git a/tests/http.test.ts b/tests/http.test.ts index 48f989a..e7dba41 100644 --- a/tests/http.test.ts +++ b/tests/http.test.ts @@ -165,6 +165,27 @@ test("scrape maps AUTH010 on extract=auto to EXTRACT_DOMAIN_NOT_ENABLED", async ); }); +test("scrape maps REQS007 on extract=auto to EXTRACT_DOMAIN_NOT_ENABLED", async () => { + const REQS007 = JSON.stringify({ + code: "REQS007", + detail: "Extract has not been prepared for example.net yet.", + status: 403, + title: "Extract not prepared (REQS007)", + }); + await withFetch( + () => new Response(REQS007, { status: 403, headers: { "content-type": "application/json" } }), + async () => { + await assert.rejects( + () => scrape("https://api.zenrows.com/v1/", "test-key", { url: "https://example.net", extract: "auto" }), + (err: unknown) => { + assert.equal((err as { code: string }).code, "EXTRACT_DOMAIN_NOT_ENABLED"); + return true; + }, + ); + }, + ); +}); + test("scrape treats AUTH010 without extract as a credits/plan error, not EXTRACT_DOMAIN", async () => { // AUTH010 is also used for other plan-gated features; only map it to // EXTRACT_DOMAIN_NOT_ENABLED when the request asked for extract=auto. From e0115d46ee7c9d2d057cd83a4c299304d2663966 Mon Sep 17 00:00:00 2001 From: Ander Date: Wed, 7 Oct 2026 13:10:04 +0200 Subject: [PATCH 2/2] test(batch,http): cover --manual geo guard, defaultMode manual and REQS007 without extract Co-Authored-By: Claude Opus 5.5 --- tests/batch-command.test.ts | 38 +++++++++++++++++++++++++++++++++++++ tests/http.test.ts | 16 ++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/tests/batch-command.test.ts b/tests/batch-command.test.ts index 4375312..44ea664 100644 --- a/tests/batch-command.test.ts +++ b/tests/batch-command.test.ts @@ -6,6 +6,7 @@ import { batch } from "../src/cli/commands/batch.ts"; import { createWorkspace } from "../src/core/workspace.ts"; import { savePolicy, defaultPolicy } from "../src/core/policy.ts"; import { saveApiKey } from "../src/core/auth.ts"; +import { loadConfig, saveConfig } from "../src/core/config.ts"; import { tempRoot } from "./helpers.ts"; const ctx = { json: true, yes: false }; @@ -296,3 +297,40 @@ test("batch estimate counts Adaptive Stealth Mode as the upper bound, --manual a assert.equal(manual.estimatedCredits, 2); }); }); + +test("batch create --manual --proxy-country without --premium-proxy is rejected before any network call", async () => { + await withBatchWorkspace({}, async (didFetch) => { + const file = writeSpec(["https://ok.example/a"]); + let code = -1; + const out = await captureOut(async () => { + code = await batch.run(["create", file, "--manual", "--proxy-country", "us"], ctx); + }); + assert.equal(code, 1); + assert.equal((JSON.parse(out) as { error: { code: string } }).error.code, "PARAM_PROXY_COUNTRY_REQUIRES_PREMIUM"); + assert.equal(didFetch(), false); + }); +}); + +test("batch create respects defaultMode: manual in config (no mode=auto)", async () => { + const bodies: Array<{ tasks: Array<{ zenrows_params?: Record }> }> = []; + await withBatchWorkspace({}, async () => { + saveConfig({ ...loadConfig(), defaultMode: "manual" }); + const orig = globalThis.fetch; + globalThis.fetch = (async (_u: unknown, init?: RequestInit) => { + if (init?.body) bodies.push(JSON.parse(String(init.body))); + return new Response(JSON.stringify({ job_id: "j1", latest_run: { status: "queued" } }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }) as unknown as typeof fetch; + try { + await captureOut(() => batch.run(["create", writeSpec(["https://ok.example/a"])], ctx)); + const est = JSON.parse(await captureOut(() => batch.run(["estimate", "jobs.jsonl"], ctx))) as { estimatedCredits: number }; + assert.equal(est.estimatedCredits, 1); + } finally { + globalThis.fetch = orig; + } + }); + assert.equal(bodies.length, 1); + assert.equal(bodies[0]!.tasks[0]!.zenrows_params?.mode, undefined); +}); diff --git a/tests/http.test.ts b/tests/http.test.ts index e7dba41..c40bcdf 100644 --- a/tests/http.test.ts +++ b/tests/http.test.ts @@ -186,6 +186,22 @@ test("scrape maps REQS007 on extract=auto to EXTRACT_DOMAIN_NOT_ENABLED", async ); }); +test("scrape does not map REQS007 to EXTRACT_DOMAIN_NOT_ENABLED without extract=auto", async () => { + const REQS007 = JSON.stringify({ code: "REQS007", detail: "Extract has not been prepared for example.net yet.", status: 403 }); + await withFetch( + () => new Response(REQS007, { status: 403, headers: { "content-type": "application/json" } }), + async () => { + await assert.rejects( + () => scrape("https://api.zenrows.com/v1/", "test-key", { url: "https://example.net", autoparse: true }), + (err: unknown) => { + assert.notEqual((err as { code: string }).code, "EXTRACT_DOMAIN_NOT_ENABLED"); + return true; + }, + ); + }, + ); +}); + test("scrape treats AUTH010 without extract as a credits/plan error, not EXTRACT_DOMAIN", async () => { // AUTH010 is also used for other plan-gated features; only map it to // EXTRACT_DOMAIN_NOT_ENABLED when the request asked for extract=auto.