feat(batch,extract): default batch to Adaptive Stealth Mode; fall back on REQS007

agent-plugin/skills/extract/SKILL.md

@@ -17,12 +17,18 @@ Use **`extract`** when you need structured JSON fields rather than a full page:
 extract({ url })                                    # mode=auto (default)
 extract({ url, mode: "autoparse" })                 # general Autoparse (any domain)
 extract({ url, mode: "css", css_extractor: '{"title":"h1","price":".price"}' })
-extract({ url, js_render: true, premium_proxy: true })  # stealth flags when needed
+extract({ url, proxy_country: "US" })               # geo-targeted
 ```
 
 `mode=auto` (`extract=auto`) is **open beta**: richest on enabled domains,
-currently free; billing may apply later. If the domain is not enabled (AUTH010),
-the tool retries once with Autoparse by default (`fallback_autoparse`).
+currently free; billing may apply later. If the domain is not enabled (AUTH010)
+or not prepared yet (REQS007), the tool retries once with Autoparse by default
+(`fallback_autoparse`).
+
+Every mode uses Adaptive Stealth Mode by default: pass just the URL, even for
+protected or dynamic sites. Set `js_render` or `premium_proxy` only to force a
+fixed configuration; that turns Adaptive Stealth Mode off. Check `empty` in the
+result: true means no field came back with a value.
 
 ## Fallback: `scrape` extraction params
 
@@ -40,7 +46,7 @@ scrape({ url, outputs: "emails,links" })            # or "*" for all built-ins
 ## When to use which
 
 - **`mode=auto`**: site-tailored fields on beta-enabled domains.
-- **`mode=autoparse`**: any website; also the automatic AUTH010 fallback.
+- **`mode=autoparse`**: any website; also the automatic AUTH010 / REQS007 fallback.
 - **`mode=css`**: known fields/selectors; deterministic.
 - **Markdown via `scrape`**: feed clean content to an LLM yourself.
 

skills/batch-jobs/SKILL.md

@@ -35,24 +35,33 @@ structured data instead of raw HTML.
 {"url": "https://example.com/products", "extract": "auto", "external_id": "p1"}
 ```
 
-An Extract task costs the same as a regular one (1 credit at base tier), so
-`batch estimate` prices it correctly. The result carries two keys: `html` (the
+An Extract task costs the same as a regular one, so `batch estimate` prices it
+correctly. The result carries two keys: `html` (the
 raw page) and `parsed` (the structured data). Validate on a single URL with
 `zenrows extract <url>` before running the full batch.
 
 ## Cloud (needs a key + beta access)
 ```
-zenrows batch create jobs.jsonl [--js-render] [--premium-proxy] [--proxy-country cc] [--wait]
+zenrows batch create jobs.jsonl [--proxy-country cc] [--wait]
+zenrows batch create jobs.jsonl --manual [--js-render] [--premium-proxy]
 zenrows batch status <id>             # run status + stats
 zenrows batch results <id> [--status successful|failed|all] [--out results.jsonl]
 zenrows batch wait <id> [--timeout ms]
 zenrows batch cancel <id>             # stop an in-flight run
 zenrows batch retry-failed <id>       # rerun only the failed tasks (new run)
 ```
 
-`create` validates the spec first, then submits. `--proxy-country` needs
-`--premium-proxy` (or `mode=auto`) and is rejected before any request. Job-level
-flags apply to every task; per-task keys in the JSONL override them.
+Every task runs in Adaptive Stealth Mode (`mode=auto`) by default, like `fetch`
+and `extract`: Zenrows enables JS rendering and premium proxies only on the pages
+that need them, and bills only the configuration that succeeds. A task that sets
+`js_render`, `premium_proxy` or `mode` itself keeps its own settings, and
+`--js-render` / `--premium-proxy` force that configuration on every task instead.
+`--manual` turns Adaptive Stealth Mode off. `batch estimate` counts auto tasks at
+the 25x upper bound, so it's the most the run can cost, not the expected cost.
+
+`create` validates the spec first, then submits. Outside Adaptive Stealth Mode,
+`--proxy-country` needs `--premium-proxy` and is rejected before any request.
+Job-level flags apply to every task; per-task keys in the JSONL override them.
 
 A run ends in one of `completed`, `failed`, `stopped`, or `deleted`. `status`,
 `wait`, and `create --wait` exit non-zero (`ok: false` under `--json`) only for

src/adapters/batch.ts

@@ -91,6 +91,24 @@ export function estimateCredits(jobs: BatchJob[]): { credits: number; perJob: nu
   return { credits: perJob.reduce((a, b) => a + b, 0), perJob };
 }
 
+/**
+ * Adaptive Stealth Mode (mode=auto) by default, like `fetch` and `extract`: returns
+ * the jobs with `mode: "auto"` on every line that doesn't force `js_render` /
+ * `premium_proxy` or set `mode` itself, and none at all when the job-level params
+ * force either flag. Applied per task rather than job-level because the API rejects
+ * a task that combines a job-level mode=auto with its own js_render or
+ * premium_proxy (REQS004).
+ */
+export function withAdaptiveStealth(jobs: BatchJob[], jobParams: Record<string, unknown> = {}): BatchJob[] {
+  const forced = (p: Record<string, unknown>) => isTruthyFlag(p.js_render) || isTruthyFlag(p.premium_proxy);
+  if (forced(jobParams) || jobParams.mode != null) return jobs;
+  return jobs.map((job) => {
+    const own = { ...(job.zenrows_params ?? {}), ...job };
+    if (forced(own) || own.mode != null) return job;
+    return { ...job, mode: "auto" };
+  });
+}
+
 /** Body for `POST /jobs` (closed, all-URLs-up-front job). */
 export interface JobBody {
   type: "regular";
@@ -117,8 +135,8 @@ export interface JobBody {
  * `protected-fetch.ts::validateAutoManual`.
  */
 export function toJobBody(jobs: BatchJob[], jobParams: Record<string, unknown> = {}): JobBody {
-  assertProxyCountryPremium(jobParams, "job-level params");
-
+  // No separate job-level check: a job-level proxy_country is valid for each task
+  // in Adaptive Stealth Mode, and the per-task check below sees job ⊕ task params.
   const tasks = jobs.map((job, idx) => {
     const { url, external_id, metadata, zenrows_params } = job;
     const flat: Record<string, unknown> = {};

src/adapters/extract.ts

@@ -8,7 +8,8 @@
  *   - outputs=<filters>    → built-in output filters → JSON
  *   - response_type=markdown|plaintext
  *
- * Default method is `extract`. On AUTH010 (domain not in Extract beta) we
+ * Default method is `extract`. On AUTH010 / REQS007 (domain not enabled or not
+ * prepared for Extract) we
  * automatically retry once with Autoparse unless the caller opted into
  * `--autoparse` (or another explicit method) or disabled the fallback.
  */
@@ -28,7 +29,7 @@ export interface ExtractOptions extends Omit<FetchOptions, "autoparse" | "cssExt
   validate?: boolean;
   /**
    * When method is `extract` (default), retry once with Autoparse if the domain
-   * is not enabled for Extract (AUTH010). Defaults to true.
+   * is not enabled or prepared for Extract (AUTH010 / REQS007). Defaults to true.
    */
   fallbackAutoparse?: boolean;
 }

src/cli/commands/batch.ts

@@ -9,7 +9,8 @@
 import { statSync } from "node:fs";
 import { join } from "node:path";
 import { log, ANSI, c } from "../../core/logger.ts";
-import { estimateCredits, toJobBody, validateJsonl } from "../../adapters/batch.ts";
+import { estimateCredits, toJobBody, validateJsonl, withAdaptiveStealth, type BatchJob } from "../../adapters/batch.ts";
+import { loadConfig } from "../../core/config.ts";
 import { requireApiKey } from "../../core/auth.ts";
 import { ensureApiKey } from "../../core/ensure-key.ts";
 import { assertUsable } from "../../core/capabilities.ts";
@@ -26,12 +27,15 @@ export const batch: Command = {
   usage: "zenrows batch <estimate|create <file.jsonl>|status <id>|results <id>|cancel <id>|wait <id>|retry-failed <id>>",
   help: [
     "Local (no key):",
-    "  estimate <file.jsonl>            validate the spec + estimate credits",
+    "  estimate <file.jsonl> [--manual] validate the spec + estimate credits (upper bound)",
     "Cloud (needs a key + Batch beta access):",
     "  create <file.jsonl> [flags]      validate, then submit the job",
-    "    --js-render                    job-level: render JavaScript",
-    "    --premium-proxy                job-level: use residential IPs",
-    "    --proxy-country <cc>           job-level: geo-target (needs --premium-proxy)",
+    "    (default)                      Adaptive Stealth Mode (mode=auto) on every task that",
+    "                                   doesn't force js_render / premium_proxy itself",
+    "    --manual                       no Adaptive Stealth Mode; plain requests unless flags below",
+    "    --js-render                    job-level: force JavaScript rendering (turns auto off)",
+    "    --premium-proxy                job-level: force residential IPs (turns auto off)",
+    "    --proxy-country <cc>           job-level: geo-target (auto mode, or with --premium-proxy)",
     "    --output <fmt>                 job-level response_type (markdown|plaintext|pdf|html)",
     "    --follow                       poll until the run finishes (alias: --wait)",
     "    --no-signup                    do not auto-create a Free plan account if no key exists",
@@ -77,18 +81,23 @@ export const batch: Command = {
   },
 };
 
+/** Jobs as they will be sent: Adaptive Stealth Mode applied unless manual or disabled in config. */
+function effectiveJobs(jobs: BatchJob[], jobParams: Record<string, unknown>, manual: boolean): BatchJob[] {
+  return manual || loadConfig().defaultMode !== "auto" ? jobs : withAdaptiveStealth(jobs, jobParams);
+}
+
 function estimateCmd(rest: string[], ctx: RunContext): number {
-  const { positionals } = parse(rest, {});
+  const { values, positionals } = parse(rest, { manual: { type: "boolean" } });
   const file = positionals[0];
   if (!file) throw needFile();
   const v = validateJsonl(file);
-  const est = estimateCredits(v.jobs);
+  const est = estimateCredits(effectiveJobs(v.jobs, {}, values.manual === true));
   if (ctx.json) {
     log.out(JSON.stringify({ ok: v.errors.length === 0, ...v, estimatedCredits: est.credits }, null, 2));
   } else {
     log.info(c(ANSI.bold, `Job spec: ${file}`));
     log.info(`valid jobs: ${v.validJobs}/${v.totalLines}`);
-    log.info(`estimated credits (1x basic / 5x js / 10x proxy / 25x both): ${est.credits}`);
+    log.info(`estimated credits, upper bound (1x basic / 5x js / 10x proxy / 25x both or auto): ${est.credits}`);
     if (v.errors.length) {
       log.warn(`${v.errors.length} invalid line(s):`);
       v.errors.slice(0, 10).forEach((e) => log.dim(`  line ${e.line}: ${e.reason}`));
@@ -104,6 +113,7 @@ async function createCmd(rest: string[], ctx: RunContext): Promise<number> {
     "js-render": { type: "boolean" },
     "premium-proxy": { type: "boolean" },
     "proxy-country": { type: "string" },
+    manual: { type: "boolean" },
     output: { type: "string" },
     follow: { type: "boolean" },
     wait: { type: "boolean" }, // back-compat alias for --follow
@@ -137,21 +147,22 @@ async function createCmd(rest: string[], ctx: RunContext): Promise<number> {
   //  1. every task URL must pass the allow/deny domain policy,
   //  2. the run must fit the batch-only page/credit caps (a batch is the one
   //     primitive that fans out into many requests, so the caps bind here).
-  const policy = loadPolicy();
-  for (const job of v.jobs) assertDomainAllowed(job.url, policy);
-  const est = estimateCredits(v.jobs);
-  assertWithinLimits({ pages: v.validJobs, credits: est.credits }, policy, "batch");
-
   const jobParams: Record<string, unknown> = {};
   if (values["js-render"] === true) jobParams.js_render = true;
   if (values["premium-proxy"] === true) jobParams.premium_proxy = true;
   const proxyCountry = asString(values["proxy-country"]);
   if (proxyCountry) jobParams.proxy_country = proxyCountry;
   const responseType = normalizeOutput(asString(values.output));
   if (responseType) jobParams.response_type = responseType;
+  const jobs = effectiveJobs(v.jobs, jobParams, values.manual === true);
+
+  const policy = loadPolicy();
+  for (const job of jobs) assertDomainAllowed(job.url, policy);
+  const est = estimateCredits(jobs);
+  assertWithinLimits({ pages: v.validJobs, credits: est.credits }, policy, "batch");
 
   // toJobBody validates proxy_country/premium_proxy BEFORE any HTTP call.
-  const body = toJobBody(v.jobs, jobParams);
+  const body = toJobBody(jobs, jobParams);
   const apiKey = await ensureApiKey(
     values["no-signup"] ? { ...policy, auto_signup: false } : policy,
     {

src/cli/commands/extract.ts

@@ -2,7 +2,7 @@
  * `zenrows extract <url> [flags]` → Extract (structured extraction on `/v1/`).
  *
  * Default: `extract=auto` (site-tailored Extract, open beta). If the domain is
- * not enabled (AUTH010), falls back once to Autoparse. Opt into Autoparse
+ * not enabled or prepared (AUTH010 / REQS007), falls back once to Autoparse. Opt into Autoparse
  * directly with `--autoparse`. Other methods: `--css`, `--outputs`, `--output`.
  */
 import { ensureApiKey } from "../../core/ensure-key.ts";
@@ -24,7 +24,7 @@ export const extract: Command = {
   usage: "zenrows extract <url> [--autoparse | --css <json> | --outputs <filters> | --output md|text] [flags]",
   help: [
     "Methods:",
-    "  (default)              extract=auto — site-tailored Extract (open beta); falls back to Autoparse if the domain is not enabled",
+    "  (default)              extract=auto — site-tailored Extract (open beta); falls back to Autoparse if the domain is not enabled or prepared",
     "  --autoparse            general-purpose Autoparse (any domain, no Extract fallback)",
     "  --css <json>           CSS selector map, e.g. '{\"title\":\"h1\",\"price\":\".price\"}'",
     "  --outputs <list>       built-in output filters → JSON: emails, phone_numbers, headings,",
@@ -130,7 +130,7 @@ export const extract: Command = {
       delete (safeParams as Record<string, unknown>).apikey;
 
       if (outcome.fellBackToAutoparse) {
-        log.info("Extract not enabled for this domain — fell back to Autoparse.");
+        log.info("Extract not available for this domain yet — fell back to Autoparse.");
       }
 
       const runDir = writeRun(

src/core/http.ts

@@ -139,6 +139,18 @@ export async function scrape(
     redactedUrl: redacted,
   };
 
+  // REQS007 on extract=auto: the domain hasn't been prepared for Extract yet.
+  // Same recovery as AUTH010 (the extract adapter falls back to autoparse), so it
+  // maps to the same code. Checked before the 403 auth branch below.
+  if (res.status === 403 && params.extract !== undefined && zrErrorCode(body) === "REQS007") {
+    throw new ToolkitError({
+      code: "EXTRACT_DOMAIN_NOT_ENABLED",
+      message: "Extract has not been prepared for this domain yet.",
+      likely_cause: zrErrorDetail(body) ?? "This domain is not prepared for Extract.",
+      next_action: "Retry with --autoparse for general-purpose extraction on any site.",
+      suggested_commands: [`zenrows extract ${params.url} --autoparse`],
+    });
+  }
   if (res.status === 401 || res.status === 403) {
     // Distinguish Zenrows auth errors from target-site 403s. Zenrows auth
     // failures surface AUTH00x codes in the body.

templates/batch-jsonl-pipeline/README.md

@@ -29,7 +29,9 @@ zenrows batch retry-failed <job-id>                     # rerun only the failed
 zenrows batch cancel <job-id>                           # stop an in-flight run
 ```
 
-Job-level scrape options come from flags (`--js-render`, `--premium-proxy`,
-`--proxy-country`, `--output`); per-line keys override them for that URL.
+Tasks use Adaptive Stealth Mode (`mode=auto`) by default. Job-level scrape
+options come from flags (`--js-render`, `--premium-proxy`, `--proxy-country`,
+`--output`, or `--manual` to turn Adaptive Stealth Mode off); per-line keys
+override them for that URL.
 Without beta access yet, validate/estimate locally or fan out with
 `zenrows fetch` per URL on a small sample first.

tests/batch-adapter.test.ts

@@ -3,7 +3,7 @@ import assert from "node:assert/strict";
 import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { estimateCredits, toJobBody, validateJsonl, type BatchJob } from "../src/adapters/batch.ts";
+import { estimateCredits, toJobBody, validateJsonl, withAdaptiveStealth, type BatchJob } from "../src/adapters/batch.ts";
 import { ToolkitError } from "../src/core/errors.ts";
 
 function specFile(lines: string[]): { file: string; cleanup: () => void } {
@@ -83,3 +83,31 @@ test("toJobBody allows proxy_country in mode=auto without premium_proxy", () =>
   const body = toJobBody(jobs, {});
   assert.deepEqual(body.tasks[0]!.zenrows_params, { proxy_country: "us", mode: "auto" });
 });
+
+test("withAdaptiveStealth adds mode=auto to lines that don't force flags or set mode", () => {
+  const jobs: BatchJob[] = [
+    { url: "https://a.com" },
+    { url: "https://b.com", js_render: true },
+    { url: "https://c.com", zenrows_params: { premium_proxy: "true" } },
+    { url: "https://d.com", mode: "auto" },
+    { url: "https://e.com", autoparse: true, js_render: false },
+  ];
+  assert.deepEqual(
+    withAdaptiveStealth(jobs).map((j) => j.mode),
+    ["auto", undefined, undefined, "auto", "auto"],
+  );
+  assert.equal(jobs[0]!.mode, undefined, "input is not mutated");
+});
+
+test("withAdaptiveStealth leaves every line alone when job-level params force a flag", () => {
+  const jobs: BatchJob[] = [{ url: "https://a.com" }];
+  assert.equal(withAdaptiveStealth(jobs, { js_render: true })[0]!.mode, undefined);
+  assert.equal(withAdaptiveStealth(jobs, { premium_proxy: true })[0]!.mode, undefined);
+});
+
+test("toJobBody accepts proxy_country without premium_proxy once Adaptive Stealth Mode is applied", () => {
+  const jobs = withAdaptiveStealth([{ url: "https://a.com" }], { proxy_country: "us" });
+  const body = toJobBody(jobs, { proxy_country: "us" });
+  assert.equal(body.tasks[0]!.zenrows_params?.mode, "auto");
+  assert.equal(body.zenrows_params?.proxy_country, "us");
+});