diff --git a/README.md b/README.md index d5306da..1ddbb1b 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ SDK to access [Zenrows](https://www.zenrows.com/) API directly from Node.js. Zen - [POST Requests](#post-requests) - [Extract](#extract) - [Batch](#batch) - - [Extensible values](#extensible-values) + - [Failed runs and extensible values](#failed-runs-and-extensible-values) - [Extract in a batch](#extract-in-a-batch) - [Concurrency](#concurrency) - [An important note about Promise.allSettled() on TypeScript](#an-important-note-about-promiseallsettled-on-typescript) @@ -227,9 +227,16 @@ const apiKey = "YOUR-API-KEY"; `client.batch` also exposes `listJobs()`, `deleteJob()`, `stopRun()`, `rerun()`, `listRuns()`, `getRun()`, `deleteRun()`, and `getTaskContent()` (returns the scraped page's raw content as a string, not JSON — the endpoint can return HTML or plain text depending on what the target page served). Scheduling, webhook config, HMAC key rotation, CSV task uploads, and results exports aren't wrapped yet — call the [Batch API](https://docs.zenrows.com) directly for those. -#### Extensible values +#### Failed runs and extensible values -Enum-valued fields on responses (run, job, task and export `status`, `ingest_status`, `failure_reason`) are typed `Extensible<...>`: the known values plus any `string`, because the server may add values. The exported named types (`RunStatus`, `JobStatus`, `TaskStatus`, `ExportStatus`, `IngestStatus`) stay closed, so they still work as request input. Give switches on these fields a default branch; an exhaustive `switch` that assigns the leftover to `never` no longer compiles. +A run the server auto-fails carries `failure_reason` and a human-readable `failure_detail`. Starting, rerunning or resuming a job whose API key hit one of its credit caps throws a `ZenRowsBatchError` with `status` 402 and `code` `"api_key_cap_reached"`; `error.problem.detail` names the cap and when it resets (UTC). + +```js +const run = await client.batch.getRun(jobId, runId); +if (run.data.failure_reason === "api_key_cap_reached") console.log(run.data.failure_detail); +``` + +Enum-valued fields on responses (run, job, task and export `status`, `ingest_status`, `failure_reason`) are typed `Extensible<...>`: the known values plus any `string`, because the server may add values. The exported named types (`RunStatus`, `JobStatus`, `TaskStatus`, `ExportStatus`, `IngestStatus`, `RunFailureReason`) stay closed, so they still work as request input. Give switches on these fields a default branch; an exhaustive `switch` that assigns the leftover to `never` no longer compiles. #### Extract in a batch diff --git a/src/batch/client.ts b/src/batch/client.ts index cc560eb..0d70c45 100644 --- a/src/batch/client.ts +++ b/src/batch/client.ts @@ -47,7 +47,9 @@ import { pollUntil } from "./waiters.js"; const DEFAULT_BATCH_API_URL = "https://async.api.zenrows.com/v1"; -const TERMINAL_RUN_STATUSES = new Set(["completed", "stopped", "deleted"]); +// Run states that never transition again: the default waiter target. `failed` is an +// account-level auto-fail (e.g. `api_key_cap_reached`, `insufficient_credits`). +const TERMINAL_RUN_STATUSES = new Set(["completed", "stopped", "failed", "deleted"]); const TERMINAL_EXPORT_STATUSES = new Set(["completed", "failed"]); export interface BatchClientConfig { @@ -110,7 +112,16 @@ export interface GetResultsOptions { export interface WaitForRunOptions { runId?: string; + /** + * Statuses that resolve the wait. Defaults to every terminal status (`completed`, + * `stopped`, `failed`, `deleted`), so an auto-failed run resolves with its + * `failure_reason` / `failure_detail` instead of polling until `timeout`. + */ targetStatuses?: Set; + /** + * Statuses that reject with `WaiterFailureError`. Takes precedence over `targetStatuses`, + * so `new Set(["failed"])` makes a failed run throw. Empty by default. + */ failureStatuses?: Set; timeout?: number; pollInterval?: number; @@ -650,7 +661,9 @@ export class ZenRowsBatchClient { return this._getRunData(jobId, options.runId); }; return pollUntil(fetchRun, { - isDone: (run) => run !== undefined && target.has(run.status), + // A status the caller named as a failure throws, even when it is also a target. + isDone: (run) => + run !== undefined && target.has(run.status) && !options.failureStatuses?.has(run.status), isFailure: (run) => run !== undefined && Boolean(options.failureStatuses?.has(run.status)), timeout: options.timeout ?? 300, initialInterval: options.pollInterval ?? 2, diff --git a/src/batch/types.ts b/src/batch/types.ts index 5aaa2ac..3a475b3 100644 --- a/src/batch/types.ts +++ b/src/batch/types.ts @@ -10,10 +10,21 @@ export type JobType = "regular" | "scheduled"; export type JobStatus = "open" | "closed" | "deleted"; export type ScheduleState = "active" | "paused"; export type RunStatus = "pending" | "running" | "completed" | "failed" | "stopped" | "deleted"; -export type TaskStatus = "pending" | "successful" | "failed"; +export type TaskStatus = "pending" | "processing" | "successful" | "failed"; export type PauseState = "active" | "paused"; export type IngestStatus = "pending" | "done"; export type ExportStatus = "pending" | "running" | "completed" | "failed"; +/** + * Why a run auto-failed (`status == "failed"`). + * - `insufficient_credits` — the account is out of credits. + * - `subscription_inactive` — the subscription is not active. + * - `api_key_cap_reached` — the job's API key reached one of its credit caps; resolves when + * that cap's window resets or the cap is raised. `Run.failure_detail` says which. + */ +export type RunFailureReason = + | "insufficient_credits" + | "subscription_inactive" + | "api_key_cap_reached"; export interface Spend { credits: number; @@ -58,7 +69,9 @@ export interface Run { last_batch_received?: boolean; pause_state?: PauseState; ingest_status?: Extensible; - failure_reason?: Extensible<"insufficient_credits" | "subscription_inactive">; + failure_reason?: Extensible; + /** Human-readable explanation of `failure_reason` (e.g. which cap, and when it resets, UTC). */ + failure_detail?: string | null; created_at?: string; updated_at?: string; } diff --git a/tests/batch-client.test.ts b/tests/batch-client.test.ts index a505e2c..27b0b0a 100644 --- a/tests/batch-client.test.ts +++ b/tests/batch-client.test.ts @@ -1,7 +1,9 @@ +import { http, HttpResponse } from "msw"; import { beforeEach, describe, expect, test } from "vitest"; import { ZenRowsBatchClient } from "../src/batch/client"; import { ZenRowsBatchError } from "../src/batch/errors"; -import "./_setup"; +import { WaiterFailureError } from "../src/batch/waiters"; +import { server } from "./_setup"; describe("ZenRowsBatchClient — retry/backoff transport", () => { let client: ZenRowsBatchClient; @@ -184,6 +186,65 @@ describe("ZenRowsBatchClient — waitForRun / downloadAllResults", () => { expect(run.status).toBe("completed"); }); + const runRunningThenFailed = () => { + let calls = 0; + server.use( + http.get("https://async.api.zenrows.com/v1/jobs/job_cap/runs/run_cap", () => { + calls += 1; + return HttpResponse.json({ + run_id: "run_cap", + job_id: "job_cap", + status: calls === 1 ? "running" : "failed", + ...(calls === 1 + ? {} + : { + failure_reason: "api_key_cap_reached", + failure_detail: "API key reached its credit cap", + }), + }); + }), + ); + return () => calls; + }; + + test("waitForRun resolves with a failed run (api_key_cap_reached) instead of timing out", async () => { + const calls = runRunningThenFailed(); + const client = new ZenRowsBatchClient("API_KEY"); + const run = await client.waitForRun("job_cap", { + runId: "run_cap", + pollInterval: 0.01, + timeout: 0.5, + }); + expect(calls()).toBe(2); + expect(run.status).toBe("failed"); + expect(run.failure_reason).toBe("api_key_cap_reached"); + expect(run.failure_detail).toBe("API key reached its credit cap"); + }); + + test("run.wait() resolves with a failed run", async () => { + runRunningThenFailed(); + const client = new ZenRowsBatchClient("API_KEY"); + const handle = await client + .run("job_cap", "run_cap") + .wait({ pollInterval: 0.01, timeout: 0.5 }); + expect(handle.data.status).toBe("failed"); + expect(handle.data.failure_reason).toBe("api_key_cap_reached"); + expect(handle.data.failure_detail).toBe("API key reached its credit cap"); + }); + + test("failureStatuses containing failed throws, even though failed is a default target", async () => { + runRunningThenFailed(); + const client = new ZenRowsBatchClient("API_KEY"); + await expect( + client.waitForRun("job_cap", { + runId: "run_cap", + failureStatuses: new Set(["failed"]), + pollInterval: 0.01, + timeout: 0.5, + }), + ).rejects.toThrow(WaiterFailureError); + }); + test("downloadAllResults starts an export, waits, and streams the zip to disk", async () => { const { mkdtemp, readFile, rm } = await import("node:fs/promises"); const { tmpdir } = await import("node:os"); diff --git a/tests/batch-extensible-enums.test.ts b/tests/batch-extensible-enums.test.ts index 9d15e20..30cfbb5 100644 --- a/tests/batch-extensible-enums.test.ts +++ b/tests/batch-extensible-enums.test.ts @@ -1,41 +1,99 @@ import { http, HttpResponse } from "msw"; import { describe, expect, expectTypeOf, test } from "vitest"; import { type ListJobsOptions, ZenRowsBatchClient } from "../src/batch/client"; -import type { JobStatus, Run, RunStatus, TaskResult, TaskStatus } from "../src/batch/types"; +import { ZenRowsBatchError } from "../src/batch/errors"; +import type { + JobStatus, + Run, + RunFailureReason, + RunStatus, + TaskResult, + TaskStatus, +} from "../src/batch/types"; import { server } from "./_setup"; const BASE = "https://async.api.zenrows.com/v1"; +function failedRun(failure_reason: string, failure_detail: string | null): Run { + return { + run_id: "run_1", + job_id: "job_cap", + run_sequence: 1, + status: "failed", + stats: { total: 1, completed: 0, successful: 0, failed: 0 }, + failure_reason, + failure_detail, + }; +} + describe("server-extensible enums", () => { test("response fields list the known values and still accept unknown ones", () => { - expectTypeOf<"failed">().toMatchTypeOf(); - expectTypeOf<"archived">().toMatchTypeOf(); + expectTypeOf<"api_key_cap_reached">().toMatchTypeOf(); expectTypeOf<"a_reason_added_later">().toMatchTypeOf>(); + expectTypeOf<"archived">().toMatchTypeOf(); + expectTypeOf<"processing">().toMatchTypeOf(); expectTypeOf<"queued">().toMatchTypeOf(); + expectTypeOf().toEqualTypeOf(); }); test("named types stay closed so they keep working as request input", () => { expectTypeOf<"archived">().not.toMatchTypeOf(); - expectTypeOf<"queued">().not.toMatchTypeOf(); + expectTypeOf<"a_reason_added_later">().not.toMatchTypeOf(); const status: JobStatus = "open"; expectTypeOf<{ status: typeof status }>().toMatchTypeOf(); }); + test("parses a run failed with api_key_cap_reached and its detail", async () => { + const detail = + "This API key reached its daily cap of 1000 credits; it resets at 2026-10-01T00:00:00Z."; + server.use( + http.get(`${BASE}/jobs/job_cap/runs/run_1`, () => + HttpResponse.json(failedRun("api_key_cap_reached", detail)), + ), + ); + const run = await new ZenRowsBatchClient("API_KEY").getRun("job_cap", "run_1"); + expect(run.data.failure_reason).toBe("api_key_cap_reached"); + expect(run.data.failure_detail).toBe(detail); + }); + test("parses a run with an unknown failure_reason and status without throwing", async () => { server.use( - http.get(`${BASE}/jobs/job_x/runs/run_1`, () => - HttpResponse.json({ - run_id: "run_1", - job_id: "job_x", - run_sequence: 1, - status: "archived", - stats: { total: 1, completed: 0, successful: 0, failed: 0 }, - failure_reason: "a_reason_added_later", - }), + http.get(`${BASE}/jobs/job_cap/runs/run_1`, () => + HttpResponse.json({ ...failedRun("a_reason_added_later", null), status: "archived" }), ), ); - const run = await new ZenRowsBatchClient("API_KEY").getRun("job_x", "run_1"); + const run = await new ZenRowsBatchClient("API_KEY").getRun("job_cap", "run_1"); expect(run.data.failure_reason).toBe("a_reason_added_later"); expect(run.status).toBe("archived"); + expect(run.data.failure_detail).toBeNull(); + }); + + test("a 402 api_key_cap_reached on rerun exposes code and detail", async () => { + const detail = + "This API key reached its monthly cap of 50000 credits; it resets at 2026-11-01T00:00:00Z."; + server.use( + http.post(`${BASE}/jobs/job_cap/rerun`, () => + HttpResponse.json( + { + type: "about:blank", + title: "Payment Required", + status: 402, + code: "api_key_cap_reached", + detail, + }, + { status: 402, headers: { "Content-Type": "application/problem+json" } }, + ), + ), + ); + await expect(new ZenRowsBatchClient("API_KEY")._postRerun("job_cap")).rejects.toSatisfy( + (error: unknown) => { + expect(error).toBeInstanceOf(ZenRowsBatchError); + const err = error as ZenRowsBatchError; + expect(err.status).toBe(402); + expect(err.code).toBe("api_key_cap_reached"); + expect(err.problem?.detail).toBe(detail); + return true; + }, + ); }); });