feat(fetch): retry uncharged busy answers so a brief capacity wait does not fail the command

README.md

@@ -112,8 +112,33 @@ zenrows fetch <url> # Adaptive Stealth Mode (mode=auto)
 zenrows fetch <url> --output markdown
 zenrows fetch <url> --manual --js-render --premium-proxy
 zenrows fetch <url> --proxy-country us --wait-for ".price"
+zenrows fetch <url> --retries 0 # send once; the default is 2 retries
 ```
 
+`fetch` and `extract` retry a 429, 502, 503 or 504 answer up to `--retries`
+times (default 2; `--retries 0` sends once). They also retry a network error
+that happened before the request reached the API: a refused connection, a name
+that did not resolve, a connect timeout or an unreachable network. Each wait
+honours `Retry-After` in whole seconds (at least 0.25 s, at most 30 s) or is a
+random delay that grows with each retry. No retry starts after 90 s, counted
+from the command's first request; Extract's fallback to Autoparse shares that
+clock. A running request is not cut short, so a command can take longer than
+90 s. None of the retried statuses is charged. The CLI has no cancel option of
+its own; Ctrl-C ends the command, including a wait between retries.
+
+The CLI does not retry:
+
+1. A client timeout (`--timeout`, default 200000 ms, above the API's own 180 s
+   budget). The API may still finish and charge that request.
+2. A connection that closed after the request was sent, for the same reason.
+3. A 429 whose code says the account or the API key is out of credits
+   (`AUTH004`, `AUTH014`).
+4. A status the target site sent, under `--original-status`.
+
+With `--json`, `attempts` in the result says how many HTTP requests the command
+sent (1 means no retry). When the retries run out, the error suggests the same
+command again rather than a costlier configuration.
+
 ## 8. Extract
 
 ```bash

skills/protected-fetch/SKILL.md

@@ -47,6 +47,13 @@ zenrows fetch <url> --manual --js-render --premium-proxy   # the most expensive
   needs `--premium-proxy` (else `PARAM_PROXY_COUNTRY_REQUIRES_PREMIUM`).
 - You are billed only for the configuration that succeeds.
 - On failure, read `zenrows trace explain <run-id>` before escalating.
+- The CLI already retries 429, 502, 503 and 504 answers, and network errors
+  that happened before the request reached the API (`--retries`, default 2;
+  `--retries 0` sends once). No retry starts after 90 s. Do not wrap it in your
+  own retry loop. `--json` reports `attempts`, the number of requests sent.
+- The CLI does not retry a timeout, a connection that closed after the request
+  was sent, an out-of-credits 429 (`AUTH004`, `AUTH014`), or a status the target
+  sent under `--original-status`.
 
 Every run is saved under `.zenrows/runs/<run-id>/` (no secrets). See [[extract]]
 for structured output and [[cost-control]] for the cost model.

skills/trace-debug/SKILL.md

@@ -26,12 +26,28 @@ zenrows trace export <run-id>     # JSON for sharing
   `--manual --js-render` run; auto mode may ignore it. Each step
   multiplies the cost of the request, and both together are the most expensive
   configuration available ([[cost-control]]).
+- `FETCH_FAILED` after HTTP 429, 502, 503 or 504 → a temporary answer the CLI
+  already retried (`--retries`, default 2). Wait a minute and run the suggested
+  command, which is the same command. Do not escalate: a costlier fetch
+  configuration does not fix it. Under `--original-status` the status may be the
+  target site's own, and the CLI sends it once.
 - `REQUEST_TIMEOUT` → the CLI stopped waiting; the API was reached. Raise
-  `--timeout` (default 120000ms, above the API's own 90s budget), or drop
+  `--timeout` (default 200000ms, above the API's own 180s budget), or drop
   `--wait-for` so the request finishes inside that budget and the API returns
-  its own error. Not a connectivity problem — do not chase the network.
+  its own error. Not a connectivity problem — do not chase the network. The CLI
+  never retries a timeout on its own: the API may still finish and charge it.
+- `SERVICE_BUSY` → HTTP 503 `RESP009`: Zenrows could not take the request now.
+  Not charged. On Extract, `likely_cause` quotes the API's title, which says
+  either that Extract could not finish this page right now (retry in about a
+  minute) or that it is at capacity for pages it has not seen before (retry
+  later). The CLI already retried it (`--retries`, default 2); wait, then run
+  the suggested command, which is the command that ran. Do not escalate to
+  `--js-render` / `--premium-proxy`.
 - `BACKEND_UNAVAILABLE` → a genuine transport failure (DNS/TCP/TLS). Check
-  connectivity and `zenrows config show`.
+  connectivity and `zenrows config show`. The CLI retries it only when the
+  request never reached the API (refused connection, DNS, connect timeout). A
+  connection that closed after the request was sent is not retried, because the
+  API may still finish and charge that request.
 - `AUTH_INVALID` → re-check the key, `zenrows login --api-key …`.
 - `KEY_CREDIT_CAP_REACHED` → this API key hit one of its own credit caps (HTTP
   402, gateway code `AUTH014`, Batch `api_key_cap_reached`). The account still

src/adapters/extract.ts

@@ -16,6 +16,7 @@
 import type { Policy, ToolkitConfig } from "../types/index.ts";
 import { ToolkitError } from "../core/errors.ts";
 import { runFetch, type FetchOptions, type FetchOutcome } from "./protected-fetch.ts";
+import { newRetryBudget } from "../core/http.ts";
 
 export type ExtractMethod = "extract" | "autoparse" | "css" | "outputs" | "markdown" | "plaintext";
 
@@ -91,11 +92,13 @@ export async function runExtract(
   }
 
   if (method === "extract" && opts.fallbackAutoparse !== false) {
+    // One budget for the call: the fallback does not restart the 90 s clock.
+    const budget = opts.budget ?? newRetryBudget();
     try {
-      return await runExtractOnce({ ...opts, method: "extract" }, config, policy, apiKey);
+      return await runExtractOnce({ ...opts, budget, method: "extract" }, config, policy, apiKey);
     } catch (err) {
       if (err instanceof ToolkitError && err.code === "EXTRACT_DOMAIN_NOT_ENABLED") {
-        const outcome = await runExtractOnce({ ...opts, method: "autoparse" }, config, policy, apiKey);
+        const outcome = await runExtractOnce({ ...opts, budget, method: "autoparse" }, config, policy, apiKey);
         return { ...outcome, fellBackToAutoparse: true };
       }
       throw err;

src/adapters/protected-fetch.ts

@@ -10,7 +10,7 @@
 import type { Policy, ToolkitConfig } from "../types/index.ts";
 import { ToolkitError } from "../core/errors.ts";
 import { assertDomainAllowed } from "../core/policy.ts";
-import { scrape, type ScraperParams, type ScraperResult } from "../core/http.ts";
+import { scrape, type RetryBudget, type ScraperParams, type ScraperResult } from "../core/http.ts";
 
 export type ResponseFormat = "html" | "markdown" | "plaintext" | "pdf";
 
@@ -51,6 +51,22 @@ export interface FetchOptions {
    * gateway's own request budget on purpose.
    */
   timeoutMs?: number;
+  /**
+   * Retries for a retryable failure (429, 502, 503, 504, or a network error).
+   * Not an API parameter. Unset means `http.ts`'s default; 0 sends the request once.
+   */
+  retries?: number;
+  /**
+   * The retry budget of the logical call this request belongs to. Not an API
+   * parameter. Extract's Autoparse fallback passes the first request's budget,
+   * so no retry starts more than 90 s after the call's first request.
+   */
+  budget?: RetryBudget;
+  /**
+   * The command the user ran, quoted for a shell. Not an API parameter.
+   * Errors that advise running the same command again suggest it verbatim.
+   */
+  rerunCommand?: string;
 }
 
 const RESPONSE_TYPE: Partial<Record<ResponseFormat, string>> = {
@@ -152,6 +168,12 @@ export async function runFetch(
   validateAutoManual(opts, config);
   const params = buildParams(opts, config);
   const mode: "auto" | "manual" = params.mode === "auto" ? "auto" : "manual";
-  const result = await scrape(config.apiBase, apiKey, params, { timeoutMs: opts.timeoutMs, projectRoot });
+  const result = await scrape(config.apiBase, apiKey, params, {
+    timeoutMs: opts.timeoutMs,
+    retries: opts.retries,
+    projectRoot,
+    budget: opts.budget,
+    rerunCommand: opts.rerunCommand,
+  });
   return { result, params, mode };
 }

src/cli/commands/extract.ts

@@ -18,6 +18,7 @@ import { formatRequestCost } from "../../core/http.ts";
 import { parse, asString, asNumber, type Command, type RunContext } from "../command.ts";
 import { printError, writeOut } from "../output.ts";
 import { refreshClaimAdvice } from "../../core/claim-link.ts";
+import { normalizeRetries, rerunCommand } from "./fetch.ts";
 
 export const extract: Command = {
   name: "extract",
@@ -35,6 +36,7 @@ export const extract: Command = {
     "Shared:",
     "  --manual --js-render --premium-proxy   manual fetch controls",
     "  --validate             fail if the result is not valid JSON",
+    "  --retries <n>          retries on 429/502/503/504 or a connect/DNS error (default 2; 0 disables)",
     "  --out <file>           write output to a file",
     "  --no-signup            do not auto-create a Free plan account if no key exists",
     "  --json                 structured result",
@@ -55,6 +57,7 @@ export const extract: Command = {
       wait: { type: "string" },
       "wait-for": { type: "string" },
       validate: { type: "boolean" },
+      retries: { type: "string" },
       out: { type: "string" },
       "no-signup": { type: "boolean" },
       json: { type: "boolean" },
@@ -105,6 +108,8 @@ export const extract: Command = {
       wait: asNumber(values.wait),
       waitFor: asString(values["wait-for"]),
       validate: values.validate === true,
+      retries: normalizeRetries(values.retries),
+      rerunCommand: rerunCommand("extract", argv, ctx),
     };
 
     assertUsable("extract");
@@ -191,6 +196,7 @@ export const extract: Command = {
               bytes: outcome.result.body.length,
               costUsd: outcome.result.costUsd,
               costCredits: outcome.result.costCredits,
+              attempts: outcome.result.attempts ?? 1,
             },
             null,
             2,

src/cli/commands/fetch.ts

@@ -13,7 +13,7 @@ import { log } from "../../core/logger.ts";
 import { newRunId, writeRun } from "../../core/artifacts.ts";
 import { ToolkitError } from "../../core/errors.ts";
 import { runFetch, type FetchOptions, type ResponseFormat } from "../../adapters/protected-fetch.ts";
-import { formatRequestCost } from "../../core/http.ts";
+import { formatRequestCost, shellQuote } from "../../core/http.ts";
 import { parse, asString, asNumber, type Command, type RunContext } from "../command.ts";
 import { printError, writeOut } from "../output.ts";
 import { refreshClaimAdvice } from "../../core/claim-link.ts";
@@ -36,7 +36,8 @@ export const fetch_: Command = {
     "  --output <fmt>         html (default) | markdown | text | pdf",
     "  --screenshot           capture an above-the-fold screenshot",
     "  --out <file>           write the response body to a file",
-    "  --timeout <ms>         client-side timeout (default 120000; above the API's own 90s budget)",
+    "  --timeout <ms>         client-side timeout per attempt (default 200000; the API's own budget is 180s)",
+    "  --retries <n>          retries on 429/502/503/504 or a connect/DNS error (default 2; 0 disables)",
     "  --no-signup            do not auto-create a Free plan account if no key exists",
     "  --json                 print a structured result",
     "",
@@ -59,6 +60,7 @@ export const fetch_: Command = {
       screenshot: { type: "boolean" },
       out: { type: "string" },
       timeout: { type: "string" },
+      retries: { type: "string" },
       "no-signup": { type: "boolean" },
       json: { type: "boolean" },
     });
@@ -101,6 +103,8 @@ export const fetch_: Command = {
       output: normalizeOutput(asString(values.output)),
       screenshot: values.screenshot === true,
       timeoutMs: normalizeTimeout(values.timeout),
+      retries: normalizeRetries(values.retries),
+      rerunCommand: rerunCommand("fetch", argv, ctx),
     };
 
     const runId = newRunId();
@@ -155,7 +159,7 @@ export const fetch_: Command = {
       if (runDir) log.dim(`  artifact: ${runDir}`);
 
       if (ctx.json || values.json) {
-        log.out(JSON.stringify({ ok: true, runId, mode, httpStatus: result.status, bytes, costUsd: result.costUsd, costCredits: result.costCredits, requestId: result.requestId, finalUrl: result.finalUrl }, null, 2));
+        log.out(JSON.stringify({ ok: true, runId, mode, httpStatus: result.status, bytes, costUsd: result.costUsd, costCredits: result.costCredits, requestId: result.requestId, finalUrl: result.finalUrl, attempts: result.attempts ?? 1 }, null, 2));
       } else if (!values.out) {
         if (result.isBinary) {
           // Never dump raw binary to the terminal — it corrupts the session and
@@ -209,6 +213,36 @@ export function normalizeTimeout(v: unknown): number | undefined {
   return ms;
 }
 
+/**
+ * `--retries <n>`: whole decimal digits, 0 or more. Like `--timeout`, a bad
+ * value is an error, never a silent fallback to the default. `1e1`, `0x2`,
+ * `+1` and values with spaces are rejected, although `Number()` would read them.
+ */
+export function normalizeRetries(v: unknown): number | undefined {
+  if (v === undefined) return undefined;
+  const n = typeof v === "string" && /^\d+$/.test(v) ? Number(v) : undefined;
+  if (n === undefined || !Number.isSafeInteger(n)) {
+    throw new ToolkitError({
+      code: "INVALID_USAGE",
+      message: `Invalid --retries value '${String(v)}'.`,
+      likely_cause: "--retries takes a whole number, 0 or more.",
+      next_action: "Pass a count, e.g. --retries 2, or --retries 0 to send the request once.",
+      suggested_commands: ["zenrows fetch <url> --retries 0"],
+    });
+  }
+  return n;
+}
+
+/**
+ * The command the user ran, quoted for a shell, for errors whose advice is
+ * "run the same command again". The router strips a global `--json`, so it is
+ * added back when it was set.
+ */
+export function rerunCommand(name: string, argv: string[], ctx: RunContext): string {
+  const args = ctx.json && !argv.includes("--json") ? [...argv, "--json"] : argv;
+  return ["zenrows", name, ...args.map(shellQuote)].join(" ");
+}
+
 export function normalizeOutput(v?: string): ResponseFormat | undefined {
   if (!v) return undefined;
   const map: Record<string, ResponseFormat> = {

src/core/backoff.ts

@@ -0,0 +1,52 @@
+/**
+ * Retry delays shared by the HTTP clients.
+ *
+ * Crawl keeps its own schedule (`crawlBackoffMs`) and reads `Retry-After`
+ * through `retryAfterMs`. Fetch and Extract use full jitter (`fullJitterMs`)
+ * and read `Retry-After` through `retryAfterDelayMs`, which also applies the
+ * 250 ms floor: the policy every Zenrows Fetch/Extract client shares.
+ */
+
+/** The longest a Retry-After may make one retry wait, so a large server value cannot hang a command. */
+export const MAX_RETRY_AFTER_SECONDS = 30;
+
+/** Crawl's schedule: 250 ms x 2^attempt, capped at 10 s, +/-20% jitter. */
+export function crawlBackoffMs(attempt: number): number {
+  return Math.min(250 * 2 ** attempt, 10_000) * (0.8 + Math.random() * 0.4);
+}
+
+/** Shortest Fetch/Extract wait, from jitter or from Retry-After: never retry in a tight loop. */
+export const JITTER_FLOOR_MS = 250;
+/** Longest full-jitter wait. */
+export const JITTER_CAP_MS = 30_000;
+
+/**
+ * Full jitter: uniform(0.25 s, max(0.25 s, min(capMs, 1 s x 2^attempt))).
+ * `attempt` is the 0-based retry index, so the first retry waits at most 1 s
+ * and the second 2 s. `random` is injectable for tests. `capMs` is 30 s; the
+ * CLI has no option to change it, and the parameter exists for the shared
+ * conformance vectors.
+ */
+export function fullJitterMs(attempt: number, random: () => number = Math.random, capMs: number = JITTER_CAP_MS): number {
+  const ceiling = Math.max(JITTER_FLOOR_MS, Math.min(capMs, 1000 * 2 ** attempt));
+  return JITTER_FLOOR_MS + random() * (ceiling - JITTER_FLOOR_MS);
+}
+
+/**
+ * `Retry-After` in whole seconds, as milliseconds, at most `MAX_RETRY_AFTER_SECONDS`.
+ * Crawl's reading: `0` means no wait. Anything but whole decimal digits is undefined.
+ */
+export function retryAfterMs(value: string | null): number | undefined {
+  return value && /^\d+$/.test(value.trim()) ? Math.min(Number(value), MAX_RETRY_AFTER_SECONDS) * 1000 : undefined;
+}
+
+/**
+ * Fetch and Extract's reading of `Retry-After`: whole decimal digits only,
+ * clamped to [0.25 s, 30 s], so `0` waits 250 ms. Undefined means "use jitter".
+ */
+export function retryAfterDelayMs(value: string | null): number | undefined {
+  const ms = retryAfterMs(value);
+  return ms === undefined ? undefined : Math.max(JITTER_FLOOR_MS, ms);
+}
+
+export const defaultSleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));

src/core/crawl-api.ts

@@ -20,6 +20,7 @@ import { readAccount } from "./agent-account.ts";
 import { registerSecret } from "./logger.ts";
 import { CLI_USER_AGENT, loadConfig } from "./config.ts";
 import { agentClientHeader } from "./agent-client.ts";
+import { crawlBackoffMs as backoffMs, defaultSleep, retryAfterMs } from "./backoff.ts";
 
 /** Production Crawl API base (no trailing slash): the Fetch/Extract host. */
 export const DEFAULT_CRAWL_API_BASE = "https://api.zenrows.com/v1";
@@ -176,8 +177,6 @@ interface RawResponse {
 }
 
 const MAX_RETRIES = 3;
-/** The longest a Retry-After may make one retry wait, so a large server value cannot hang a command. */
-const MAX_RETRY_AFTER_SECONDS = 30;
 const READ_RETRY_STATUSES: ReadonlySet<number> = new Set([429, 502, 503, 504]);
 // 429 on create is the active-job limit: a retry only waits for a slot that may never free.
 const CREATE_RETRY_STATUSES: ReadonlySet<number> = new Set([502, 503, 504]);
@@ -194,18 +193,6 @@ function retryStatuses(method: string, headers: Record<string, string>): Readonl
   return headers["Idempotency-Key"] ? CREATE_RETRY_STATUSES : undefined;
 }
 
-/** 250 ms x 2^attempt, capped at 10 s, +/-20% jitter. */
-function backoffMs(attempt: number): number {
-  return Math.min(250 * 2 ** attempt, 10_000) * (0.8 + Math.random() * 0.4);
-}
-
-/** `Retry-After` in whole seconds, as milliseconds, at most `MAX_RETRY_AFTER_SECONDS`. */
-function retryAfterMs(value: string | null): number | undefined {
-  return value && /^\d+$/.test(value.trim()) ? Math.min(Number(value), MAX_RETRY_AFTER_SECONDS) * 1000 : undefined;
-}
-
-const defaultSleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
-
 /** One HTTP attempt with its own timeout; the body is read inside it. */
 async function attempt(url: string, init: RequestInit, opts: RequestOpts): Promise<RawResponse> {
   const controller = new AbortController();

src/core/errors.ts

@@ -12,6 +12,7 @@ export type ErrorCode =
   | "AUTH_INVALID"
   | "BACKEND_UNAVAILABLE"
   | "REQUEST_TIMEOUT"
+  | "SERVICE_BUSY"
   | "CAPABILITY_UNAVAILABLE"
   | "PARAM_CONFLICT_AUTO_MANUAL"
   | "PARAM_PROXY_COUNTRY_REQUIRES_PREMIUM"