From cd3c88cbb4c5a19dfc8df4d1909f14467efd0245 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Wed, 14 Oct 2026 14:26:06 +0000 Subject: [PATCH 1/6] feat(crawl): read the discovery strategies a crawl ran with, and save a page as a file A crawl reports `discovery`, the strategies it finds URLs with on each page it walks, and a create can ask for them. `links` follows the links in the page's HTML, and is what a crawl runs with when the create leaves it out. The field reads `["links"]` when the server does not send it, so the model parses an older answer too. `content(..., download=True)` asks for the page as a file to save rather than one to show. The text is the same; the server sends a file name built from the content id. `CrawlErrorCode` gains `api_key_cap_reached`, which a crawl ends with when the key that created it reaches one of its credit caps. It parsed as an unknown value before. --- acceptance/crawl/surface/test.py | 71 +++++++++++++++++++++++++++++++- src/zenrows/crawl/__init__.py | 2 + src/zenrows/crawl/client.py | 20 ++++++++- src/zenrows/crawl/models.py | 20 ++++++++- 4 files changed, 109 insertions(+), 4 deletions(-) diff --git a/acceptance/crawl/surface/test.py b/acceptance/crawl/surface/test.py index 1f1ba40..709f353 100644 --- a/acceptance/crawl/surface/test.py +++ b/acceptance/crawl/surface/test.py @@ -248,7 +248,8 @@ def do_GET(self): threading.Thread(target=srv.serve_forever, daemon=True).start() from zenrows import ZenRowsCrawlClient # noqa: E402 (imported once installed, as a user would) -from zenrows.crawl import CrawlAPIError # noqa: E402 +from zenrows.batch._open_enum import is_unknown # noqa: E402 +from zenrows.crawl import CrawlAPIError, CrawlErrorCode # noqa: E402 cap = ZenRowsCrawlClient( "capture-key", base_url=f"http://127.0.0.1:{srv.server_address[1]}/v1", retries=0 @@ -839,6 +840,72 @@ def c21(): live=True, ) + +def c22(): + """discovery: sent as given, echoed back, and ["links"] when left out.""" + given = create(discovery=["links"], **BASE) + left_out = create(**{**BASE, "max_items": 1}) + for c in (given, left_out): + client.stop(c.crawl_id) + values = [[d.value for d in c.discovery] for c in (given, left_out)] + return ( + pass_if(values == [["links"], ["links"]]), + json.dumps({"discovery=['links']": values[0], "left out": values[1]}), + "", + ) + + +def c23(): + """content(download=True): the same page, asked for as a file to save.""" + b = state.get("B") + if not b: + return SKIP, "no html crawl to read a page from", "" + fetched = next( + (r for r in client.get(b.crawl_id, limit=100).results if r.content_url), + None, + ) + if not fetched: + return SKIP, "no result had been fetched", "" + plain = client.content(b.crawl_id, fetched) + as_file = client.content(b.crawl_id, fetched, download=True) + return ( + pass_if(plain == as_file and bool(plain)), + f"same {len(plain)} chars with and without download=True", + trim(plain, 400), + ) + + +def c24(): + """api_key_cap_reached parses as the named member, not as UNKNOWN.""" + code = CrawlErrorCode("api_key_cap_reached") + return ( + pass_if(code is CrawlErrorCode.API_KEY_CAP_REACHED and not is_unknown(code)), + f"{code!r} (name {code.name})", + "", + ) + + +run_check( + "22", + "Discovery", + 'create(discovery=["links"]) is echoed back, and a create without it reads ["links"]', + c22, + live=True, +) +run_check( + "23", + "Content as a file", + "content(download=True) returns the same page as content() does", + c23, + live=True, +) +run_check( + "24", + "Run error codes", + "api_key_cap_reached is a named CrawlErrorCode member, not an unknown value", + c24, +) + with open(f"{RESULTS}/naming.json", "w") as f: json.dump( { @@ -856,11 +923,13 @@ def c21(): "include_patterns", "exclude_patterns", "output_format", + "discovery", "idempotency_key", "cursor", "limit", "timeout", "content_id", + "download", ] }, }, diff --git a/src/zenrows/crawl/__init__.py b/src/zenrows/crawl/__init__.py index 2f7eb56..d0e22e8 100644 --- a/src/zenrows/crawl/__init__.py +++ b/src/zenrows/crawl/__init__.py @@ -28,6 +28,7 @@ CrawlStatus, CrawlStop, CrawlWithResults, + DiscoveryStrategy, DownloadLine, OutputFormat, RunError, @@ -47,6 +48,7 @@ "CrawlStatus", "CrawlStop", "CrawlWithResults", + "DiscoveryStrategy", "DownloadLine", "OutputFormat", "RunError", diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index abac80b..7b911f9 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -172,6 +172,7 @@ def create( include_patterns: list[str] | None = None, exclude_patterns: list[str] | None = None, output_format: Literal["html", "markdown"] | None = None, + discovery: list[Literal["links"]] | None = None, idempotency_key: str | None = None, ) -> Crawl: """`POST /crawls` — start a crawl and return it, `running`. @@ -190,6 +191,9 @@ def create( - `output_format="html"` or `"markdown"`: also fetch every kept URL's page, as HTML or as Markdown, readable with `content` / `download`. Omit for URLs only. + - `discovery`: how the crawl finds the URLs it follows on each + page it walks. `["links"]` follows the links in the page's + HTML, and is what a crawl runs with when you leave it out. - `idempotency_key`: a retry with the same key and body answers with the crawl the first request created. @@ -206,6 +210,7 @@ def create( "include_patterns": include_patterns, "exclude_patterns": exclude_patterns, "output_format": output_format, + "discovery": discovery, } body.update({k: v for k, v in optional.items() if v is not None}) headers = {"Content-Type": "application/json"} @@ -300,7 +305,13 @@ def stop(self, crawl_id: str) -> CrawlStop: # ===== contents ===== - def content(self, crawl_id: str, content_id: str | CrawlResult) -> str: + def content( + self, + crawl_id: str, + content_id: str | CrawlResult, + *, + download: bool = False, + ) -> str: """`GET /crawls/{crawl_id}/contents/{content_id}` — one kept URL's page, as text in the crawl's `output_format` (HTML or Markdown). @@ -310,6 +321,10 @@ def content(self, crawl_id: str, content_id: str | CrawlResult) -> str: segment of the `content_url`). Only crawls created with an `output_format` have contents; anything else is 404 `content_not_found`. + + `download=True` asks for the page as a file to save: the server + sends `Content-Disposition: attachment` with a file name built + from the content id. The text returned is the same either way. """ if isinstance(content_id, CrawlResult): if not content_id.content_url: @@ -319,7 +334,8 @@ def content(self, crawl_id: str, content_id: str | CrawlResult) -> str: ) content_id = content_id.content_url segment = content_id.rstrip("/").rsplit("/", 1)[-1] - return self._t.request("GET", f"/crawls/{crawl_id}/contents/{segment}").text + params = {"download": "true"} if download else None + return self._t.request("GET", f"/crawls/{crawl_id}/contents/{segment}", params=params).text def download(self, crawl_id: str) -> CrawlDownload: """`GET /crawls/{crawl_id}/download` — open the NDJSON file and diff --git a/src/zenrows/crawl/models.py b/src/zenrows/crawl/models.py index 85050d3..35f8fda 100644 --- a/src/zenrows/crawl/models.py +++ b/src/zenrows/crawl/models.py @@ -12,7 +12,7 @@ from enum import Enum from typing import Any -from pydantic import BaseModel +from pydantic import BaseModel, Field from zenrows.batch._open_enum import open_enum_missing @@ -61,10 +61,23 @@ class OutputFormat(Enum): _missing_ = classmethod(open_enum_missing) +class DiscoveryStrategy(Enum): + """How the crawl finds the URLs it follows on each page it walks. + + `links` follows the links in the page's HTML. The server may run a + crawl with a strategy this version does not list, and it parses as + an `UNKNOWN` member keeping the raw value.""" + + LINKS = "links" + + _missing_ = classmethod(open_enum_missing) + + class CrawlErrorCode(Enum): """Why a crawl failed (`Crawl.error.code`).""" INSUFFICIENT_CREDITS = "insufficient_credits" + API_KEY_CAP_REACHED = "api_key_cap_reached" SEED_UNREACHABLE = "seed_unreachable" DOMAIN_NOT_ALLOWED = "domain_not_allowed" NO_ITEMS_FOUND = "no_items_found" @@ -104,6 +117,10 @@ class Crawl(BaseModel): created without one: it has no depth limit.""" max_items: int max_pages: int + discovery: list[DiscoveryStrategy] = Field(default_factory=lambda: [DiscoveryStrategy.LINKS]) + """The strategies the crawl finds URLs with. `["links"]` when the + create left it out, which is also what this reads when the server + does not send the field at all.""" coverage: Coverage created_at: datetime stop_reason: StopReason | None = None @@ -179,6 +196,7 @@ class DownloadLine(BaseModel): "CrawlStatus", "CrawlStop", "CrawlWithResults", + "DiscoveryStrategy", "DownloadLine", "OutputFormat", "RunError", From 9ebf1f55ad513eafbb71f1e64c2352bba175a507 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Wed, 14 Oct 2026 15:19:56 +0000 Subject: [PATCH 2/6] test(crawl): read a page of the acceptance check's own crawl when the stopped one has none The check that reads a page as a file borrowed the crawl another check stops mid-flight. That stop can land before any page is fetched, and the check then skipped instead of covering anything. It now falls back to a one-item crawl of its own and waits for it, as the check beside it already did. --- acceptance/crawl/surface/test.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/acceptance/crawl/surface/test.py b/acceptance/crawl/surface/test.py index 709f353..92557ca 100644 --- a/acceptance/crawl/surface/test.py +++ b/acceptance/crawl/surface/test.py @@ -857,17 +857,27 @@ def c22(): def c23(): """content(download=True): the same page, asked for as a file to save.""" + + def first_fetched(crawl_id): + return next( + (r for r in client.get(crawl_id, limit=100).results if r.content_url), + None, + ) + + # The stop in check 20 can land before any of B's pages is fetched. Then read + # a one-item html crawl that runs to the end instead, as check 9 does. b = state.get("B") - if not b: - return SKIP, "no html crawl to read a page from", "" - fetched = next( - (r for r in client.get(b.crawl_id, limit=100).results if r.content_url), - None, - ) + src = b.crawl_id if b else None + fetched = first_fetched(src) if src else None + if not fetched: + own = create(**{**BASE, "max_items": 1, "output_format": "html"}) + client.wait(own.crawl_id) + src = own.crawl_id + fetched = first_fetched(src) if not fetched: - return SKIP, "no result had been fetched", "" - plain = client.content(b.crawl_id, fetched) - as_file = client.content(b.crawl_id, fetched, download=True) + return FAIL, f"no result with a content_url on {src}", "" + plain = client.content(src, fetched) + as_file = client.content(src, fetched, download=True) return ( pass_if(plain == as_file and bool(plain)), f"same {len(plain)} chars with and without download=True", From d7f65587d9663fea6b2e398a042f6d4847a97f57 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 08:27:55 +0000 Subject: [PATCH 3/6] chore(crawl): record the contract this surface follows The lock names the API description this surface was synced to, by its blob id, so a later drift check compares against it instead of reporting the surface as never synced. Two operations in that description are not served to callers and no surface exposes them; the lock lists them as pending, so a change to either does not read as drift here. --- acceptance/crawl/contract.lock | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 acceptance/crawl/contract.lock diff --git a/acceptance/crawl/contract.lock b/acceptance/crawl/contract.lock new file mode 100644 index 0000000..435df9b --- /dev/null +++ b/acceptance/crawl/contract.lock @@ -0,0 +1,3 @@ +1adcbc2fd87e3f508317f3ce24c3d48573aec657 +pending GET /healthz +pending POST /internal/v1/crawls/{crawl_id}/stop From b7baa72bdd683bcb4fb61411c9d947ecc1cf8665 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 12:54:34 +0000 Subject: [PATCH 4/6] docs(crawl): say that a crawl out of credits ends stopped, with error set The Crawl API now ends a crawl that runs out of credits, or whose API key hits a credit cap, as `stopped` with `error.code` `insufficient_credits` or `api_key_cap_reached` and no `stop_reason`, instead of `failed`. The pages it fetched before that stay readable. The docstrings, README and example said `error` appears only on a `failed` crawl and that `stopped` always means the caller stopped it. They now describe both cases. A unit test pins that `wait` returns such a crawl with its error and no stop reason. --- README.md | 10 +++++++--- examples/11_crawl.py | 6 +++++- src/zenrows/crawl/client.py | 2 ++ src/zenrows/crawl/models.py | 16 ++++++++++++---- tests/test_crawl_client.py | 28 ++++++++++++++++++++++++++++ 5 files changed, 54 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 6fd860b..388ce1a 100644 --- a/README.md +++ b/README.md @@ -532,7 +532,10 @@ for result in client.results(crawl.crawl_id): # follows next_cursor `wait` returns the crawl when it ends, or when `timeout` (seconds, default 600) runs out. It does not raise: on timeout the crawl is still `running` (call `wait` again, or `stop`), and a `failed` crawl comes -back with `crawl.error` set. `results` does not poll: it returns at the first empty page, so on a +back with `crawl.error` set. A crawl that ran out of credits, or whose +API key hit a credit cap, ends `stopped` with `crawl.error.code` +`insufficient_credits` or `api_key_cap_reached` and no `stop_reason`; the +pages it fetched before that stay readable. `results` does not poll: it returns at the first empty page, so on a running crawl it yields only the URLs kept so far. Call `wait` first to read every result. `results(crawl_id, limit=...)` sets the page size of each request. `get(crawl_id, cursor=..., limit=...)` returns one raw page. @@ -613,8 +616,9 @@ except CrawlAPIError as exc: | 503 | `crawl_busy` | `stop` could not be saved yet; the crawl is still running. `stop` retries it up to `retries` times, waiting `exc.retry_after` seconds, then raises it. Repeating the stop is safe. | `code` is None when the response has no problem body with a code (for -example a 502 from a proxy). A crawl that ended `failed` is not an error -response: read `crawl.error.code` and `crawl.error.detail` on it. Response enums (`CrawlStatus`, `StopReason`, +example a 502 from a proxy). A crawl that ended `failed`, or `stopped` on +credits, is not an error response: read `crawl.error.code` and +`crawl.error.detail` on it. Response enums (`CrawlStatus`, `StopReason`, `ContentStatus`, ...) are open: a value added later parses as an `UNKNOWN` member that keeps the raw value. diff --git a/examples/11_crawl.py b/examples/11_crawl.py index 1832256..da3872e 100644 --- a/examples/11_crawl.py +++ b/examples/11_crawl.py @@ -4,7 +4,8 @@ - `create(url, depth=..., include_patterns=..., output_format="html")` starts a crawl and returns at once, `running`. - `wait(crawl_id)` polls until it ends, or 600 s pass, and returns - the `Crawl` (still `running` on timeout; `error` set when `failed`). + the `Crawl` (still `running` on timeout; `error` set when `failed`, + or when it `stopped` because credits ran out or the key hit a cap). - `results(crawl_id)` pages through the kept URLs. - `content(crawl_id, result)` reads one kept URL's HTML. - `download(crawl_id)` reads every result as NDJSON, with the crawl's @@ -58,6 +59,9 @@ def main() -> None: ) if crawl.status is CrawlStatus.FAILED and crawl.error: raise SystemExit(f"crawl failed: {crawl.error.code.value}: {crawl.error.detail}") + if crawl.status is CrawlStatus.STOPPED and crawl.error: + # Out of credits or at the key's cap: what it fetched stays readable. + print(f"stopped: {crawl.error.code.value}: {crawl.error.detail}") if crawl.status is CrawlStatus.RUNNING: print("still running: the results below are what it has kept so far") diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index 7b911f9..d4e4039 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -357,6 +357,8 @@ def wait(self, crawl_id: str, *, timeout: float = 600.0) -> Crawl: `failed`) or `timeout` seconds pass, and return it. A `failed` crawl is returned, not raised: read `error` on it. + So is a crawl that `stopped` because credits ran out or the API + key hit a credit cap; `error` says which. On timeout the crawl is returned still `running`; call `wait` again to keep waiting, or `stop` to end it. """ diff --git a/src/zenrows/crawl/models.py b/src/zenrows/crawl/models.py index 35f8fda..ab01e89 100644 --- a/src/zenrows/crawl/models.py +++ b/src/zenrows/crawl/models.py @@ -32,7 +32,9 @@ class CrawlStatus(Enum): class StopReason(Enum): """What ended the crawl before nothing was left to open: a limit (`max_items` / `max_pages`, with `completed`) or the caller (`user`, - with `stopped`).""" + with `stopped`). Absent on a crawl that stopped because the account + ran out of credits or the API key hit a credit cap: that one carries + `error` instead.""" MAX_ITEMS = "max_items" MAX_PAGES = "max_pages" @@ -74,7 +76,11 @@ class DiscoveryStrategy(Enum): class CrawlErrorCode(Enum): - """Why a crawl failed (`Crawl.error.code`).""" + """Why a crawl failed, or why it stopped on credits (`Crawl.error.code`). + + `insufficient_credits` and `api_key_cap_reached` come with + `status` `stopped`: the pages fetched before that stay readable. + The other codes come with `failed`.""" INSUFFICIENT_CREDITS = "insufficient_credits" API_KEY_CAP_REACHED = "api_key_cap_reached" @@ -87,8 +93,10 @@ class CrawlErrorCode(Enum): class RunError(BaseModel): - """Why a crawl failed. Present only when `status` is `failed`; part - of the crawl, not an error response.""" + """Why a crawl failed, or why it stopped because the account ran out + of credits or the API key hit a credit cap. Present when `status` is + `failed`, and when it is `stopped` with `code` `insufficient_credits` + or `api_key_cap_reached`; part of the crawl, not an error response.""" code: CrawlErrorCode detail: str diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index 962022d..4482e04 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -232,6 +232,34 @@ def test_get_parses_failed_crawl_error(client: ZenRowsCrawlClient): assert page.error is not None and page.error.code is CrawlErrorCode.SEED_UNREACHABLE +@pytest.mark.parametrize( + ("code", "member"), + [ + ("insufficient_credits", CrawlErrorCode.INSUFFICIENT_CREDITS), + ("api_key_cap_reached", CrawlErrorCode.API_KEY_CAP_REACHED), + ], +) +@respx.mock +def test_wait_returns_crawl_stopped_on_credits( + client: ZenRowsCrawlClient, no_sleep, code: str, member: CrawlErrorCode +): + respx.get(f"{BASE_URL}/crawls/c_1").mock( + return_value=Response( + 200, + json=crawl_body( + status="stopped", + error={"code": code, "detail": "Batch stopped the crawl."}, + results=[], + next_cursor=None, + ), + ) + ) + crawl = client.wait("c_1") + assert crawl.status is CrawlStatus.STOPPED + assert crawl.stop_reason is None + assert crawl.error is not None and crawl.error.code is member + + @respx.mock def test_get_omits_unset_query_params(client: ZenRowsCrawlClient): route = respx.get(f"{BASE_URL}/crawls/c_1").mock( From c00fb918a72cf8a9cc9f75534307796ec8a0c767 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 13:46:58 +0000 Subject: [PATCH 5/6] test(crawl): use the customer wording for credit-stop details The fixtures now carry the error.detail the server returns when credits end a crawl, one per error code. --- tests/test_crawl_client.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index 4482e04..208ff36 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -232,6 +232,13 @@ def test_get_parses_failed_crawl_error(client: ZenRowsCrawlClient): assert page.error is not None and page.error.code is CrawlErrorCode.SEED_UNREACHABLE +# What the server says when credits end a crawl, as error.detail. +CREDIT_STOP_DETAIL = { + "insufficient_credits": "The crawl stopped because your account has no credits left.", + "api_key_cap_reached": "The crawl stopped because your API key reached one of its credit caps.", +} + + @pytest.mark.parametrize( ("code", "member"), [ @@ -248,7 +255,7 @@ def test_wait_returns_crawl_stopped_on_credits( 200, json=crawl_body( status="stopped", - error={"code": code, "detail": "Batch stopped the crawl."}, + error={"code": code, "detail": CREDIT_STOP_DETAIL[code]}, results=[], next_cursor=None, ), From 36a5364bff21f4e53f58b9be725d479806a2df10 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Fri, 16 Oct 2026 12:25:55 +0200 Subject: [PATCH 6/6] test(crawl): drop the surface lock The lock lives beside the contract, not in a surface. main already dropped it, and this branch had brought it back. --- acceptance/crawl/contract.lock | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 acceptance/crawl/contract.lock diff --git a/acceptance/crawl/contract.lock b/acceptance/crawl/contract.lock deleted file mode 100644 index 435df9b..0000000 --- a/acceptance/crawl/contract.lock +++ /dev/null @@ -1,3 +0,0 @@ -1adcbc2fd87e3f508317f3ce24c3d48573aec657 -pending GET /healthz -pending POST /internal/v1/crawls/{crawl_id}/stop