From cd3c88cbb4c5a19dfc8df4d1909f14467efd0245 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Wed, 14 Oct 2026 14:26:06 +0000 Subject: [PATCH 01/10] feat(crawl): read the discovery strategies a crawl ran with, and save a page as a file A crawl reports `discovery`, the strategies it finds URLs with on each page it walks, and a create can ask for them. `links` follows the links in the page's HTML, and is what a crawl runs with when the create leaves it out. The field reads `["links"]` when the server does not send it, so the model parses an older answer too. `content(..., download=True)` asks for the page as a file to save rather than one to show. The text is the same; the server sends a file name built from the content id. `CrawlErrorCode` gains `api_key_cap_reached`, which a crawl ends with when the key that created it reaches one of its credit caps. It parsed as an unknown value before. --- acceptance/crawl/surface/test.py | 71 +++++++++++++++++++++++++++++++- src/zenrows/crawl/__init__.py | 2 + src/zenrows/crawl/client.py | 20 ++++++++- src/zenrows/crawl/models.py | 20 ++++++++- 4 files changed, 109 insertions(+), 4 deletions(-) diff --git a/acceptance/crawl/surface/test.py b/acceptance/crawl/surface/test.py index 1f1ba40..709f353 100644 --- a/acceptance/crawl/surface/test.py +++ b/acceptance/crawl/surface/test.py @@ -248,7 +248,8 @@ def do_GET(self): threading.Thread(target=srv.serve_forever, daemon=True).start() from zenrows import ZenRowsCrawlClient # noqa: E402 (imported once installed, as a user would) -from zenrows.crawl import CrawlAPIError # noqa: E402 +from zenrows.batch._open_enum import is_unknown # noqa: E402 +from zenrows.crawl import CrawlAPIError, CrawlErrorCode # noqa: E402 cap = ZenRowsCrawlClient( "capture-key", base_url=f"http://127.0.0.1:{srv.server_address[1]}/v1", retries=0 @@ -839,6 +840,72 @@ def c21(): live=True, ) + +def c22(): + """discovery: sent as given, echoed back, and ["links"] when left out.""" + given = create(discovery=["links"], **BASE) + left_out = create(**{**BASE, "max_items": 1}) + for c in (given, left_out): + client.stop(c.crawl_id) + values = [[d.value for d in c.discovery] for c in (given, left_out)] + return ( + pass_if(values == [["links"], ["links"]]), + json.dumps({"discovery=['links']": values[0], "left out": values[1]}), + "", + ) + + +def c23(): + """content(download=True): the same page, asked for as a file to save.""" + b = state.get("B") + if not b: + return SKIP, "no html crawl to read a page from", "" + fetched = next( + (r for r in client.get(b.crawl_id, limit=100).results if r.content_url), + None, + ) + if not fetched: + return SKIP, "no result had been fetched", "" + plain = client.content(b.crawl_id, fetched) + as_file = client.content(b.crawl_id, fetched, download=True) + return ( + pass_if(plain == as_file and bool(plain)), + f"same {len(plain)} chars with and without download=True", + trim(plain, 400), + ) + + +def c24(): + """api_key_cap_reached parses as the named member, not as UNKNOWN.""" + code = CrawlErrorCode("api_key_cap_reached") + return ( + pass_if(code is CrawlErrorCode.API_KEY_CAP_REACHED and not is_unknown(code)), + f"{code!r} (name {code.name})", + "", + ) + + +run_check( + "22", + "Discovery", + 'create(discovery=["links"]) is echoed back, and a create without it reads ["links"]', + c22, + live=True, +) +run_check( + "23", + "Content as a file", + "content(download=True) returns the same page as content() does", + c23, + live=True, +) +run_check( + "24", + "Run error codes", + "api_key_cap_reached is a named CrawlErrorCode member, not an unknown value", + c24, +) + with open(f"{RESULTS}/naming.json", "w") as f: json.dump( { @@ -856,11 +923,13 @@ def c21(): "include_patterns", "exclude_patterns", "output_format", + "discovery", "idempotency_key", "cursor", "limit", "timeout", "content_id", + "download", ] }, }, diff --git a/src/zenrows/crawl/__init__.py b/src/zenrows/crawl/__init__.py index 2f7eb56..d0e22e8 100644 --- a/src/zenrows/crawl/__init__.py +++ b/src/zenrows/crawl/__init__.py @@ -28,6 +28,7 @@ CrawlStatus, CrawlStop, CrawlWithResults, + DiscoveryStrategy, DownloadLine, OutputFormat, RunError, @@ -47,6 +48,7 @@ "CrawlStatus", "CrawlStop", "CrawlWithResults", + "DiscoveryStrategy", "DownloadLine", "OutputFormat", "RunError", diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index abac80b..7b911f9 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -172,6 +172,7 @@ def create( include_patterns: list[str] | None = None, exclude_patterns: list[str] | None = None, output_format: Literal["html", "markdown"] | None = None, + discovery: list[Literal["links"]] | None = None, idempotency_key: str | None = None, ) -> Crawl: """`POST /crawls` — start a crawl and return it, `running`. @@ -190,6 +191,9 @@ def create( - `output_format="html"` or `"markdown"`: also fetch every kept URL's page, as HTML or as Markdown, readable with `content` / `download`. Omit for URLs only. + - `discovery`: how the crawl finds the URLs it follows on each + page it walks. `["links"]` follows the links in the page's + HTML, and is what a crawl runs with when you leave it out. - `idempotency_key`: a retry with the same key and body answers with the crawl the first request created. @@ -206,6 +210,7 @@ def create( "include_patterns": include_patterns, "exclude_patterns": exclude_patterns, "output_format": output_format, + "discovery": discovery, } body.update({k: v for k, v in optional.items() if v is not None}) headers = {"Content-Type": "application/json"} @@ -300,7 +305,13 @@ def stop(self, crawl_id: str) -> CrawlStop: # ===== contents ===== - def content(self, crawl_id: str, content_id: str | CrawlResult) -> str: + def content( + self, + crawl_id: str, + content_id: str | CrawlResult, + *, + download: bool = False, + ) -> str: """`GET /crawls/{crawl_id}/contents/{content_id}` — one kept URL's page, as text in the crawl's `output_format` (HTML or Markdown). @@ -310,6 +321,10 @@ def content(self, crawl_id: str, content_id: str | CrawlResult) -> str: segment of the `content_url`). Only crawls created with an `output_format` have contents; anything else is 404 `content_not_found`. + + `download=True` asks for the page as a file to save: the server + sends `Content-Disposition: attachment` with a file name built + from the content id. The text returned is the same either way. """ if isinstance(content_id, CrawlResult): if not content_id.content_url: @@ -319,7 +334,8 @@ def content(self, crawl_id: str, content_id: str | CrawlResult) -> str: ) content_id = content_id.content_url segment = content_id.rstrip("/").rsplit("/", 1)[-1] - return self._t.request("GET", f"/crawls/{crawl_id}/contents/{segment}").text + params = {"download": "true"} if download else None + return self._t.request("GET", f"/crawls/{crawl_id}/contents/{segment}", params=params).text def download(self, crawl_id: str) -> CrawlDownload: """`GET /crawls/{crawl_id}/download` — open the NDJSON file and diff --git a/src/zenrows/crawl/models.py b/src/zenrows/crawl/models.py index 85050d3..35f8fda 100644 --- a/src/zenrows/crawl/models.py +++ b/src/zenrows/crawl/models.py @@ -12,7 +12,7 @@ from enum import Enum from typing import Any -from pydantic import BaseModel +from pydantic import BaseModel, Field from zenrows.batch._open_enum import open_enum_missing @@ -61,10 +61,23 @@ class OutputFormat(Enum): _missing_ = classmethod(open_enum_missing) +class DiscoveryStrategy(Enum): + """How the crawl finds the URLs it follows on each page it walks. + + `links` follows the links in the page's HTML. The server may run a + crawl with a strategy this version does not list, and it parses as + an `UNKNOWN` member keeping the raw value.""" + + LINKS = "links" + + _missing_ = classmethod(open_enum_missing) + + class CrawlErrorCode(Enum): """Why a crawl failed (`Crawl.error.code`).""" INSUFFICIENT_CREDITS = "insufficient_credits" + API_KEY_CAP_REACHED = "api_key_cap_reached" SEED_UNREACHABLE = "seed_unreachable" DOMAIN_NOT_ALLOWED = "domain_not_allowed" NO_ITEMS_FOUND = "no_items_found" @@ -104,6 +117,10 @@ class Crawl(BaseModel): created without one: it has no depth limit.""" max_items: int max_pages: int + discovery: list[DiscoveryStrategy] = Field(default_factory=lambda: [DiscoveryStrategy.LINKS]) + """The strategies the crawl finds URLs with. `["links"]` when the + create left it out, which is also what this reads when the server + does not send the field at all.""" coverage: Coverage created_at: datetime stop_reason: StopReason | None = None @@ -179,6 +196,7 @@ class DownloadLine(BaseModel): "CrawlStatus", "CrawlStop", "CrawlWithResults", + "DiscoveryStrategy", "DownloadLine", "OutputFormat", "RunError", From 9ebf1f55ad513eafbb71f1e64c2352bba175a507 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Wed, 14 Oct 2026 15:19:56 +0000 Subject: [PATCH 02/10] test(crawl): read a page of the acceptance check's own crawl when the stopped one has none The check that reads a page as a file borrowed the crawl another check stops mid-flight. That stop can land before any page is fetched, and the check then skipped instead of covering anything. It now falls back to a one-item crawl of its own and waits for it, as the check beside it already did. --- acceptance/crawl/surface/test.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/acceptance/crawl/surface/test.py b/acceptance/crawl/surface/test.py index 709f353..92557ca 100644 --- a/acceptance/crawl/surface/test.py +++ b/acceptance/crawl/surface/test.py @@ -857,17 +857,27 @@ def c22(): def c23(): """content(download=True): the same page, asked for as a file to save.""" + + def first_fetched(crawl_id): + return next( + (r for r in client.get(crawl_id, limit=100).results if r.content_url), + None, + ) + + # The stop in check 20 can land before any of B's pages is fetched. Then read + # a one-item html crawl that runs to the end instead, as check 9 does. b = state.get("B") - if not b: - return SKIP, "no html crawl to read a page from", "" - fetched = next( - (r for r in client.get(b.crawl_id, limit=100).results if r.content_url), - None, - ) + src = b.crawl_id if b else None + fetched = first_fetched(src) if src else None + if not fetched: + own = create(**{**BASE, "max_items": 1, "output_format": "html"}) + client.wait(own.crawl_id) + src = own.crawl_id + fetched = first_fetched(src) if not fetched: - return SKIP, "no result had been fetched", "" - plain = client.content(b.crawl_id, fetched) - as_file = client.content(b.crawl_id, fetched, download=True) + return FAIL, f"no result with a content_url on {src}", "" + plain = client.content(src, fetched) + as_file = client.content(src, fetched, download=True) return ( pass_if(plain == as_file and bool(plain)), f"same {len(plain)} chars with and without download=True", From d7f65587d9663fea6b2e398a042f6d4847a97f57 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 08:27:55 +0000 Subject: [PATCH 03/10] chore(crawl): record the contract this surface follows The lock names the API description this surface was synced to, by its blob id, so a later drift check compares against it instead of reporting the surface as never synced. Two operations in that description are not served to callers and no surface exposes them; the lock lists them as pending, so a change to either does not read as drift here. --- acceptance/crawl/contract.lock | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 acceptance/crawl/contract.lock diff --git a/acceptance/crawl/contract.lock b/acceptance/crawl/contract.lock new file mode 100644 index 0000000..435df9b --- /dev/null +++ b/acceptance/crawl/contract.lock @@ -0,0 +1,3 @@ +1adcbc2fd87e3f508317f3ce24c3d48573aec657 +pending GET /healthz +pending POST /internal/v1/crawls/{crawl_id}/stop From b7baa72bdd683bcb4fb61411c9d947ecc1cf8665 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 12:54:34 +0000 Subject: [PATCH 04/10] docs(crawl): say that a crawl out of credits ends stopped, with error set The Crawl API now ends a crawl that runs out of credits, or whose API key hits a credit cap, as `stopped` with `error.code` `insufficient_credits` or `api_key_cap_reached` and no `stop_reason`, instead of `failed`. The pages it fetched before that stay readable. The docstrings, README and example said `error` appears only on a `failed` crawl and that `stopped` always means the caller stopped it. They now describe both cases. A unit test pins that `wait` returns such a crawl with its error and no stop reason. --- README.md | 10 +++++++--- examples/11_crawl.py | 6 +++++- src/zenrows/crawl/client.py | 2 ++ src/zenrows/crawl/models.py | 16 ++++++++++++---- tests/test_crawl_client.py | 28 ++++++++++++++++++++++++++++ 5 files changed, 54 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 6fd860b..388ce1a 100644 --- a/README.md +++ b/README.md @@ -532,7 +532,10 @@ for result in client.results(crawl.crawl_id): # follows next_cursor `wait` returns the crawl when it ends, or when `timeout` (seconds, default 600) runs out. It does not raise: on timeout the crawl is still `running` (call `wait` again, or `stop`), and a `failed` crawl comes -back with `crawl.error` set. `results` does not poll: it returns at the first empty page, so on a +back with `crawl.error` set. A crawl that ran out of credits, or whose +API key hit a credit cap, ends `stopped` with `crawl.error.code` +`insufficient_credits` or `api_key_cap_reached` and no `stop_reason`; the +pages it fetched before that stay readable. `results` does not poll: it returns at the first empty page, so on a running crawl it yields only the URLs kept so far. Call `wait` first to read every result. `results(crawl_id, limit=...)` sets the page size of each request. `get(crawl_id, cursor=..., limit=...)` returns one raw page. @@ -613,8 +616,9 @@ except CrawlAPIError as exc: | 503 | `crawl_busy` | `stop` could not be saved yet; the crawl is still running. `stop` retries it up to `retries` times, waiting `exc.retry_after` seconds, then raises it. Repeating the stop is safe. | `code` is None when the response has no problem body with a code (for -example a 502 from a proxy). A crawl that ended `failed` is not an error -response: read `crawl.error.code` and `crawl.error.detail` on it. Response enums (`CrawlStatus`, `StopReason`, +example a 502 from a proxy). A crawl that ended `failed`, or `stopped` on +credits, is not an error response: read `crawl.error.code` and +`crawl.error.detail` on it. Response enums (`CrawlStatus`, `StopReason`, `ContentStatus`, ...) are open: a value added later parses as an `UNKNOWN` member that keeps the raw value. diff --git a/examples/11_crawl.py b/examples/11_crawl.py index 1832256..da3872e 100644 --- a/examples/11_crawl.py +++ b/examples/11_crawl.py @@ -4,7 +4,8 @@ - `create(url, depth=..., include_patterns=..., output_format="html")` starts a crawl and returns at once, `running`. - `wait(crawl_id)` polls until it ends, or 600 s pass, and returns - the `Crawl` (still `running` on timeout; `error` set when `failed`). + the `Crawl` (still `running` on timeout; `error` set when `failed`, + or when it `stopped` because credits ran out or the key hit a cap). - `results(crawl_id)` pages through the kept URLs. - `content(crawl_id, result)` reads one kept URL's HTML. - `download(crawl_id)` reads every result as NDJSON, with the crawl's @@ -58,6 +59,9 @@ def main() -> None: ) if crawl.status is CrawlStatus.FAILED and crawl.error: raise SystemExit(f"crawl failed: {crawl.error.code.value}: {crawl.error.detail}") + if crawl.status is CrawlStatus.STOPPED and crawl.error: + # Out of credits or at the key's cap: what it fetched stays readable. + print(f"stopped: {crawl.error.code.value}: {crawl.error.detail}") if crawl.status is CrawlStatus.RUNNING: print("still running: the results below are what it has kept so far") diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index 7b911f9..d4e4039 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -357,6 +357,8 @@ def wait(self, crawl_id: str, *, timeout: float = 600.0) -> Crawl: `failed`) or `timeout` seconds pass, and return it. A `failed` crawl is returned, not raised: read `error` on it. + So is a crawl that `stopped` because credits ran out or the API + key hit a credit cap; `error` says which. On timeout the crawl is returned still `running`; call `wait` again to keep waiting, or `stop` to end it. """ diff --git a/src/zenrows/crawl/models.py b/src/zenrows/crawl/models.py index 35f8fda..ab01e89 100644 --- a/src/zenrows/crawl/models.py +++ b/src/zenrows/crawl/models.py @@ -32,7 +32,9 @@ class CrawlStatus(Enum): class StopReason(Enum): """What ended the crawl before nothing was left to open: a limit (`max_items` / `max_pages`, with `completed`) or the caller (`user`, - with `stopped`).""" + with `stopped`). Absent on a crawl that stopped because the account + ran out of credits or the API key hit a credit cap: that one carries + `error` instead.""" MAX_ITEMS = "max_items" MAX_PAGES = "max_pages" @@ -74,7 +76,11 @@ class DiscoveryStrategy(Enum): class CrawlErrorCode(Enum): - """Why a crawl failed (`Crawl.error.code`).""" + """Why a crawl failed, or why it stopped on credits (`Crawl.error.code`). + + `insufficient_credits` and `api_key_cap_reached` come with + `status` `stopped`: the pages fetched before that stay readable. + The other codes come with `failed`.""" INSUFFICIENT_CREDITS = "insufficient_credits" API_KEY_CAP_REACHED = "api_key_cap_reached" @@ -87,8 +93,10 @@ class CrawlErrorCode(Enum): class RunError(BaseModel): - """Why a crawl failed. Present only when `status` is `failed`; part - of the crawl, not an error response.""" + """Why a crawl failed, or why it stopped because the account ran out + of credits or the API key hit a credit cap. Present when `status` is + `failed`, and when it is `stopped` with `code` `insufficient_credits` + or `api_key_cap_reached`; part of the crawl, not an error response.""" code: CrawlErrorCode detail: str diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index 962022d..4482e04 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -232,6 +232,34 @@ def test_get_parses_failed_crawl_error(client: ZenRowsCrawlClient): assert page.error is not None and page.error.code is CrawlErrorCode.SEED_UNREACHABLE +@pytest.mark.parametrize( + ("code", "member"), + [ + ("insufficient_credits", CrawlErrorCode.INSUFFICIENT_CREDITS), + ("api_key_cap_reached", CrawlErrorCode.API_KEY_CAP_REACHED), + ], +) +@respx.mock +def test_wait_returns_crawl_stopped_on_credits( + client: ZenRowsCrawlClient, no_sleep, code: str, member: CrawlErrorCode +): + respx.get(f"{BASE_URL}/crawls/c_1").mock( + return_value=Response( + 200, + json=crawl_body( + status="stopped", + error={"code": code, "detail": "Batch stopped the crawl."}, + results=[], + next_cursor=None, + ), + ) + ) + crawl = client.wait("c_1") + assert crawl.status is CrawlStatus.STOPPED + assert crawl.stop_reason is None + assert crawl.error is not None and crawl.error.code is member + + @respx.mock def test_get_omits_unset_query_params(client: ZenRowsCrawlClient): route = respx.get(f"{BASE_URL}/crawls/c_1").mock( From c00fb918a72cf8a9cc9f75534307796ec8a0c767 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 13:46:58 +0000 Subject: [PATCH 05/10] test(crawl): use the customer wording for credit-stop details The fixtures now carry the error.detail the server returns when credits end a crawl, one per error code. --- tests/test_crawl_client.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index 4482e04..208ff36 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -232,6 +232,13 @@ def test_get_parses_failed_crawl_error(client: ZenRowsCrawlClient): assert page.error is not None and page.error.code is CrawlErrorCode.SEED_UNREACHABLE +# What the server says when credits end a crawl, as error.detail. +CREDIT_STOP_DETAIL = { + "insufficient_credits": "The crawl stopped because your account has no credits left.", + "api_key_cap_reached": "The crawl stopped because your API key reached one of its credit caps.", +} + + @pytest.mark.parametrize( ("code", "member"), [ @@ -248,7 +255,7 @@ def test_wait_returns_crawl_stopped_on_credits( 200, json=crawl_body( status="stopped", - error={"code": code, "detail": "Batch stopped the crawl."}, + error={"code": code, "detail": CREDIT_STOP_DETAIL[code]}, results=[], next_cursor=None, ), From fb2627042ceed37ef4e51776a867463e9df2c4e0 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 17:40:33 +0000 Subject: [PATCH 06/10] docs(crawl): say that patterns filter what a crawl returns, not what it opens The Crawl API now treats include and exclude patterns as a filter on what a crawl returns. The crawl still opens on-site pages that fail them, within `depth` and `max_pages`, to follow their links to pages that match, and never returns those pages. Every page it opens counts toward `max_pages` and is billed. The `create` docstring and the README now say so, and that a crawl whose patterns match nothing ends `completed` with no results. --- README.md | 5 ++++- src/zenrows/crawl/client.py | 12 +++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 388ce1a..8c9f646 100644 --- a/README.md +++ b/README.md @@ -500,7 +500,10 @@ Crawl is still evolving: new features are coming, limits may be tuned, and the c A crawl starts from one URL, follows the links on each page, and keeps the URLs that match your patterns, up to `max_items`. It stops at -`max_items` or `max_pages`, or when no link is left. `depth` limits the +`max_items` or `max_pages`, or when no link is left. Patterns filter +what the crawl returns, not what it opens: it still opens on-site pages +that fail them to reach pages that match, and every page it opens counts +toward `max_pages` and is billed. `depth` limits the link hops from the start URL; without it the crawl has no depth limit. It stays on the start URL's registrable domain; subdomains count. It runs in the background: `create` returns at once with diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index d4e4039..779fd8c 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -183,11 +183,17 @@ def create( - `max_items`: stop after keeping this many URLs (server default 100). - `max_pages`: stop after fetching this many pages, which - bounds the cost (server default 100). + bounds the cost (server default 100). Every page the crawl + opens counts and is billed, returned or not. - `include_patterns` / `exclude_patterns`: substrings matched against each normalized URL. A URL is kept when it matches - an include pattern (if any) and no exclude pattern. A crawl - stays on the start URL's registrable domain; subdomains count. + an include pattern (if any) and no exclude pattern. They + filter what the crawl returns, not what it opens: it still + opens on-site pages that fail them, to reach pages that + match, and those fetches count toward `max_pages` and are + billed. A crawl whose patterns match nothing ends `completed` + with no results. A crawl stays on the start URL's registrable + domain; subdomains count. - `output_format="html"` or `"markdown"`: also fetch every kept URL's page, as HTML or as Markdown, readable with `content` / `download`. Omit for URLs only. From a93da1e08f935d6b4680ed9548a046a3d5775823 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 17:40:33 +0000 Subject: [PATCH 07/10] test(crawl): expect a crawl whose patterns match nothing to end completed Acceptance check 19 asserted that an include pattern matching nothing ends the crawl `failed` with `no_items_found`. The Crawl API now ends such a crawl `completed` with 0 results and no `error`, so the check asserts that, and reads the crawl's results to confirm none came back. Check 21 still reads the defaults from the same crawl. The unit test fixture for a failed crawl carries the server's current `no_items_found` detail. --- acceptance/crawl/README.md | 2 +- acceptance/crawl/surface/test.py | 15 ++++++--------- tests/test_crawl_client.py | 6 +++++- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/acceptance/crawl/README.md b/acceptance/crawl/README.md index 86d782f..bedf539 100644 --- a/acceptance/crawl/README.md +++ b/acceptance/crawl/README.md @@ -78,7 +78,7 @@ and signal checks. | 16 | Errors carry the server code | SDKs: `code` holds it, empty when the body has none; MCP, CLI: `server_code` | | 17 | Naming | the API's operation and parameter names, in this surface's casing | | 18 | Copy | Crawl is "new", never "beta"; one "still evolving" line; the 429 limit reads as the account's active jobs, shared with Batch | -| 19 | Failed crawl | an include pattern that matches nothing ends `failed` with `no_items_found`; SDKs return it, not raise it; MCP and CLI say `ok: false` (CLI: exit 1, `CRAWL_FAILED`) | +| 19 | Patterns match nothing | an include pattern that matches nothing ends `completed` with 0 results and no `error`; SDKs return it; MCP and CLI say `ok: true` (CLI: exit 0) | | 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results ends at once (MCP and CLI: `partial: false`, no note to call again), and the same URLs come back once the pages in flight finish | | 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (100) | diff --git a/acceptance/crawl/surface/test.py b/acceptance/crawl/surface/test.py index 92557ca..43138ac 100644 --- a/acceptance/crawl/surface/test.py +++ b/acceptance/crawl/surface/test.py @@ -693,15 +693,12 @@ def c19(): f = state["F"] = create(depth=1, include_patterns=[NO_MATCH]) w = client.wait(f.crawl_id) err = w.error.model_dump(mode="json") if w.error else None - ok = ( - w.status.value == "failed" - and err is not None - and err.get("code") == "no_items_found" - and bool(err.get("detail")) - ) + urls = [r.url for r in client.results(f.crawl_id)] + ok = w.status.value == "completed" and err is None and w.coverage.items_found == 0 and not urls return ( pass_if(ok), - f"{f.crawl_id}: {w.status.value}; error {json.dumps(err)}; returned, no exception", + f"{f.crawl_id}: {w.status.value}; items_found {w.coverage.items_found}, " + f"{len(urls)} results; error {json.dumps(err)}", w.model_dump_json(), ) @@ -826,8 +823,8 @@ def c21(): srv.shutdown() run_check( "19", - "Failed crawl", - "a crawl that keeps nothing ends failed (no_items_found); wait returns it, not raises", + "Patterns match nothing", + "a crawl whose patterns match nothing ends completed with 0 results and no error", c19, live=True, ) diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index 208ff36..e07cab0 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -680,7 +680,11 @@ def test_wait_returns_failed_crawl(client: ZenRowsCrawlClient, no_sleep): 200, json=crawl_body( status="failed", - error={"code": "no_items_found", "detail": "No URL matched."}, + error={ + "code": "no_items_found", + "detail": "The crawl found no items: the pages it fetched held no links " + "or listings it could follow.", + }, results=[], next_cursor=None, ), From 24b876f5f944dd4059185d814b5185d56812d772 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Fri, 16 Oct 2026 11:20:58 +0000 Subject: [PATCH 08/10] docs(crawl): say that patterns limit what a crawl opens, and that 0 results is not a failure A URL that fails include_patterns or exclude_patterns is neither returned nor opened, so the docs no longer say the crawl opens such pages or bills them. A crawl whose pages held nothing new to follow ends completed with no results and no error. seed_unreachable covers an empty start page, and no_items_found means Extract found no data on the start page. --- README.md | 12 ++++++------ src/zenrows/crawl/client.py | 14 +++++--------- src/zenrows/crawl/models.py | 8 +++++++- tests/test_crawl_client.py | 5 +++-- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 8c9f646..68a15b1 100644 --- a/README.md +++ b/README.md @@ -500,10 +500,7 @@ Crawl is still evolving: new features are coming, limits may be tuned, and the c A crawl starts from one URL, follows the links on each page, and keeps the URLs that match your patterns, up to `max_items`. It stops at -`max_items` or `max_pages`, or when no link is left. Patterns filter -what the crawl returns, not what it opens: it still opens on-site pages -that fail them to reach pages that match, and every page it opens counts -toward `max_pages` and is billed. `depth` limits the +`max_items` or `max_pages`, or when no link is left. `depth` limits the link hops from the start URL; without it the crawl has no depth limit. It stays on the start URL's registrable domain; subdomains count. It runs in the background: `create` returns at once with @@ -535,8 +532,11 @@ for result in client.results(crawl.crawl_id): # follows next_cursor `wait` returns the crawl when it ends, or when `timeout` (seconds, default 600) runs out. It does not raise: on timeout the crawl is still `running` (call `wait` again, or `stop`), and a `failed` crawl comes -back with `crawl.error` set. A crawl that ran out of credits, or whose -API key hit a credit cap, ends `stopped` with `crawl.error.code` +back with `crawl.error` set. A crawl whose pages held nothing new to +follow (no link, only links to other sites, or only links the patterns +reject) ends `completed` with no results and no `error`: check the +patterns against the URLs the start page links to. A crawl that ran +out of credits, or whose API key hit a credit cap, ends `stopped` with `crawl.error.code` `insufficient_credits` or `api_key_cap_reached` and no `stop_reason`; the pages it fetched before that stay readable. `results` does not poll: it returns at the first empty page, so on a running crawl it yields only the URLs kept so far. Call `wait` first to diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index 779fd8c..dac1517 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -183,17 +183,13 @@ def create( - `max_items`: stop after keeping this many URLs (server default 100). - `max_pages`: stop after fetching this many pages, which - bounds the cost (server default 100). Every page the crawl - opens counts and is billed, returned or not. + bounds the cost (server default 100). - `include_patterns` / `exclude_patterns`: substrings matched against each normalized URL. A URL is kept when it matches - an include pattern (if any) and no exclude pattern. They - filter what the crawl returns, not what it opens: it still - opens on-site pages that fail them, to reach pages that - match, and those fetches count toward `max_pages` and are - billed. A crawl whose patterns match nothing ends `completed` - with no results. A crawl stays on the start URL's registrable - domain; subdomains count. + an include pattern (if any) and no exclude pattern. A crawl + whose patterns reject every link ends `completed` with no + results and no `error`. A crawl stays on the start URL's + registrable domain; subdomains count. - `output_format="html"` or `"markdown"`: also fetch every kept URL's page, as HTML or as Markdown, readable with `content` / `download`. Omit for URLs only. diff --git a/src/zenrows/crawl/models.py b/src/zenrows/crawl/models.py index ab01e89..5d9243f 100644 --- a/src/zenrows/crawl/models.py +++ b/src/zenrows/crawl/models.py @@ -80,7 +80,13 @@ class CrawlErrorCode(Enum): `insufficient_credits` and `api_key_cap_reached` come with `status` `stopped`: the pages fetched before that stay readable. - The other codes come with `failed`.""" + The other codes come with `failed`. `seed_unreachable` covers a + start page that could not be fetched, was blocked, or came back + empty, with no visible text and no link. `no_items_found` means + that, in a crawl that runs Extract, Extract read the start page + and found no data on it. A crawl that returns nothing for any + other reason is not a failure: it ends `completed` with no + results.""" INSUFFICIENT_CREDITS = "insufficient_credits" API_KEY_CAP_REACHED = "api_key_cap_reached" diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index e07cab0..f09eb25 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -682,8 +682,9 @@ def test_wait_returns_failed_crawl(client: ZenRowsCrawlClient, no_sleep): status="failed", error={ "code": "no_items_found", - "detail": "The crawl found no items: the pages it fetched held no links " - "or listings it could follow.", + "detail": "Extract read the start page and found no data on it, so the " + "crawl had nothing to follow. Start from a page Extract reads, such as a " + "listing or a product page.", }, results=[], next_cursor=None, From f0fa568c4beab11704df93828c6b91aae9b918a4 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Fri, 16 Oct 2026 11:25:02 +0000 Subject: [PATCH 09/10] docs(crawl): describe no_items_found and seed_unreachable without product names no_items_found now reads as the crawl finding no data it could use on the start page, with the advice to start from a listing or product page. --- src/zenrows/crawl/models.py | 14 +++++++------- tests/test_crawl_client.py | 5 ++--- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/zenrows/crawl/models.py b/src/zenrows/crawl/models.py index 5d9243f..e6858cb 100644 --- a/src/zenrows/crawl/models.py +++ b/src/zenrows/crawl/models.py @@ -80,13 +80,13 @@ class CrawlErrorCode(Enum): `insufficient_credits` and `api_key_cap_reached` come with `status` `stopped`: the pages fetched before that stay readable. - The other codes come with `failed`. `seed_unreachable` covers a - start page that could not be fetched, was blocked, or came back - empty, with no visible text and no link. `no_items_found` means - that, in a crawl that runs Extract, Extract read the start page - and found no data on it. A crawl that returns nothing for any - other reason is not a failure: it ends `completed` with no - results.""" + The other codes come with `failed`. `seed_unreachable`: the start + page could not be reached, was blocked, or came back empty, with + no visible text and no link. `no_items_found`: the crawl read the + start page but found no data it could use, so it had nothing to + follow; start from a listing or product page. A crawl that returns + nothing for any other reason is not a failure: it ends `completed` + with no results.""" INSUFFICIENT_CREDITS = "insufficient_credits" API_KEY_CAP_REACHED = "api_key_cap_reached" diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index f09eb25..4b7267f 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -682,9 +682,8 @@ def test_wait_returns_failed_crawl(client: ZenRowsCrawlClient, no_sleep): status="failed", error={ "code": "no_items_found", - "detail": "Extract read the start page and found no data on it, so the " - "crawl had nothing to follow. Start from a page Extract reads, such as a " - "listing or a product page.", + "detail": "The crawl read the start page but found no data it could use, " + "so it had nothing to follow. Start from a listing or product page.", }, results=[], next_cursor=None, From de4207f9bf5c79b41212997c6a07c5ed5a5d1b1c Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Fri, 16 Oct 2026 11:28:45 +0000 Subject: [PATCH 10/10] test(crawl): name acceptance check 19 in the report as the patterns that match nothing The check expects a completed crawl with 0 results, so the report no longer calls it a failed crawl. --- acceptance/crawl/tools/report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/crawl/tools/report.py b/acceptance/crawl/tools/report.py index 37eccf4..9f46140 100644 --- a/acceptance/crawl/tools/report.py +++ b/acceptance/crawl/tools/report.py @@ -38,7 +38,7 @@ "16": "Errors carry the server code", "17": "Naming", "18": "Copy", - "19": "Failed crawl", + "19": "Patterns match nothing", "20": "Results after stop", "21": "Documented defaults", }