diff --git a/README.md b/README.md index 388ce1a..68a15b1 100644 --- a/README.md +++ b/README.md @@ -532,8 +532,11 @@ for result in client.results(crawl.crawl_id): # follows next_cursor `wait` returns the crawl when it ends, or when `timeout` (seconds, default 600) runs out. It does not raise: on timeout the crawl is still `running` (call `wait` again, or `stop`), and a `failed` crawl comes -back with `crawl.error` set. A crawl that ran out of credits, or whose -API key hit a credit cap, ends `stopped` with `crawl.error.code` +back with `crawl.error` set. A crawl whose pages held nothing new to +follow (no link, only links to other sites, or only links the patterns +reject) ends `completed` with no results and no `error`: check the +patterns against the URLs the start page links to. A crawl that ran +out of credits, or whose API key hit a credit cap, ends `stopped` with `crawl.error.code` `insufficient_credits` or `api_key_cap_reached` and no `stop_reason`; the pages it fetched before that stay readable. `results` does not poll: it returns at the first empty page, so on a running crawl it yields only the URLs kept so far. Call `wait` first to diff --git a/acceptance/crawl/README.md b/acceptance/crawl/README.md index 86d782f..bedf539 100644 --- a/acceptance/crawl/README.md +++ b/acceptance/crawl/README.md @@ -78,7 +78,7 @@ and signal checks. | 16 | Errors carry the server code | SDKs: `code` holds it, empty when the body has none; MCP, CLI: `server_code` | | 17 | Naming | the API's operation and parameter names, in this surface's casing | | 18 | Copy | Crawl is "new", never "beta"; one "still evolving" line; the 429 limit reads as the account's active jobs, shared with Batch | -| 19 | Failed crawl | an include pattern that matches nothing ends `failed` with `no_items_found`; SDKs return it, not raise it; MCP and CLI say `ok: false` (CLI: exit 1, `CRAWL_FAILED`) | +| 19 | Patterns match nothing | an include pattern that matches nothing ends `completed` with 0 results and no `error`; SDKs return it; MCP and CLI say `ok: true` (CLI: exit 0) | | 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results ends at once (MCP and CLI: `partial: false`, no note to call again), and the same URLs come back once the pages in flight finish | | 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (100) | diff --git a/acceptance/crawl/contract.lock b/acceptance/crawl/contract.lock new file mode 100644 index 0000000..435df9b --- /dev/null +++ b/acceptance/crawl/contract.lock @@ -0,0 +1,3 @@ +1adcbc2fd87e3f508317f3ce24c3d48573aec657 +pending GET /healthz +pending POST /internal/v1/crawls/{crawl_id}/stop diff --git a/acceptance/crawl/surface/test.py b/acceptance/crawl/surface/test.py index 2203ce2..eef2aba 100644 --- a/acceptance/crawl/surface/test.py +++ b/acceptance/crawl/surface/test.py @@ -704,15 +704,12 @@ def c19(): f = state["F"] = create(depth=1, include_patterns=[NO_MATCH]) w = client.wait(f.crawl_id) err = w.error.model_dump(mode="json") if w.error else None - ok = ( - w.status.value == "failed" - and err is not None - and err.get("code") == "no_items_found" - and bool(err.get("detail")) - ) + urls = [r.url for r in client.results(f.crawl_id)] + ok = w.status.value == "completed" and err is None and w.coverage.items_found == 0 and not urls return ( pass_if(ok), - f"{f.crawl_id}: {w.status.value}; error {json.dumps(err)}; returned, no exception", + f"{f.crawl_id}: {w.status.value}; items_found {w.coverage.items_found}, " + f"{len(urls)} results; error {json.dumps(err)}", w.model_dump_json(), ) @@ -837,8 +834,8 @@ def c21(): srv.shutdown() run_check( "19", - "Failed crawl", - "a crawl that keeps nothing ends failed (no_items_found); wait returns it, not raises", + "Patterns match nothing", + "a crawl whose patterns match nothing ends completed with 0 results and no error", c19, live=True, ) diff --git a/acceptance/crawl/tools/report.py b/acceptance/crawl/tools/report.py index 37eccf4..9f46140 100644 --- a/acceptance/crawl/tools/report.py +++ b/acceptance/crawl/tools/report.py @@ -38,7 +38,7 @@ "16": "Errors carry the server code", "17": "Naming", "18": "Copy", - "19": "Failed crawl", + "19": "Patterns match nothing", "20": "Results after stop", "21": "Documented defaults", } diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index d4e4039..dac1517 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -187,7 +187,9 @@ def create( - `include_patterns` / `exclude_patterns`: substrings matched against each normalized URL. A URL is kept when it matches an include pattern (if any) and no exclude pattern. A crawl - stays on the start URL's registrable domain; subdomains count. + whose patterns reject every link ends `completed` with no + results and no `error`. A crawl stays on the start URL's + registrable domain; subdomains count. - `output_format="html"` or `"markdown"`: also fetch every kept URL's page, as HTML or as Markdown, readable with `content` / `download`. Omit for URLs only. diff --git a/src/zenrows/crawl/models.py b/src/zenrows/crawl/models.py index ab01e89..e6858cb 100644 --- a/src/zenrows/crawl/models.py +++ b/src/zenrows/crawl/models.py @@ -80,7 +80,13 @@ class CrawlErrorCode(Enum): `insufficient_credits` and `api_key_cap_reached` come with `status` `stopped`: the pages fetched before that stay readable. - The other codes come with `failed`.""" + The other codes come with `failed`. `seed_unreachable`: the start + page could not be reached, was blocked, or came back empty, with + no visible text and no link. `no_items_found`: the crawl read the + start page but found no data it could use, so it had nothing to + follow; start from a listing or product page. A crawl that returns + nothing for any other reason is not a failure: it ends `completed` + with no results.""" INSUFFICIENT_CREDITS = "insufficient_credits" API_KEY_CAP_REACHED = "api_key_cap_reached" diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index 208ff36..4b7267f 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -680,7 +680,11 @@ def test_wait_returns_failed_crawl(client: ZenRowsCrawlClient, no_sleep): 200, json=crawl_body( status="failed", - error={"code": "no_items_found", "detail": "No URL matched."}, + error={ + "code": "no_items_found", + "detail": "The crawl read the start page but found no data it could use, " + "so it had nothing to follow. Start from a listing or product page.", + }, results=[], next_cursor=None, ),