diff --git a/README.md b/README.md index 6fd860b..388ce1a 100644 --- a/README.md +++ b/README.md @@ -532,7 +532,10 @@ for result in client.results(crawl.crawl_id): # follows next_cursor `wait` returns the crawl when it ends, or when `timeout` (seconds, default 600) runs out. It does not raise: on timeout the crawl is still `running` (call `wait` again, or `stop`), and a `failed` crawl comes -back with `crawl.error` set. `results` does not poll: it returns at the first empty page, so on a +back with `crawl.error` set. A crawl that ran out of credits, or whose +API key hit a credit cap, ends `stopped` with `crawl.error.code` +`insufficient_credits` or `api_key_cap_reached` and no `stop_reason`; the +pages it fetched before that stay readable. `results` does not poll: it returns at the first empty page, so on a running crawl it yields only the URLs kept so far. Call `wait` first to read every result. `results(crawl_id, limit=...)` sets the page size of each request. `get(crawl_id, cursor=..., limit=...)` returns one raw page. @@ -613,8 +616,9 @@ except CrawlAPIError as exc: | 503 | `crawl_busy` | `stop` could not be saved yet; the crawl is still running. `stop` retries it up to `retries` times, waiting `exc.retry_after` seconds, then raises it. Repeating the stop is safe. | `code` is None when the response has no problem body with a code (for -example a 502 from a proxy). A crawl that ended `failed` is not an error -response: read `crawl.error.code` and `crawl.error.detail` on it. Response enums (`CrawlStatus`, `StopReason`, +example a 502 from a proxy). A crawl that ended `failed`, or `stopped` on +credits, is not an error response: read `crawl.error.code` and +`crawl.error.detail` on it. Response enums (`CrawlStatus`, `StopReason`, `ContentStatus`, ...) are open: a value added later parses as an `UNKNOWN` member that keeps the raw value. diff --git a/examples/11_crawl.py b/examples/11_crawl.py index 1832256..da3872e 100644 --- a/examples/11_crawl.py +++ b/examples/11_crawl.py @@ -4,7 +4,8 @@ - `create(url, depth=..., include_patterns=..., output_format="html")` starts a crawl and returns at once, `running`. - `wait(crawl_id)` polls until it ends, or 600 s pass, and returns - the `Crawl` (still `running` on timeout; `error` set when `failed`). + the `Crawl` (still `running` on timeout; `error` set when `failed`, + or when it `stopped` because credits ran out or the key hit a cap). - `results(crawl_id)` pages through the kept URLs. - `content(crawl_id, result)` reads one kept URL's HTML. - `download(crawl_id)` reads every result as NDJSON, with the crawl's @@ -58,6 +59,9 @@ def main() -> None: ) if crawl.status is CrawlStatus.FAILED and crawl.error: raise SystemExit(f"crawl failed: {crawl.error.code.value}: {crawl.error.detail}") + if crawl.status is CrawlStatus.STOPPED and crawl.error: + # Out of credits or at the key's cap: what it fetched stays readable. + print(f"stopped: {crawl.error.code.value}: {crawl.error.detail}") if crawl.status is CrawlStatus.RUNNING: print("still running: the results below are what it has kept so far") diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index 7b911f9..d4e4039 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -357,6 +357,8 @@ def wait(self, crawl_id: str, *, timeout: float = 600.0) -> Crawl: `failed`) or `timeout` seconds pass, and return it. A `failed` crawl is returned, not raised: read `error` on it. + So is a crawl that `stopped` because credits ran out or the API + key hit a credit cap; `error` says which. On timeout the crawl is returned still `running`; call `wait` again to keep waiting, or `stop` to end it. """ diff --git a/src/zenrows/crawl/models.py b/src/zenrows/crawl/models.py index 35f8fda..ab01e89 100644 --- a/src/zenrows/crawl/models.py +++ b/src/zenrows/crawl/models.py @@ -32,7 +32,9 @@ class CrawlStatus(Enum): class StopReason(Enum): """What ended the crawl before nothing was left to open: a limit (`max_items` / `max_pages`, with `completed`) or the caller (`user`, - with `stopped`).""" + with `stopped`). Absent on a crawl that stopped because the account + ran out of credits or the API key hit a credit cap: that one carries + `error` instead.""" MAX_ITEMS = "max_items" MAX_PAGES = "max_pages" @@ -74,7 +76,11 @@ class DiscoveryStrategy(Enum): class CrawlErrorCode(Enum): - """Why a crawl failed (`Crawl.error.code`).""" + """Why a crawl failed, or why it stopped on credits (`Crawl.error.code`). + + `insufficient_credits` and `api_key_cap_reached` come with + `status` `stopped`: the pages fetched before that stay readable. + The other codes come with `failed`.""" INSUFFICIENT_CREDITS = "insufficient_credits" API_KEY_CAP_REACHED = "api_key_cap_reached" @@ -87,8 +93,10 @@ class CrawlErrorCode(Enum): class RunError(BaseModel): - """Why a crawl failed. Present only when `status` is `failed`; part - of the crawl, not an error response.""" + """Why a crawl failed, or why it stopped because the account ran out + of credits or the API key hit a credit cap. Present when `status` is + `failed`, and when it is `stopped` with `code` `insufficient_credits` + or `api_key_cap_reached`; part of the crawl, not an error response.""" code: CrawlErrorCode detail: str diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index 962022d..208ff36 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -232,6 +232,41 @@ def test_get_parses_failed_crawl_error(client: ZenRowsCrawlClient): assert page.error is not None and page.error.code is CrawlErrorCode.SEED_UNREACHABLE +# What the server says when credits end a crawl, as error.detail. +CREDIT_STOP_DETAIL = { + "insufficient_credits": "The crawl stopped because your account has no credits left.", + "api_key_cap_reached": "The crawl stopped because your API key reached one of its credit caps.", +} + + +@pytest.mark.parametrize( + ("code", "member"), + [ + ("insufficient_credits", CrawlErrorCode.INSUFFICIENT_CREDITS), + ("api_key_cap_reached", CrawlErrorCode.API_KEY_CAP_REACHED), + ], +) +@respx.mock +def test_wait_returns_crawl_stopped_on_credits( + client: ZenRowsCrawlClient, no_sleep, code: str, member: CrawlErrorCode +): + respx.get(f"{BASE_URL}/crawls/c_1").mock( + return_value=Response( + 200, + json=crawl_body( + status="stopped", + error={"code": code, "detail": CREDIT_STOP_DETAIL[code]}, + results=[], + next_cursor=None, + ), + ) + ) + crawl = client.wait("c_1") + assert crawl.status is CrawlStatus.STOPPED + assert crawl.stop_reason is None + assert crawl.error is not None and crawl.error.code is member + + @respx.mock def test_get_omits_unset_query_params(client: ZenRowsCrawlClient): route = respx.get(f"{BASE_URL}/crawls/c_1").mock(