From d7b7bcad8e1f65bd2e41594e0483c7c8d17225b0 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 08:47:11 +0000 Subject: [PATCH 1/4] feat(extract): stop falling back to autoparse on REQS007 by default extract() and extract_async() no longer retry with autoparse when Extract answers 403 REQS007 (the domain isn't prepared for Extract yet) or the legacy 402 AUTH010. Both error responses are now returned to the caller as is, without a second request. The fallback stays available as an opt-in: pass fallback_to_autoparse=True to retry once with deprecated autoparse. To serve a domain that isn't prepared yet, prepare it with POST /v1/extract/prepared-domains. The README describes REQS007 and the opt-in fallback, and marks the autoparse fetch parameter as deprecated. --- README.md | 10 ++- src/zenrows/client.py | 33 +++++--- tests/test_fetch_extract.py | 155 +++++++++++++++++++++--------------- 3 files changed, 119 insertions(+), 79 deletions(-) diff --git a/README.md b/README.md index 6fd860b..aaeb9de 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ client = ZenRowsClient("YOUR-API-KEY", retries=1) url = "https://www.zenrows.com/" response = client.fetch(url, params={ - # Our algorithm allows to automatically extract content from any website + # Deprecated: use client.extract() for structured data instead "autoparse": False, # CSS Selectors for data extraction (i.e. {"links":"a @href"} to get href attributes from links) @@ -204,6 +204,14 @@ print(response.json()) `extract()` also sends [Adaptive Stealth Mode](https://docs.zenrows.com/fetch/features/adaptive-stealth-mode) (`mode: "auto"`) by default, except when `params` sets `js_render` or `premium_proxy`: the API rejects either combined with Adaptive Stealth Mode, so your fixed configuration is sent as is. Pass `adaptive_stealth=False` to turn it off entirely. +Extract works on any prepared domain (you can prepare one yourself with `POST /v1/extract/prepared-domains`). A `403` with `code: "REQS007"` means the domain isn't prepared for Extract yet: prepare it with `POST /v1/extract/prepared-domains`, or pass `fallback_to_autoparse=True` to retry with deprecated autoparse. The legacy `402` with `code: "AUTH010"` is handled the same way. By default `extract()` returns both error responses to you as is, without a second request. + +```python +# Opt in to one retry with the deprecated autoparse on REQS007 / AUTH010. +# The retry is a second, separately charged request. +response = client.extract(url, fallback_to_autoparse=True) +``` + A `422` with `code: "no_data_extracted"` means Extract knows this kind of page but found no data on it. It isn't charged, and the client never retries it, even with `retries` set. `is_no_data_extracted(response)` tells you when you got one. See the [Extract docs](https://docs.zenrows.com/extract/setup). ## Quickstart — Batch API (`ZenRowsBatchClient`) diff --git a/src/zenrows/client.py b/src/zenrows/client.py index 2a38502..9cae501 100644 --- a/src/zenrows/client.py +++ b/src/zenrows/client.py @@ -58,7 +58,8 @@ def is_no_data_extracted(response: requests.Response) -> bool: # Extract answers these when it can't serve the target domain: 403 REQS007 # (domain not prepared yet) or the legacy 402 AUTH010 (domain not enabled). -# extract(mode="auto") retries both once with Autoparse. +# extract(mode="auto", fallback_to_autoparse=True) retries both once with the +# deprecated Autoparse; by default they are returned to the caller as is. _EXTRACT_DOMAIN_UNAVAILABLE = {(403, "REQS007"), (402, "AUTH010")} @@ -168,7 +169,7 @@ def extract( params: dict | None = None, headers: dict | None = None, mode: str = "auto", - fallback_to_autoparse: bool = True, + fallback_to_autoparse: bool = False, adaptive_stealth: bool = True, **kwargs: Any, ) -> requests.Response: @@ -177,13 +178,18 @@ def extract( or "standard". Thin wrapper over `fetch()` with the `extract` param set: no separate endpoint or auth. - Extract works on any domain once Zenrows has prepared it, with any - active API key. A domain that isn't prepared yet answers 403 with - `code: "REQS007"` (prepare it with `POST /v1/extract/prepared-domains`); - the legacy domain-not-enabled code is a 402 with `code: "AUTH010"`. - On either, `mode="auto"` retries once with `autoparse=True` instead of - returning the error response. Pass `fallback_to_autoparse=False` to - disable that and get the raw error response back. + Extract works on any prepared domain (you can prepare one yourself + with `POST /v1/extract/prepared-domains`), with any active API key. + A 403 with `code: "REQS007"` means the domain isn't prepared for + Extract yet: prepare it with `POST /v1/extract/prepared-domains`, or + pass `fallback_to_autoparse=True` to retry with deprecated autoparse. + The legacy domain-not-enabled code is a 402 with `code: "AUTH010"`. + By default both error responses are returned to you as is. + + `fallback_to_autoparse=True` (opt-in, default False) makes + `mode="auto"` retry once with `autoparse=True` on either code instead + of returning the error response. Autoparse is deprecated, and the + retry is a second, separately charged request. A 422 with `code: "no_data_extracted"` means Extract knows this kind of page but found no data on it. It is not charged and is never @@ -191,7 +197,8 @@ def extract( `adaptive_stealth=True` (default) also sends Adaptive Stealth Mode (`mode: "auto"` at the wire-param level - unrelated to this method's own - `mode` argument) on both the extract attempt and the Autoparse fallback, + `mode` argument) on the extract attempt and on the Autoparse fallback + when `fallback_to_autoparse=True`, so a target needing `js_render`/`premium_proxy` gets escalated automatically instead of failing with REQS002. It is skipped when `params` sets `js_render` or `premium_proxy` (the API rejects either @@ -258,13 +265,13 @@ async def extract_async( params: dict | None = None, headers: dict | None = None, mode: str = "auto", - fallback_to_autoparse: bool = True, + fallback_to_autoparse: bool = False, adaptive_stealth: bool = True, **kwargs: Any, ) -> requests.Response: """Async counterpart of `extract()` - see its docstring for the - REQS007 / AUTH010 -> Autoparse fallback behavior, - `fallback_to_autoparse`, and `adaptive_stealth`.""" + REQS007 / AUTH010 handling, the opt-in `fallback_to_autoparse`, and + `adaptive_stealth`.""" final_params = dict(params) if params else {} final_params["extract"] = mode _with_adaptive_stealth(final_params, adaptive_stealth) diff --git a/tests/test_fetch_extract.py b/tests/test_fetch_extract.py index f417928..fdf0248 100644 --- a/tests/test_fetch_extract.py +++ b/tests/test_fetch_extract.py @@ -126,20 +126,47 @@ def test_extract_does_not_mutate_caller_supplied_params_dict(self, mock_request) class TestExtractAutoparseFallback(TestCase): """`extract(mode="auto")`: 403 REQS007 means the target domain isn't prepared for Extract yet, and the legacy 402 AUTH010 that it isn't - enabled. On either, by default this retries once with Autoparse instead - of returning the error - same behavior as the CLI's extract adapter.""" + enabled. By default both are returned as is, without a second request. + With `fallback_to_autoparse=True` the client retries exactly once with + the deprecated Autoparse.""" def setUp(self): self.client = ZenRowsClient(apikey) @mock.patch.object(Session, "request") - def test_falls_back_to_autoparse_on_auth010(self, mock_request): + def test_reqs007_is_returned_without_fallback_by_default(self, mock_request): + mock_request.return_value = _fake_response( + 403, b'{"code": "REQS007", "title": "Domain not prepared"}' + ) + + response = self.client.extract(url) + + self.assertEqual(response.status_code, 403) + self.assertEqual(mock_request.call_count, 1) + _, kwargs = mock_request.call_args + self.assertNotIn("autoparse", kwargs["params"]) + + @mock.patch.object(Session, "request") + def test_auth010_is_returned_without_fallback_by_default(self, mock_request): + mock_request.return_value = _fake_response( + 402, b'{"code": "AUTH010", "title": "Domain not enabled"}' + ) + + response = self.client.extract(url) + + self.assertEqual(response.status_code, 402) + self.assertEqual(mock_request.call_count, 1) + _, kwargs = mock_request.call_args + self.assertNotIn("autoparse", kwargs["params"]) + + @mock.patch.object(Session, "request") + def test_opt_in_falls_back_to_autoparse_on_reqs007(self, mock_request): mock_request.side_effect = [ - _fake_response(402, b'{"code": "AUTH010", "title": "Domain not enabled"}'), + _fake_response(403, b'{"code": "REQS007", "title": "Domain not prepared"}'), _fake_response(200, b'[{"found": "via autoparse"}]'), ] - response = self.client.extract(url) + response = self.client.extract(url, fallback_to_autoparse=True) self.assertEqual(response.status_code, 200) self.assertEqual(mock_request.call_count, 2) @@ -148,13 +175,13 @@ def test_falls_back_to_autoparse_on_auth010(self, mock_request): self.assertNotIn("extract", fallback_kwargs["params"]) @mock.patch.object(Session, "request") - def test_falls_back_to_autoparse_on_reqs007(self, mock_request): + def test_opt_in_falls_back_to_autoparse_on_auth010(self, mock_request): mock_request.side_effect = [ - _fake_response(403, b'{"code": "REQS007", "title": "Domain not prepared"}'), + _fake_response(402, b'{"code": "AUTH010", "title": "Domain not enabled"}'), _fake_response(200, b'[{"found": "via autoparse"}]'), ] - response = self.client.extract(url) + response = self.client.extract(url, fallback_to_autoparse=True) self.assertEqual(response.status_code, 200) self.assertEqual(mock_request.call_count, 2) @@ -163,70 +190,53 @@ def test_falls_back_to_autoparse_on_reqs007(self, mock_request): self.assertNotIn("extract", fallback_kwargs["params"]) @mock.patch.object(Session, "request") - def test_reqs007_fallback_disabled_returns_error_response(self, mock_request): - mock_request.return_value = _fake_response(403, b'{"code": "REQS007"}') - - response = self.client.extract(url, fallback_to_autoparse=False) - - self.assertEqual(response.status_code, 403) - self.assertEqual(mock_request.call_count, 1) - - @mock.patch.object(Session, "request") - def test_403_without_reqs007_does_not_fall_back(self, mock_request): - """A Zenrows auth 403 must not be mistaken for a domain that isn't - prepared yet.""" - mock_request.return_value = _fake_response( - 403, b'{"code": "AUTH002", "title": "Invalid API key"}' - ) + def test_opt_in_retries_autoparse_only_once(self, mock_request): + """A fallback that fails again is returned, not retried a second time.""" + mock_request.side_effect = [ + _fake_response(403, b'{"code": "REQS007"}'), + _fake_response(403, b'{"code": "REQS007"}'), + _fake_response(200, b"{}"), + ] - response = self.client.extract(url) + response = self.client.extract(url, fallback_to_autoparse=True) self.assertEqual(response.status_code, 403) - self.assertEqual(mock_request.call_count, 1) - - @mock.patch.object(Session, "request") - def test_code_on_the_wrong_status_does_not_fall_back(self, mock_request): - """REQS007 is only the not-prepared signal on a 403.""" - mock_request.return_value = _fake_response(400, b'{"code": "REQS007"}') - - response = self.client.extract(url) - - self.assertEqual(response.status_code, 400) - self.assertEqual(mock_request.call_count, 1) - - @mock.patch.object(Session, "request") - def test_fallback_disabled_returns_error_response(self, mock_request): - mock_request.return_value = _fake_response(402, b'{"code": "AUTH010"}') - - response = self.client.extract(url, fallback_to_autoparse=False) - - self.assertEqual(response.status_code, 402) - self.assertEqual(mock_request.call_count, 1) + self.assertEqual(mock_request.call_count, 2) @mock.patch.object(Session, "request") - def test_402_without_auth010_does_not_fall_back(self, mock_request): - """A real credits-exhausted 402 (e.g. AUTH004) must not be mistaken - for the domain-not-enabled error.""" - mock_request.return_value = _fake_response( - 402, b'{"code": "AUTH004", "title": "No credit available"}' - ) + def test_other_4xx_never_fall_back(self, mock_request): + """Neither an auth 403, a credits-exhausted 402, nor REQS007 / AUTH010 + on the wrong status is mistaken for a domain that isn't prepared, + even with the fallback on.""" + cases = [ + (403, b'{"code": "AUTH002", "title": "Invalid API key"}'), + (402, b'{"code": "AUTH004", "title": "No credit available"}'), + (400, b'{"code": "REQS007"}'), + (400, b'{"code": "AUTH010"}'), + (404, b"{}"), + ] + for status, body in cases: + for fallback in (False, True): + with self.subTest(status=status, body=body, fallback=fallback): + mock_request.reset_mock() + mock_request.return_value = _fake_response(status, body) - response = self.client.extract(url) + response = self.client.extract(url, fallback_to_autoparse=fallback) - self.assertEqual(response.status_code, 402) - self.assertEqual(mock_request.call_count, 1) + self.assertEqual(response.status_code, status) + self.assertEqual(mock_request.call_count, 1) @mock.patch.object(Session, "request") def test_no_fallback_for_non_auto_mode(self, mock_request): """Neither code triggers a fallback for native/standard modes - - only "auto" takes the fallback path.""" + only "auto" takes the fallback path, even when it is turned on.""" mock_request.side_effect = [ _fake_response(402, b'{"code": "AUTH010"}'), _fake_response(403, b'{"code": "REQS007"}'), ] - first = self.client.extract(url, mode="native") - second = self.client.extract(url, mode="standard") + first = self.client.extract(url, mode="native", fallback_to_autoparse=True) + second = self.client.extract(url, mode="standard", fallback_to_autoparse=True) self.assertEqual(first.status_code, 402) self.assertEqual(second.status_code, 403) @@ -240,7 +250,7 @@ def test_fallback_does_not_mutate_caller_supplied_params_dict(self, mock_request ] caller_params = {"js_render": True} - self.client.extract(url, params=caller_params) + self.client.extract(url, params=caller_params, fallback_to_autoparse=True) self.assertEqual(caller_params, {"js_render": True}) @@ -278,7 +288,7 @@ def test_fallback_request_also_carries_adaptive_stealth(self, mock_request): _fake_response(200, b"{}"), ] - self.client.extract(url) + self.client.extract(url, fallback_to_autoparse=True) _, fallback_kwargs = mock_request.call_args self.assertEqual(fallback_kwargs["params"]["mode"], "auto") @@ -329,7 +339,7 @@ def test_fallback_request_skips_adaptive_stealth_with_premium_proxy(self, mock_r _fake_response(200, b"{}"), ] - self.client.extract(url, params={"premium_proxy": True}) + self.client.extract(url, params={"premium_proxy": True}, fallback_to_autoparse=True) first_kwargs = mock_request.call_args_list[0].kwargs _, fallback_kwargs = mock_request.call_args @@ -339,19 +349,34 @@ def test_fallback_request_skips_adaptive_stealth_with_premium_proxy(self, mock_r class TestExtractAsyncAutoparseFallback(IsolatedAsyncioTestCase): - """Async counterpart - same REQS007 / AUTH010 -> Autoparse behavior.""" + """Async counterpart - same REQS007 / AUTH010 handling: returned as is by + default, one Autoparse retry when `fallback_to_autoparse=True`.""" def setUp(self): self.client = ZenRowsClient(apikey, concurrency=2) @mock.patch.object(Session, "request") - async def test_falls_back_to_autoparse_on_auth010(self, mock_request): + async def test_reqs007_and_auth010_are_returned_without_fallback_by_default(self, mock_request): + mock_request.side_effect = [ + _fake_response(403, b'{"code": "REQS007"}'), + _fake_response(402, b'{"code": "AUTH010"}'), + ] + + first = await self.client.extract_async(url) + second = await self.client.extract_async(url) + + self.assertEqual(first.status_code, 403) + self.assertEqual(second.status_code, 402) + self.assertEqual(mock_request.call_count, 2) + + @mock.patch.object(Session, "request") + async def test_opt_in_falls_back_to_autoparse_on_auth010(self, mock_request): mock_request.side_effect = [ _fake_response(402, b'{"code": "AUTH010"}'), _fake_response(200, b'[{"found": "via autoparse"}]'), ] - response = await self.client.extract_async(url) + response = await self.client.extract_async(url, fallback_to_autoparse=True) self.assertEqual(response.status_code, 200) self.assertEqual(mock_request.call_count, 2) @@ -363,7 +388,7 @@ async def test_skips_adaptive_stealth_when_js_render_is_set(self, mock_request): _fake_response(200, b"{}"), ] - await self.client.extract_async(url, params={"js_render": True}) + await self.client.extract_async(url, params={"js_render": True}, fallback_to_autoparse=True) first_kwargs = mock_request.call_args_list[0].kwargs _, fallback_kwargs = mock_request.call_args @@ -371,13 +396,13 @@ async def test_skips_adaptive_stealth_when_js_render_is_set(self, mock_request): self.assertNotIn("mode", fallback_kwargs["params"]) @mock.patch.object(Session, "request") - async def test_falls_back_to_autoparse_on_reqs007(self, mock_request): + async def test_opt_in_falls_back_to_autoparse_on_reqs007(self, mock_request): mock_request.side_effect = [ _fake_response(403, b'{"code": "REQS007"}'), _fake_response(200, b'[{"found": "via autoparse"}]'), ] - response = await self.client.extract_async(url) + response = await self.client.extract_async(url, fallback_to_autoparse=True) self.assertEqual(response.status_code, 200) self.assertEqual(mock_request.call_count, 2) From 94ab2157894f9f52753a9a55471eea0382639cc1 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 09:39:30 +0000 Subject: [PATCH 2/4] feat(extract): add prepare_domain and prepared_domain_status ZenRowsClient.prepare_domain(domain, example_urls) submits a domain for Extract with POST /v1/extract/prepared-domains, and prepared_domain_status(domain) reads its status with GET /v1/extract/prepared-domains/{domain}. Both send the API key in the X-API-Key header and return the parsed status object. Neither waits for the preparation: callers poll the status themselves. An error answer raises the new PreparedDomainError, which carries the status code, the problem code (invalid_argument, not_found, in_progress, out_of_allowance, overloaded, disabled), detail, reason, Retry-After and the whole problem body. The extract() docstring and the README point REQS007 at both methods. --- README.md | 35 +++++- src/zenrows/__init__.py | 3 +- src/zenrows/client.py | 128 ++++++++++++++++++++- tests/test_prepared_domains.py | 201 +++++++++++++++++++++++++++++++++ 4 files changed, 363 insertions(+), 4 deletions(-) create mode 100644 tests/test_prepared_domains.py diff --git a/README.md b/README.md index aaeb9de..916a110 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ This package ships three clients: - [PUT Requests](#put-requests) - [Concurrency](#concurrency) - [Extract](#extract) + - [Preparing a domain](#preparing-a-domain) - [Quickstart — Batch API (`ZenRowsBatchClient`)](#quickstart--batch-api-zenrowsbatchclient) - [Extract in a batch](#extract-in-a-batch) - [Upload URLs from a CSV](#upload-urls-from-a-csv) @@ -204,7 +205,7 @@ print(response.json()) `extract()` also sends [Adaptive Stealth Mode](https://docs.zenrows.com/fetch/features/adaptive-stealth-mode) (`mode: "auto"`) by default, except when `params` sets `js_render` or `premium_proxy`: the API rejects either combined with Adaptive Stealth Mode, so your fixed configuration is sent as is. Pass `adaptive_stealth=False` to turn it off entirely. -Extract works on any prepared domain (you can prepare one yourself with `POST /v1/extract/prepared-domains`). A `403` with `code: "REQS007"` means the domain isn't prepared for Extract yet: prepare it with `POST /v1/extract/prepared-domains`, or pass `fallback_to_autoparse=True` to retry with deprecated autoparse. The legacy `402` with `code: "AUTH010"` is handled the same way. By default `extract()` returns both error responses to you as is, without a second request. +Extract works on any prepared domain (you can prepare one yourself with `POST /v1/extract/prepared-domains`). A `403` with `code: "REQS007"` means the domain isn't prepared for Extract yet: prepare it with `POST /v1/extract/prepared-domains` (`client.prepare_domain()`, see below), or pass `fallback_to_autoparse=True` to retry with deprecated autoparse. The legacy `402` with `code: "AUTH010"` is handled the same way. By default `extract()` returns both error responses to you as is, without a second request. ```python # Opt in to one retry with the deprecated autoparse on REQS007 / AUTH010. @@ -212,6 +213,38 @@ Extract works on any prepared domain (you can prepare one yourself with `POST /v response = client.extract(url, fallback_to_autoparse=True) ``` +#### Preparing a domain + +`client.prepare_domain(domain, example_urls)` submits a domain for Extract (`POST /v1/extract/prepared-domains`), and `client.prepared_domain_status(domain)` reads where it is (`GET /v1/extract/prepared-domains/{domain}`). Both return the parsed status object. Pass the domain without a scheme and 1 to 5 example URLs, one per kind of page you want to extract. These calls cost no credits. + +Preparation takes minutes, and the client does not wait for it. Poll the status until it is no longer `"preparing"`: + +```python +import time + +from zenrows import PreparedDomainError, ZenRowsClient + +client = ZenRowsClient("YOUR-API-KEY") + +try: + status = client.prepare_domain( + "www.scrapingcourse.com", + ["https://www.scrapingcourse.com/ecommerce/"], + ) +except PreparedDomainError as err: + # e.g. 409 in_progress, 429 out_of_allowance, 503 overloaded + print(err.status_code, err.code, err.detail) + raise + +while status["status"] == "preparing": + time.sleep(30) + status = client.prepared_domain_status("www.scrapingcourse.com") + +print(status["status"]) # "ready" or "failed" (see status["failure"]) +``` + +Both methods raise `PreparedDomainError` on an error answer, with `status_code`, `code` (`invalid_argument`, `not_found`, `in_progress`, `out_of_allowance`, `overloaded`, `disabled`, ...), `detail`, `reason` and the whole `problem` body. See the [Extract endpoints docs](https://docs.zenrows.com/extract/endpoints) for every status and failure code. + A `422` with `code: "no_data_extracted"` means Extract knows this kind of page but found no data on it. It isn't charged, and the client never retries it, even with `retries` set. `is_no_data_extracted(response)` tells you when you got one. See the [Extract docs](https://docs.zenrows.com/extract/setup). ## Quickstart — Batch API (`ZenRowsBatchClient`) diff --git a/src/zenrows/__init__.py b/src/zenrows/__init__.py index 8a13b06..75626f1 100644 --- a/src/zenrows/__init__.py +++ b/src/zenrows/__init__.py @@ -15,11 +15,12 @@ from zenrows.__version__ import __version__ from zenrows.batch import ZenRowsBatchClient -from zenrows.client import ZenRowsClient, is_no_data_extracted +from zenrows.client import PreparedDomainError, ZenRowsClient, is_no_data_extracted from zenrows.crawl import CrawlAPIError, ZenRowsCrawlClient __all__ = [ "CrawlAPIError", + "PreparedDomainError", "ZenRowsBatchClient", "ZenRowsClient", "ZenRowsCrawlClient", diff --git a/src/zenrows/client.py b/src/zenrows/client.py index 9cae501..3836101 100644 --- a/src/zenrows/client.py +++ b/src/zenrows/client.py @@ -12,11 +12,13 @@ """ import asyncio +import math import os import time from concurrent.futures import ThreadPoolExecutor from functools import partial from typing import Any +from urllib.parse import quote import requests import urllib3 @@ -63,6 +65,18 @@ def is_no_data_extracted(response: requests.Response) -> bool: _EXTRACT_DOMAIN_UNAVAILABLE = {(403, "REQS007"), (402, "AUTH010")} +def _prepared_domain_body(response: requests.Response) -> dict: + if not response.ok: + raise PreparedDomainError.from_response(response) + try: + body = response.json() + except ValueError: + body = None + if not isinstance(body, dict): + raise PreparedDomainError(response.status_code, None) + return body + + def _is_extract_domain_unavailable(response: requests.Response) -> bool: """True when the response is Extract saying it can't serve the target domain yet (403 REQS007, or the legacy 402 AUTH010).""" @@ -72,6 +86,61 @@ def _is_extract_domain_unavailable(response: requests.Response) -> bool: return (response.status_code, code.upper()) in _EXTRACT_DOMAIN_UNAVAILABLE +class PreparedDomainError(Exception): + """A non-2xx answer from the prepared-domains endpoints + (`prepare_domain()` / `prepared_domain_status()`). + + Branch on `status_code` + `code`; display `detail`, don't parse it. + `code` is the problem's `code` member, for example `invalid_argument` + (400), `not_found` (404), `in_progress` (409: a preparation is already + running, read its status instead of resubmitting), `out_of_allowance` + (429: no preparations left this month), or `overloaded` / `disabled` + (503); None when the body has no code. `problem` holds the whole body, + or None when it was not JSON. + """ + + def __init__( + self, + status_code: int, + problem: dict | None, + *, + retry_after: float | None = None, + ): + self.status_code = status_code + self.problem = problem + # Seconds the server asked to wait (`Retry-After`), when it sent one. + self.retry_after = retry_after + + def member(name: str) -> str | None: + value = problem.get(name) if problem else None + return value if isinstance(value, str) else None + + self.code = member("code") + self.detail = member("detail") + self.reason = member("reason") + title = member("title") or "Prepared-domains request failed" + explanation = self.detail or self.code + super().__init__(f"{status_code} {title}" + (f": {explanation}" if explanation else "")) + + @classmethod + def from_response(cls, response: requests.Response) -> "PreparedDomainError": + try: + body = response.json() + except ValueError: + body = None + try: + retry_after: float | None = float(response.headers.get("Retry-After", "")) + except ValueError: + retry_after = None + if retry_after is not None and (not math.isfinite(retry_after) or retry_after < 0): + retry_after = None + return cls( + response.status_code, + body if isinstance(body, dict) else None, + retry_after=retry_after, + ) + + # Adaptive Stealth Mode (wire `mode=auto`) picks js_render / premium_proxy # itself, so the API rejects it combined with either one set by hand. _ADAPTIVE_STEALTH_CONFLICTS = ("js_render", "premium_proxy") @@ -181,8 +250,10 @@ def extract( Extract works on any prepared domain (you can prepare one yourself with `POST /v1/extract/prepared-domains`), with any active API key. A 403 with `code: "REQS007"` means the domain isn't prepared for - Extract yet: prepare it with `POST /v1/extract/prepared-domains`, or - pass `fallback_to_autoparse=True` to retry with deprecated autoparse. + Extract yet: prepare it with `POST /v1/extract/prepared-domains` + (`prepare_domain()`, then check it with `prepared_domain_status()`), + or pass `fallback_to_autoparse=True` to retry with deprecated + autoparse. The legacy domain-not-enabled code is a 402 with `code: "AUTH010"`. By default both error responses are returned to you as is. @@ -238,6 +309,59 @@ def put( ) -> requests.Response: return self._worker("PUT", url, params, headers, data=data, **kwargs) + # ---- Extract prepared domains ---- + + def prepare_domain(self, domain: str, example_urls: list[str]) -> dict: + """Submit `domain` for Extract preparation + (`POST /v1/extract/prepared-domains`) and return the parsed status + object the API answers with `202`: `domain`, `status` (`preparing`, + `ready` or `failed`), `state`, `submitted_at`, `updated_at`. + + `domain` is the domain without a scheme. `example_urls` are 1 to 5 + URLs on that domain, one per kind of page you want to extract. + Submitting a domain that is already prepared returns its current + status instead of starting a new run. + + Preparation takes minutes. This method does not wait for it: call + `prepared_domain_status(domain)` until `status` is no longer + `"preparing"`. These endpoints don't fetch pages and cost no + credits. + + Raises `PreparedDomainError` on a non-2xx answer, for example 400 + `invalid_argument`, 409 `in_progress`, 429 `out_of_allowance` or + 503 `overloaded` / `disabled`. + """ + response = self.requests_session.request( + "POST", + self._prepared_domains_url(), + json={"domain": domain, "example_urls": list(example_urls)}, + headers=self._prepared_domains_headers(), + ) + return _prepared_domain_body(response) + + def prepared_domain_status(self, domain: str) -> dict: + """Read the preparation status of `domain` + (`GET /v1/extract/prepared-domains/{domain}`) and return the parsed + status object. `status` is `preparing`, `ready` or `failed`; a + failed preparation carries a `failure` object with `code`, `detail` + and `slot_consumed`. `state` and `roles` report progress. + + Raises `PreparedDomainError` on a non-2xx answer, for example 404 + `not_found` when your account never submitted `domain`. + """ + response = self.requests_session.request( + "GET", + f"{self._prepared_domains_url()}/{quote(domain, safe='')}", + headers=self._prepared_domains_headers(), + ) + return _prepared_domain_body(response) + + def _prepared_domains_url(self) -> str: + return self.api_url.rstrip("/") + "/extract/prepared-domains" + + def _prepared_domains_headers(self) -> dict: + return {"X-API-Key": self.apikey, "User-Agent": DEFAULT_USER_AGENT} + # ---- async-flavoured wrappers (thread-pool offload) ---- async def fetch_async( diff --git a/tests/test_prepared_domains.py b/tests/test_prepared_domains.py new file mode 100644 index 0000000..c09c028 --- /dev/null +++ b/tests/test_prepared_domains.py @@ -0,0 +1,201 @@ +"""prepare_domain() / prepared_domain_status() against a stubbed HTTP server: +the request each one sends, the parsed status it returns, and the +PreparedDomainError it raises on every documented error answer.""" + +import json +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer +from unittest import TestCase + +from zenrows import PreparedDomainError, ZenRowsClient + +apikey = "APIKEY" + +PREPARING = { + "domain": "www.example.com", + "status": "preparing", + "state": "received", + "submitted_at": "2026-10-15T10:00:00Z", + "updated_at": "2026-10-15T10:00:00Z", +} + + +def _problem(status: int, code: str, **extra: str) -> dict: + return { + "code": code, + "detail": f"detail for {code}", + "status": status, + "title": f"Title ({code})", + "type": f"https://docs.zenrows.com/api-error-codes#{code}", + **extra, + } + + +class _StubAPI: + """A local HTTP server that records each request and answers with the + next canned (status, body, headers) reply.""" + + def __init__(self) -> None: + self.requests: list[dict] = [] + self.replies: list[tuple[int, object, dict]] = [] + stub = self + + class Handler(BaseHTTPRequestHandler): + def _answer(self) -> None: + length = int(self.headers.get("Content-Length") or 0) + raw = self.rfile.read(length) if length else b"" + stub.requests.append( + { + "method": self.command, + "path": self.path, + "headers": dict(self.headers), + "body": json.loads(raw) if raw else None, + } + ) + status, body, headers = stub.replies.pop(0) + payload = body if isinstance(body, bytes) else json.dumps(body).encode() + self.send_response(status) + content_type = "application/json" if status < 400 else "application/problem+json" + self.send_header("Content-Type", content_type) + for name, value in headers.items(): + self.send_header(name, value) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + do_GET = _answer + do_POST = _answer + + def log_message(self, *args: object) -> None: + pass + + self.server = HTTPServer(("127.0.0.1", 0), Handler) + self.base_url = f"http://127.0.0.1:{self.server.server_address[1]}/v1/" + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + + def reply(self, status: int, body: object, headers: dict | None = None) -> None: + self.replies.append((status, body, headers or {})) + + def close(self) -> None: + self.server.shutdown() + self.server.server_close() + + +class TestPreparedDomains(TestCase): + def setUp(self): + self.api = _StubAPI() + self.addCleanup(self.api.close) + self.client = ZenRowsClient(apikey, base_url=self.api.base_url) + self.addCleanup(self.client.close) + + def test_prepare_domain_posts_domain_and_example_urls(self): + self.api.reply(202, PREPARING) + + status = self.client.prepare_domain( + "www.example.com", ["https://www.example.com/a", "https://www.example.com/b"] + ) + + self.assertEqual(status, PREPARING) + self.assertEqual(len(self.api.requests), 1) + sent = self.api.requests[0] + self.assertEqual(sent["method"], "POST") + self.assertEqual(sent["path"], "/v1/extract/prepared-domains") + self.assertEqual( + sent["body"], + { + "domain": "www.example.com", + "example_urls": ["https://www.example.com/a", "https://www.example.com/b"], + }, + ) + self.assertEqual(sent["headers"]["X-API-Key"], apikey) + self.assertNotIn(apikey, sent["path"]) + + def test_prepared_domain_status_gets_the_domain_and_returns_the_status(self): + ready = { + **PREPARING, + "status": "ready", + "state": "ready", + "roles": [{"role": "listing", "status": "ready", "url": "https://www.example.com/a"}], + } + self.api.reply(200, ready) + + status = self.client.prepared_domain_status("www.example.com") + + self.assertEqual(status, ready) + sent = self.api.requests[0] + self.assertEqual(sent["method"], "GET") + self.assertEqual(sent["path"], "/v1/extract/prepared-domains/www.example.com") + self.assertEqual(sent["headers"]["X-API-Key"], apikey) + + def test_status_returns_a_failed_preparation_without_raising(self): + failed = { + **PREPARING, + "status": "failed", + "state": "failed", + "failure": {"code": "unsupported", "detail": "x", "slot_consumed": True}, + } + self.api.reply(200, failed) + + status = self.client.prepared_domain_status("www.example.com") + + self.assertEqual(status["failure"]["code"], "unsupported") + + def test_status_escapes_the_domain_in_the_path(self): + self.api.reply(200, PREPARING) + + self.client.prepared_domain_status("bad/domain?x") + + self.assertEqual( + self.api.requests[0]["path"], "/v1/extract/prepared-domains/bad%2Fdomain%3Fx" + ) + + def test_prepare_domain_does_not_poll(self): + self.api.reply(202, PREPARING) + + self.client.prepare_domain("www.example.com", ["https://www.example.com/"]) + + self.assertEqual(len(self.api.requests), 1) + + def test_error_answers_raise_prepared_domain_error(self): + cases = [ + (400, "invalid_argument", {"reason": "invalid_domain"}), + (409, "in_progress", {}), + (429, "out_of_allowance", {}), + (503, "overloaded", {}), + (503, "disabled", {}), + ] + for status, code, extra in cases: + with self.subTest(code=code): + self.api.reply(status, _problem(status, code, **extra), {"Retry-After": "7"}) + + with self.assertRaises(PreparedDomainError) as caught: + self.client.prepare_domain("www.example.com", ["https://www.example.com/"]) + + err = caught.exception + self.assertEqual(err.status_code, status) + self.assertEqual(err.code, code) + self.assertEqual(err.detail, f"detail for {code}") + self.assertEqual(err.reason, extra.get("reason")) + self.assertEqual(err.retry_after, 7.0) + self.assertEqual(err.problem["status"], status) + self.assertIn(code, str(err)) + + def test_status_of_an_unknown_domain_raises_not_found(self): + self.api.reply(404, _problem(404, "not_found")) + + with self.assertRaises(PreparedDomainError) as caught: + self.client.prepared_domain_status("www.example.com") + + self.assertEqual(caught.exception.status_code, 404) + self.assertEqual(caught.exception.code, "not_found") + + def test_error_without_a_json_body(self): + self.api.reply(502, b"bad gateway") + + with self.assertRaises(PreparedDomainError) as caught: + self.client.prepared_domain_status("www.example.com") + + self.assertEqual(caught.exception.status_code, 502) + self.assertIsNone(caught.exception.code) + self.assertIsNone(caught.exception.problem) From 1147ec208989232f6e61c4bcc065b8c2f51ac8b6 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 13:49:26 +0000 Subject: [PATCH 3/4] fix(extract): reject empty domains and skip retries in prepared-domain calls prepare_domain() and prepared_domain_status() now raise ValueError for an empty or whitespace-only domain before sending anything. An empty domain used to request /v1/extract/prepared-domains/, which can resolve to the list endpoint and come back looking like a status. Both calls no longer go through the client's retry adapter, so a 429 out_of_allowance or 503 overloaded is returned on the first answer. GET used to be retried on 429/503 when retries > 0 while POST was not; the Go and Node SDKs skip client retries here too. Add prepare_domain_async() and prepared_domain_status_async(), run on the client's thread pool like the other _async wrappers. --- README.md | 2 + src/zenrows/client.py | 43 ++++++++++++-- tests/test_prepared_domains.py | 102 ++++++++++++++++++++++++++++++++- 3 files changed, 142 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 916a110..85e2307 100644 --- a/README.md +++ b/README.md @@ -243,6 +243,8 @@ while status["status"] == "preparing": print(status["status"]) # "ready" or "failed" (see status["failure"]) ``` +`prepare_domain_async()` and `prepared_domain_status_async()` are the async counterparts. The client's `retries` setting doesn't apply to these calls, and an empty `domain` raises `ValueError` before any request. + Both methods raise `PreparedDomainError` on an error answer, with `status_code`, `code` (`invalid_argument`, `not_found`, `in_progress`, `out_of_allowance`, `overloaded`, `disabled`, ...), `detail`, `reason` and the whole `problem` body. See the [Extract endpoints docs](https://docs.zenrows.com/extract/endpoints) for every status and failure code. A `422` with `code: "no_data_extracted"` means Extract knows this kind of page but found no data on it. It isn't charged, and the client never retries it, even with `retries` set. `is_no_data_extracted(response)` tells you when you got one. See the [Extract docs](https://docs.zenrows.com/extract/setup). diff --git a/src/zenrows/client.py b/src/zenrows/client.py index 3836101..cf91702 100644 --- a/src/zenrows/client.py +++ b/src/zenrows/client.py @@ -65,6 +65,21 @@ def is_no_data_extracted(response: requests.Response) -> bool: _EXTRACT_DOMAIN_UNAVAILABLE = {(403, "REQS007"), (402, "AUTH010")} +def _check_domain(domain: str) -> None: + # An empty domain would request the list endpoint and read its answer as + # a status, so it never leaves the client. + if not isinstance(domain, str) or not domain.strip(): + raise ValueError("domain must be a non-empty string") + + +def _prepared_domains_request(method: str, url: str, **kwargs: Any) -> requests.Response: + # Not through `requests_session`: its retry adapter would retry a 429 or + # 503, which these endpoints use for answers to read (out_of_allowance, + # overloaded). The Go and Node SDKs skip client retries here too. + with requests.Session() as session: + return session.request(method, url, **kwargs) + + def _prepared_domain_body(response: requests.Response) -> dict: if not response.ok: raise PreparedDomainError.from_response(response) @@ -327,11 +342,14 @@ def prepare_domain(self, domain: str, example_urls: list[str]) -> dict: `"preparing"`. These endpoints don't fetch pages and cost no credits. - Raises `PreparedDomainError` on a non-2xx answer, for example 400 + The client's `retries` setting doesn't apply to these endpoints. + Raises `ValueError` for an empty `domain`, before any request, and + `PreparedDomainError` on a non-2xx answer, for example 400 `invalid_argument`, 409 `in_progress`, 429 `out_of_allowance` or 503 `overloaded` / `disabled`. """ - response = self.requests_session.request( + _check_domain(domain) + response = _prepared_domains_request( "POST", self._prepared_domains_url(), json={"domain": domain, "example_urls": list(example_urls)}, @@ -346,10 +364,12 @@ def prepared_domain_status(self, domain: str) -> dict: failed preparation carries a `failure` object with `code`, `detail` and `slot_consumed`. `state` and `roles` report progress. - Raises `PreparedDomainError` on a non-2xx answer, for example 404 + Raises `ValueError` for an empty `domain`, before any request, and + `PreparedDomainError` on a non-2xx answer, for example 404 `not_found` when your account never submitted `domain`. """ - response = self.requests_session.request( + _check_domain(domain) + response = _prepared_domains_request( "GET", f"{self._prepared_domains_url()}/{quote(domain, safe='')}", headers=self._prepared_domains_headers(), @@ -409,6 +429,21 @@ async def extract_async( return response + async def prepare_domain_async(self, domain: str, example_urls: list[str]) -> dict: + """Async counterpart of `prepare_domain()`, run on the client's thread pool.""" + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + self.executor, partial(self.prepare_domain, domain, example_urls) + ) + + async def prepared_domain_status_async(self, domain: str) -> dict: + """Async counterpart of `prepared_domain_status()`, run on the client's + thread pool.""" + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + self.executor, partial(self.prepared_domain_status, domain) + ) + async def post_async( self, url: str, diff --git a/tests/test_prepared_domains.py b/tests/test_prepared_domains.py index c09c028..bf3f1fb 100644 --- a/tests/test_prepared_domains.py +++ b/tests/test_prepared_domains.py @@ -5,7 +5,7 @@ import json import threading from http.server import BaseHTTPRequestHandler, HTTPServer -from unittest import TestCase +from unittest import IsolatedAsyncioTestCase, TestCase from zenrows import PreparedDomainError, ZenRowsClient @@ -199,3 +199,103 @@ def test_error_without_a_json_body(self): self.assertEqual(caught.exception.status_code, 502) self.assertIsNone(caught.exception.code) self.assertIsNone(caught.exception.problem) + + def test_empty_domain_is_rejected_before_any_request(self): + for domain in ("", " ", "\t\n"): + with self.subTest(domain=domain): + with self.assertRaises(ValueError): + self.client.prepared_domain_status(domain) + with self.assertRaises(ValueError): + self.client.prepare_domain(domain, ["https://www.example.com/"]) + + self.assertEqual(self.api.requests, []) + + +class TestPreparedDomainsIgnoreRetries(TestCase): + """The client's `retries` apply to scraping only: a 429 or 503 from the + prepared-domains endpoints is returned on the first answer.""" + + def setUp(self): + self.api = _StubAPI() + self.addCleanup(self.api.close) + self.client = ZenRowsClient(apikey, retries=1, base_url=self.api.base_url) + self.addCleanup(self.client.close) + + def test_status_503_is_not_retried(self): + self.api.reply(503, _problem(503, "overloaded")) + self.api.reply(200, PREPARING) + + with self.assertRaises(PreparedDomainError) as caught: + self.client.prepared_domain_status("www.example.com") + + self.assertEqual(caught.exception.code, "overloaded") + self.assertEqual(len(self.api.requests), 1) + + def test_prepare_429_is_not_retried(self): + self.api.reply(429, _problem(429, "out_of_allowance")) + self.api.reply(202, PREPARING) + + with self.assertRaises(PreparedDomainError) as caught: + self.client.prepare_domain("www.example.com", ["https://www.example.com/"]) + + self.assertEqual(caught.exception.code, "out_of_allowance") + self.assertEqual(len(self.api.requests), 1) + + def test_scraping_keeps_its_retries(self): + """Control: the same client still retries a 503 on a scraping request.""" + self.api.reply(503, b"busy") + self.api.reply(200, b"ok") + + response = self.client.fetch("https://www.example.com/") + + self.assertEqual(response.status_code, 200) + self.assertEqual(len(self.api.requests), 2) + + +class TestPreparedDomainsAsync(IsolatedAsyncioTestCase): + def setUp(self): + self.api = _StubAPI() + self.addCleanup(self.api.close) + self.client = ZenRowsClient(apikey, concurrency=2, base_url=self.api.base_url) + self.addCleanup(self.client.close) + + async def test_prepare_domain_async_returns_the_status(self): + self.api.reply(202, PREPARING) + + status = await self.client.prepare_domain_async( + "www.example.com", ["https://www.example.com/a"] + ) + + self.assertEqual(status, PREPARING) + sent = self.api.requests[0] + self.assertEqual(sent["method"], "POST") + self.assertEqual( + sent["body"], + {"domain": "www.example.com", "example_urls": ["https://www.example.com/a"]}, + ) + + async def test_prepared_domain_status_async_returns_the_status(self): + self.api.reply(200, {**PREPARING, "status": "ready"}) + + status = await self.client.prepared_domain_status_async("www.example.com") + + self.assertEqual(status["status"], "ready") + self.assertEqual( + self.api.requests[0]["path"], "/v1/extract/prepared-domains/www.example.com" + ) + + async def test_async_errors_raise_prepared_domain_error(self): + self.api.reply(404, _problem(404, "not_found")) + + with self.assertRaises(PreparedDomainError) as caught: + await self.client.prepared_domain_status_async("www.example.com") + + self.assertEqual(caught.exception.code, "not_found") + + async def test_async_rejects_an_empty_domain(self): + with self.assertRaises(ValueError): + await self.client.prepared_domain_status_async(" ") + with self.assertRaises(ValueError): + await self.client.prepare_domain_async("", ["https://www.example.com/"]) + + self.assertEqual(self.api.requests, []) From 533f742cde45fc5b735ab6aa2534d65d285dab69 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Fri, 16 Oct 2026 07:41:35 +0000 Subject: [PATCH 4/4] feat(extract): remove the autoparse fallback and document the REQS007 reason extract() and extract_async() no longer retry with autoparse on any answer. REQS007 and AUTH010 come back as is, in one request. fallback_to_autoparse stays in both signatures so existing calls keep working, but it has no effect: passing True emits a DeprecationWarning that points at prepare_domain(). The docstrings and the README describe the reason field of a 403 REQS007 problem body and its three cases: domain_not_prepared (prepare the domain, then poll its status), page_not_recognised (try a product, article or listing page) and domain_not_enabled (use css_extractor or contact support). Older responses may not carry a reason. --- README.md | 22 +++- src/zenrows/client.py | 78 +++++------- tests/test_fetch_extract.py | 233 ++++++++---------------------------- 3 files changed, 103 insertions(+), 230 deletions(-) diff --git a/README.md b/README.md index 85e2307..ff959b2 100644 --- a/README.md +++ b/README.md @@ -205,14 +205,28 @@ print(response.json()) `extract()` also sends [Adaptive Stealth Mode](https://docs.zenrows.com/fetch/features/adaptive-stealth-mode) (`mode: "auto"`) by default, except when `params` sets `js_render` or `premium_proxy`: the API rejects either combined with Adaptive Stealth Mode, so your fixed configuration is sent as is. Pass `adaptive_stealth=False` to turn it off entirely. -Extract works on any prepared domain (you can prepare one yourself with `POST /v1/extract/prepared-domains`). A `403` with `code: "REQS007"` means the domain isn't prepared for Extract yet: prepare it with `POST /v1/extract/prepared-domains` (`client.prepare_domain()`, see below), or pass `fallback_to_autoparse=True` to retry with deprecated autoparse. The legacy `402` with `code: "AUTH010"` is handled the same way. By default `extract()` returns both error responses to you as is, without a second request. +Extract works on any prepared domain (you can prepare one yourself with `POST /v1/extract/prepared-domains`). A `403` with `code: "REQS007"` means Extract can't extract this page. Check `reason` in the problem body when present; older responses may not carry it: + +- `domain_not_prepared`: prepare the domain with `client.prepare_domain()` (see below), poll `client.prepared_domain_status()` until `status` is `"ready"`, then retry. +- `page_not_recognised`: the domain is served, but this page isn't one Extract extracts (a home, search, login or category page). Try a product, article or listing page, or retry once in a few minutes. Preparing the domain again changes nothing. +- `domain_not_enabled`: preparing doesn't help. Use `css_extractor`, or contact support. ```python -# Opt in to one retry with the deprecated autoparse on REQS007 / AUTH010. -# The retry is a second, separately charged request. -response = client.extract(url, fallback_to_autoparse=True) +response = client.extract(url) +if response.status_code == 403: + problem = response.json() + if problem.get("code") == "REQS007": + reason = problem.get("reason") # None on older responses + if reason == "domain_not_prepared": + ... # client.prepare_domain(...), then poll client.prepared_domain_status(...) + elif reason == "page_not_recognised": + ... # try a product, article or listing page + elif reason == "domain_not_enabled": + ... # use css_extractor, or contact support ``` +The legacy `402` with `code: "AUTH010"` means the domain isn't enabled. `extract()` returns both error responses to you as is, in one request, and never retries them with autoparse. `fallback_to_autoparse` is deprecated and has no effect; passing `True` emits a `DeprecationWarning`. + #### Preparing a domain `client.prepare_domain(domain, example_urls)` submits a domain for Extract (`POST /v1/extract/prepared-domains`), and `client.prepared_domain_status(domain)` reads where it is (`GET /v1/extract/prepared-domains/{domain}`). Both return the parsed status object. Pass the domain without a scheme and 1 to 5 example URLs, one per kind of page you want to extract. These calls cost no credits. diff --git a/src/zenrows/client.py b/src/zenrows/client.py index cf91702..bae8a5e 100644 --- a/src/zenrows/client.py +++ b/src/zenrows/client.py @@ -15,6 +15,7 @@ import math import os import time +import warnings from concurrent.futures import ThreadPoolExecutor from functools import partial from typing import Any @@ -58,11 +59,11 @@ def is_no_data_extracted(response: requests.Response) -> bool: return response.status_code == 422 and code is not None and code.lower() == _NO_DATA_EXTRACTED -# Extract answers these when it can't serve the target domain: 403 REQS007 -# (domain not prepared yet) or the legacy 402 AUTH010 (domain not enabled). -# extract(mode="auto", fallback_to_autoparse=True) retries both once with the -# deprecated Autoparse; by default they are returned to the caller as is. -_EXTRACT_DOMAIN_UNAVAILABLE = {(403, "REQS007"), (402, "AUTH010")} +_FALLBACK_DEPRECATED = ( + "fallback_to_autoparse is deprecated and has no effect: extract() no longer " + "retries with autoparse. On a 403 REQS007, check `reason` in the problem " + "body; for domain_not_prepared, prepare the domain with prepare_domain()." +) def _check_domain(domain: str) -> None: @@ -92,15 +93,6 @@ def _prepared_domain_body(response: requests.Response) -> dict: return body -def _is_extract_domain_unavailable(response: requests.Response) -> bool: - """True when the response is Extract saying it can't serve the target - domain yet (403 REQS007, or the legacy 402 AUTH010).""" - code = _error_code(response) - if code is None: - return False - return (response.status_code, code.upper()) in _EXTRACT_DOMAIN_UNAVAILABLE - - class PreparedDomainError(Exception): """A non-2xx answer from the prepared-domains endpoints (`prepare_domain()` / `prepared_domain_status()`). @@ -264,18 +256,26 @@ def extract( Extract works on any prepared domain (you can prepare one yourself with `POST /v1/extract/prepared-domains`), with any active API key. - A 403 with `code: "REQS007"` means the domain isn't prepared for - Extract yet: prepare it with `POST /v1/extract/prepared-domains` - (`prepare_domain()`, then check it with `prepared_domain_status()`), - or pass `fallback_to_autoparse=True` to retry with deprecated - autoparse. + A 403 with `code: "REQS007"` means Extract can't extract this page. + Check `reason` in the problem body (`response.json().get("reason")`) + when present; older responses may not carry it: + + - `domain_not_prepared`: prepare the domain with `prepare_domain()`, + poll `prepared_domain_status()` until `status` is `"ready"`, then + retry. + - `page_not_recognised`: the domain is served, but this page isn't + one Extract extracts (a home, search, login or category page). + Try a product, article or listing page, or retry once in a few + minutes. Preparing the domain again changes nothing. + - `domain_not_enabled`: preparing doesn't help. Use + `css_extractor`, or contact support. + The legacy domain-not-enabled code is a 402 with `code: "AUTH010"`. - By default both error responses are returned to you as is. + Both error responses are returned to you as is, in one request. - `fallback_to_autoparse=True` (opt-in, default False) makes - `mode="auto"` retry once with `autoparse=True` on either code instead - of returning the error response. Autoparse is deprecated, and the - retry is a second, separately charged request. + `fallback_to_autoparse` is deprecated and has no effect: extract() + never retries with autoparse. Passing True emits a + `DeprecationWarning`. A 422 with `code: "no_data_extracted"` means Extract knows this kind of page but found no data on it. It is not charged and is never @@ -283,26 +283,18 @@ def extract( `adaptive_stealth=True` (default) also sends Adaptive Stealth Mode (`mode: "auto"` at the wire-param level - unrelated to this method's own - `mode` argument) on the extract attempt and on the Autoparse fallback - when `fallback_to_autoparse=True`, - so a target needing `js_render`/`premium_proxy` gets escalated + `mode` argument), so a target needing `js_render`/`premium_proxy` gets escalated automatically instead of failing with REQS002. It is skipped when `params` sets `js_render` or `premium_proxy` (the API rejects either combined with Adaptive Stealth Mode) or its own wire `mode`. Pass `adaptive_stealth=False` to turn it off entirely. """ + if fallback_to_autoparse: + warnings.warn(_FALLBACK_DEPRECATED, DeprecationWarning, stacklevel=2) final_params = dict(params) if params else {} final_params["extract"] = mode _with_adaptive_stealth(final_params, adaptive_stealth) - response = self.fetch(url, final_params, headers, **kwargs) - - if mode == "auto" and fallback_to_autoparse and _is_extract_domain_unavailable(response): - autoparse_params = dict(params) if params else {} - autoparse_params["autoparse"] = True - _with_adaptive_stealth(autoparse_params, adaptive_stealth) - return self.fetch(url, autoparse_params, headers, **kwargs) - - return response + return self.fetch(url, final_params, headers, **kwargs) def post( self, @@ -414,20 +406,14 @@ async def extract_async( **kwargs: Any, ) -> requests.Response: """Async counterpart of `extract()` - see its docstring for the - REQS007 / AUTH010 handling, the opt-in `fallback_to_autoparse`, and + REQS007 `reason` cases, the deprecated `fallback_to_autoparse`, and `adaptive_stealth`.""" + if fallback_to_autoparse: + warnings.warn(_FALLBACK_DEPRECATED, DeprecationWarning, stacklevel=2) final_params = dict(params) if params else {} final_params["extract"] = mode _with_adaptive_stealth(final_params, adaptive_stealth) - response = await self.fetch_async(url, final_params, headers, **kwargs) - - if mode == "auto" and fallback_to_autoparse and _is_extract_domain_unavailable(response): - autoparse_params = dict(params) if params else {} - autoparse_params["autoparse"] = True - _with_adaptive_stealth(autoparse_params, adaptive_stealth) - return await self.fetch_async(url, autoparse_params, headers, **kwargs) - - return response + return await self.fetch_async(url, final_params, headers, **kwargs) async def prepare_domain_async(self, domain: str, example_urls: list[str]) -> dict: """Async counterpart of `prepare_domain()`, run on the client's thread pool.""" diff --git a/tests/test_fetch_extract.py b/tests/test_fetch_extract.py index fdf0248..e1ef11d 100644 --- a/tests/test_fetch_extract.py +++ b/tests/test_fetch_extract.py @@ -7,6 +7,7 @@ `fetch()`/`extract()`/`get()` actually *return* and *raise* (or don't). """ +import warnings from unittest import IsolatedAsyncioTestCase, TestCase, mock from requests import Response, Session @@ -123,136 +124,62 @@ def test_extract_does_not_mutate_caller_supplied_params_dict(self, mock_request) self.assertEqual(caller_params, {"js_render": True}) -class TestExtractAutoparseFallback(TestCase): - """`extract(mode="auto")`: 403 REQS007 means the target domain isn't - prepared for Extract yet, and the legacy 402 AUTH010 that it isn't - enabled. By default both are returned as is, without a second request. - With `fallback_to_autoparse=True` the client retries exactly once with - the deprecated Autoparse.""" +_DOMAIN_UNAVAILABLE = ( + (403, b'{"code": "REQS007", "reason": "domain_not_prepared"}'), + (402, b'{"code": "AUTH010", "title": "Domain not enabled"}'), +) + + +class TestExtractNeverFallsBackToAutoparse(TestCase): + """`extract()` returns 403 REQS007 and the legacy 402 AUTH010 as is, in + one request. The deprecated `fallback_to_autoparse` has no effect beyond + a DeprecationWarning.""" def setUp(self): self.client = ZenRowsClient(apikey) @mock.patch.object(Session, "request") - def test_reqs007_is_returned_without_fallback_by_default(self, mock_request): - mock_request.return_value = _fake_response( - 403, b'{"code": "REQS007", "title": "Domain not prepared"}' - ) + def test_returned_as_is_in_one_request(self, mock_request): + for status, body in _DOMAIN_UNAVAILABLE: + with self.subTest(status=status): + mock_request.reset_mock() + mock_request.side_effect = [_fake_response(status, body), _fake_response(200)] - response = self.client.extract(url) + with warnings.catch_warnings(): + warnings.simplefilter("error") + response = self.client.extract(url) - self.assertEqual(response.status_code, 403) - self.assertEqual(mock_request.call_count, 1) - _, kwargs = mock_request.call_args - self.assertNotIn("autoparse", kwargs["params"]) + self.assertEqual(response.status_code, status) + self.assertEqual(mock_request.call_count, 1) + _, kwargs = mock_request.call_args + self.assertNotIn("autoparse", kwargs["params"]) + + @mock.patch.object(Session, "request") + def test_deprecated_option_has_no_effect_and_warns(self, mock_request): + for status, body in _DOMAIN_UNAVAILABLE: + with self.subTest(status=status): + mock_request.reset_mock() + mock_request.side_effect = [_fake_response(status, body), _fake_response(200)] + + with self.assertWarns(DeprecationWarning) as caught: + response = self.client.extract(url, fallback_to_autoparse=True) + + self.assertEqual(response.status_code, status) + self.assertEqual(mock_request.call_count, 1) + _, kwargs = mock_request.call_args + self.assertNotIn("autoparse", kwargs["params"]) + self.assertIn("prepare_domain()", str(caught.warning)) + self.assertEqual(caught.filename, __file__) @mock.patch.object(Session, "request") - def test_auth010_is_returned_without_fallback_by_default(self, mock_request): + def test_reason_is_readable_from_the_response(self, mock_request): mock_request.return_value = _fake_response( - 402, b'{"code": "AUTH010", "title": "Domain not enabled"}' + 403, b'{"code": "REQS007", "reason": "page_not_recognised"}' ) response = self.client.extract(url) - self.assertEqual(response.status_code, 402) - self.assertEqual(mock_request.call_count, 1) - _, kwargs = mock_request.call_args - self.assertNotIn("autoparse", kwargs["params"]) - - @mock.patch.object(Session, "request") - def test_opt_in_falls_back_to_autoparse_on_reqs007(self, mock_request): - mock_request.side_effect = [ - _fake_response(403, b'{"code": "REQS007", "title": "Domain not prepared"}'), - _fake_response(200, b'[{"found": "via autoparse"}]'), - ] - - response = self.client.extract(url, fallback_to_autoparse=True) - - self.assertEqual(response.status_code, 200) - self.assertEqual(mock_request.call_count, 2) - _, fallback_kwargs = mock_request.call_args - self.assertTrue(fallback_kwargs["params"].get("autoparse")) - self.assertNotIn("extract", fallback_kwargs["params"]) - - @mock.patch.object(Session, "request") - def test_opt_in_falls_back_to_autoparse_on_auth010(self, mock_request): - mock_request.side_effect = [ - _fake_response(402, b'{"code": "AUTH010", "title": "Domain not enabled"}'), - _fake_response(200, b'[{"found": "via autoparse"}]'), - ] - - response = self.client.extract(url, fallback_to_autoparse=True) - - self.assertEqual(response.status_code, 200) - self.assertEqual(mock_request.call_count, 2) - _, fallback_kwargs = mock_request.call_args - self.assertTrue(fallback_kwargs["params"].get("autoparse")) - self.assertNotIn("extract", fallback_kwargs["params"]) - - @mock.patch.object(Session, "request") - def test_opt_in_retries_autoparse_only_once(self, mock_request): - """A fallback that fails again is returned, not retried a second time.""" - mock_request.side_effect = [ - _fake_response(403, b'{"code": "REQS007"}'), - _fake_response(403, b'{"code": "REQS007"}'), - _fake_response(200, b"{}"), - ] - - response = self.client.extract(url, fallback_to_autoparse=True) - - self.assertEqual(response.status_code, 403) - self.assertEqual(mock_request.call_count, 2) - - @mock.patch.object(Session, "request") - def test_other_4xx_never_fall_back(self, mock_request): - """Neither an auth 403, a credits-exhausted 402, nor REQS007 / AUTH010 - on the wrong status is mistaken for a domain that isn't prepared, - even with the fallback on.""" - cases = [ - (403, b'{"code": "AUTH002", "title": "Invalid API key"}'), - (402, b'{"code": "AUTH004", "title": "No credit available"}'), - (400, b'{"code": "REQS007"}'), - (400, b'{"code": "AUTH010"}'), - (404, b"{}"), - ] - for status, body in cases: - for fallback in (False, True): - with self.subTest(status=status, body=body, fallback=fallback): - mock_request.reset_mock() - mock_request.return_value = _fake_response(status, body) - - response = self.client.extract(url, fallback_to_autoparse=fallback) - - self.assertEqual(response.status_code, status) - self.assertEqual(mock_request.call_count, 1) - - @mock.patch.object(Session, "request") - def test_no_fallback_for_non_auto_mode(self, mock_request): - """Neither code triggers a fallback for native/standard modes - - only "auto" takes the fallback path, even when it is turned on.""" - mock_request.side_effect = [ - _fake_response(402, b'{"code": "AUTH010"}'), - _fake_response(403, b'{"code": "REQS007"}'), - ] - - first = self.client.extract(url, mode="native", fallback_to_autoparse=True) - second = self.client.extract(url, mode="standard", fallback_to_autoparse=True) - - self.assertEqual(first.status_code, 402) - self.assertEqual(second.status_code, 403) - self.assertEqual(mock_request.call_count, 2) - - @mock.patch.object(Session, "request") - def test_fallback_does_not_mutate_caller_supplied_params_dict(self, mock_request): - mock_request.side_effect = [ - _fake_response(402, b'{"code": "AUTH010"}'), - _fake_response(200, b"{}"), - ] - caller_params = {"js_render": True} - - self.client.extract(url, params=caller_params, fallback_to_autoparse=True) - - self.assertEqual(caller_params, {"js_render": True}) + self.assertEqual(response.json().get("reason"), "page_not_recognised") class TestExtractAdaptiveStealth(TestCase): @@ -281,18 +208,6 @@ def test_omits_wire_mode_when_disabled(self, mock_request): _, kwargs = mock_request.call_args self.assertNotIn("mode", kwargs["params"]) - @mock.patch.object(Session, "request") - def test_fallback_request_also_carries_adaptive_stealth(self, mock_request): - mock_request.side_effect = [ - _fake_response(402, b'{"code": "AUTH010"}'), - _fake_response(200, b"{}"), - ] - - self.client.extract(url, fallback_to_autoparse=True) - - _, fallback_kwargs = mock_request.call_args - self.assertEqual(fallback_kwargs["params"]["mode"], "auto") - @mock.patch.object(Session, "request") def test_skips_adaptive_stealth_when_js_render_is_set(self, mock_request): """The API rejects mode=auto combined with js_render, so a caller's @@ -332,31 +247,16 @@ def test_keeps_a_wire_mode_the_caller_set(self, mock_request): _, kwargs = mock_request.call_args self.assertEqual(kwargs["params"]["mode"], "custom") - @mock.patch.object(Session, "request") - def test_fallback_request_skips_adaptive_stealth_with_premium_proxy(self, mock_request): - mock_request.side_effect = [ - _fake_response(403, b'{"code": "REQS007"}'), - _fake_response(200, b"{}"), - ] - - self.client.extract(url, params={"premium_proxy": True}, fallback_to_autoparse=True) - - first_kwargs = mock_request.call_args_list[0].kwargs - _, fallback_kwargs = mock_request.call_args - self.assertNotIn("mode", first_kwargs["params"]) - self.assertNotIn("mode", fallback_kwargs["params"]) - self.assertTrue(fallback_kwargs["params"]["autoparse"]) - -class TestExtractAsyncAutoparseFallback(IsolatedAsyncioTestCase): - """Async counterpart - same REQS007 / AUTH010 handling: returned as is by - default, one Autoparse retry when `fallback_to_autoparse=True`.""" +class TestExtractAsyncNeverFallsBackToAutoparse(IsolatedAsyncioTestCase): + """Async counterpart: one request, error returned as is, and a + DeprecationWarning for `fallback_to_autoparse=True`.""" def setUp(self): self.client = ZenRowsClient(apikey, concurrency=2) @mock.patch.object(Session, "request") - async def test_reqs007_and_auth010_are_returned_without_fallback_by_default(self, mock_request): + async def test_returned_as_is_in_one_request(self, mock_request): mock_request.side_effect = [ _fake_response(403, b'{"code": "REQS007"}'), _fake_response(402, b'{"code": "AUTH010"}'), @@ -370,44 +270,17 @@ async def test_reqs007_and_auth010_are_returned_without_fallback_by_default(self self.assertEqual(mock_request.call_count, 2) @mock.patch.object(Session, "request") - async def test_opt_in_falls_back_to_autoparse_on_auth010(self, mock_request): - mock_request.side_effect = [ - _fake_response(402, b'{"code": "AUTH010"}'), - _fake_response(200, b'[{"found": "via autoparse"}]'), - ] - - response = await self.client.extract_async(url, fallback_to_autoparse=True) - - self.assertEqual(response.status_code, 200) - self.assertEqual(mock_request.call_count, 2) - - @mock.patch.object(Session, "request") - async def test_skips_adaptive_stealth_when_js_render_is_set(self, mock_request): + async def test_deprecated_option_has_no_effect_and_warns(self, mock_request): mock_request.side_effect = [ _fake_response(403, b'{"code": "REQS007"}'), - _fake_response(200, b"{}"), + _fake_response(200), ] - await self.client.extract_async(url, params={"js_render": True}, fallback_to_autoparse=True) - - first_kwargs = mock_request.call_args_list[0].kwargs - _, fallback_kwargs = mock_request.call_args - self.assertNotIn("mode", first_kwargs["params"]) - self.assertNotIn("mode", fallback_kwargs["params"]) + with self.assertWarns(DeprecationWarning): + response = await self.client.extract_async(url, fallback_to_autoparse=True) - @mock.patch.object(Session, "request") - async def test_opt_in_falls_back_to_autoparse_on_reqs007(self, mock_request): - mock_request.side_effect = [ - _fake_response(403, b'{"code": "REQS007"}'), - _fake_response(200, b'[{"found": "via autoparse"}]'), - ] - - response = await self.client.extract_async(url, fallback_to_autoparse=True) - - self.assertEqual(response.status_code, 200) - self.assertEqual(mock_request.call_count, 2) - _, fallback_kwargs = mock_request.call_args - self.assertTrue(fallback_kwargs["params"].get("autoparse")) + self.assertEqual(response.status_code, 403) + self.assertEqual(mock_request.call_count, 1) class TestFetchExtractAsync(IsolatedAsyncioTestCase):