diff --git a/README.md b/README.md index 6fd860b..ff959b2 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ This package ships three clients: - [PUT Requests](#put-requests) - [Concurrency](#concurrency) - [Extract](#extract) + - [Preparing a domain](#preparing-a-domain) - [Quickstart — Batch API (`ZenRowsBatchClient`)](#quickstart--batch-api-zenrowsbatchclient) - [Extract in a batch](#extract-in-a-batch) - [Upload URLs from a CSV](#upload-urls-from-a-csv) @@ -68,7 +69,7 @@ client = ZenRowsClient("YOUR-API-KEY", retries=1) url = "https://www.zenrows.com/" response = client.fetch(url, params={ - # Our algorithm allows to automatically extract content from any website + # Deprecated: use client.extract() for structured data instead "autoparse": False, # CSS Selectors for data extraction (i.e. {"links":"a @href"} to get href attributes from links) @@ -204,6 +205,62 @@ print(response.json()) `extract()` also sends [Adaptive Stealth Mode](https://docs.zenrows.com/fetch/features/adaptive-stealth-mode) (`mode: "auto"`) by default, except when `params` sets `js_render` or `premium_proxy`: the API rejects either combined with Adaptive Stealth Mode, so your fixed configuration is sent as is. Pass `adaptive_stealth=False` to turn it off entirely. +Extract works on any prepared domain (you can prepare one yourself with `POST /v1/extract/prepared-domains`). A `403` with `code: "REQS007"` means Extract can't extract this page. Check `reason` in the problem body when present; older responses may not carry it: + +- `domain_not_prepared`: prepare the domain with `client.prepare_domain()` (see below), poll `client.prepared_domain_status()` until `status` is `"ready"`, then retry. +- `page_not_recognised`: the domain is served, but this page isn't one Extract extracts (a home, search, login or category page). Try a product, article or listing page, or retry once in a few minutes. Preparing the domain again changes nothing. +- `domain_not_enabled`: preparing doesn't help. Use `css_extractor`, or contact support. + +```python +response = client.extract(url) +if response.status_code == 403: + problem = response.json() + if problem.get("code") == "REQS007": + reason = problem.get("reason") # None on older responses + if reason == "domain_not_prepared": + ... # client.prepare_domain(...), then poll client.prepared_domain_status(...) + elif reason == "page_not_recognised": + ... # try a product, article or listing page + elif reason == "domain_not_enabled": + ... # use css_extractor, or contact support +``` + +The legacy `402` with `code: "AUTH010"` means the domain isn't enabled. `extract()` returns both error responses to you as is, in one request, and never retries them with autoparse. `fallback_to_autoparse` is deprecated and has no effect; passing `True` emits a `DeprecationWarning`. + +#### Preparing a domain + +`client.prepare_domain(domain, example_urls)` submits a domain for Extract (`POST /v1/extract/prepared-domains`), and `client.prepared_domain_status(domain)` reads where it is (`GET /v1/extract/prepared-domains/{domain}`). Both return the parsed status object. Pass the domain without a scheme and 1 to 5 example URLs, one per kind of page you want to extract. These calls cost no credits. + +Preparation takes minutes, and the client does not wait for it. Poll the status until it is no longer `"preparing"`: + +```python +import time + +from zenrows import PreparedDomainError, ZenRowsClient + +client = ZenRowsClient("YOUR-API-KEY") + +try: + status = client.prepare_domain( + "www.scrapingcourse.com", + ["https://www.scrapingcourse.com/ecommerce/"], + ) +except PreparedDomainError as err: + # e.g. 409 in_progress, 429 out_of_allowance, 503 overloaded + print(err.status_code, err.code, err.detail) + raise + +while status["status"] == "preparing": + time.sleep(30) + status = client.prepared_domain_status("www.scrapingcourse.com") + +print(status["status"]) # "ready" or "failed" (see status["failure"]) +``` + +`prepare_domain_async()` and `prepared_domain_status_async()` are the async counterparts. The client's `retries` setting doesn't apply to these calls, and an empty `domain` raises `ValueError` before any request. + +Both methods raise `PreparedDomainError` on an error answer, with `status_code`, `code` (`invalid_argument`, `not_found`, `in_progress`, `out_of_allowance`, `overloaded`, `disabled`, ...), `detail`, `reason` and the whole `problem` body. See the [Extract endpoints docs](https://docs.zenrows.com/extract/endpoints) for every status and failure code. + A `422` with `code: "no_data_extracted"` means Extract knows this kind of page but found no data on it. It isn't charged, and the client never retries it, even with `retries` set. `is_no_data_extracted(response)` tells you when you got one. See the [Extract docs](https://docs.zenrows.com/extract/setup). ## Quickstart — Batch API (`ZenRowsBatchClient`) diff --git a/src/zenrows/__init__.py b/src/zenrows/__init__.py index 8a13b06..75626f1 100644 --- a/src/zenrows/__init__.py +++ b/src/zenrows/__init__.py @@ -15,11 +15,12 @@ from zenrows.__version__ import __version__ from zenrows.batch import ZenRowsBatchClient -from zenrows.client import ZenRowsClient, is_no_data_extracted +from zenrows.client import PreparedDomainError, ZenRowsClient, is_no_data_extracted from zenrows.crawl import CrawlAPIError, ZenRowsCrawlClient __all__ = [ "CrawlAPIError", + "PreparedDomainError", "ZenRowsBatchClient", "ZenRowsClient", "ZenRowsCrawlClient", diff --git a/src/zenrows/client.py b/src/zenrows/client.py index 2a38502..bae8a5e 100644 --- a/src/zenrows/client.py +++ b/src/zenrows/client.py @@ -12,11 +12,14 @@ """ import asyncio +import math import os import time +import warnings from concurrent.futures import ThreadPoolExecutor from functools import partial from typing import Any +from urllib.parse import quote import requests import urllib3 @@ -56,19 +59,93 @@ def is_no_data_extracted(response: requests.Response) -> bool: return response.status_code == 422 and code is not None and code.lower() == _NO_DATA_EXTRACTED -# Extract answers these when it can't serve the target domain: 403 REQS007 -# (domain not prepared yet) or the legacy 402 AUTH010 (domain not enabled). -# extract(mode="auto") retries both once with Autoparse. -_EXTRACT_DOMAIN_UNAVAILABLE = {(403, "REQS007"), (402, "AUTH010")} +_FALLBACK_DEPRECATED = ( + "fallback_to_autoparse is deprecated and has no effect: extract() no longer " + "retries with autoparse. On a 403 REQS007, check `reason` in the problem " + "body; for domain_not_prepared, prepare the domain with prepare_domain()." +) -def _is_extract_domain_unavailable(response: requests.Response) -> bool: - """True when the response is Extract saying it can't serve the target - domain yet (403 REQS007, or the legacy 402 AUTH010).""" - code = _error_code(response) - if code is None: - return False - return (response.status_code, code.upper()) in _EXTRACT_DOMAIN_UNAVAILABLE +def _check_domain(domain: str) -> None: + # An empty domain would request the list endpoint and read its answer as + # a status, so it never leaves the client. + if not isinstance(domain, str) or not domain.strip(): + raise ValueError("domain must be a non-empty string") + + +def _prepared_domains_request(method: str, url: str, **kwargs: Any) -> requests.Response: + # Not through `requests_session`: its retry adapter would retry a 429 or + # 503, which these endpoints use for answers to read (out_of_allowance, + # overloaded). The Go and Node SDKs skip client retries here too. + with requests.Session() as session: + return session.request(method, url, **kwargs) + + +def _prepared_domain_body(response: requests.Response) -> dict: + if not response.ok: + raise PreparedDomainError.from_response(response) + try: + body = response.json() + except ValueError: + body = None + if not isinstance(body, dict): + raise PreparedDomainError(response.status_code, None) + return body + + +class PreparedDomainError(Exception): + """A non-2xx answer from the prepared-domains endpoints + (`prepare_domain()` / `prepared_domain_status()`). + + Branch on `status_code` + `code`; display `detail`, don't parse it. + `code` is the problem's `code` member, for example `invalid_argument` + (400), `not_found` (404), `in_progress` (409: a preparation is already + running, read its status instead of resubmitting), `out_of_allowance` + (429: no preparations left this month), or `overloaded` / `disabled` + (503); None when the body has no code. `problem` holds the whole body, + or None when it was not JSON. + """ + + def __init__( + self, + status_code: int, + problem: dict | None, + *, + retry_after: float | None = None, + ): + self.status_code = status_code + self.problem = problem + # Seconds the server asked to wait (`Retry-After`), when it sent one. + self.retry_after = retry_after + + def member(name: str) -> str | None: + value = problem.get(name) if problem else None + return value if isinstance(value, str) else None + + self.code = member("code") + self.detail = member("detail") + self.reason = member("reason") + title = member("title") or "Prepared-domains request failed" + explanation = self.detail or self.code + super().__init__(f"{status_code} {title}" + (f": {explanation}" if explanation else "")) + + @classmethod + def from_response(cls, response: requests.Response) -> "PreparedDomainError": + try: + body = response.json() + except ValueError: + body = None + try: + retry_after: float | None = float(response.headers.get("Retry-After", "")) + except ValueError: + retry_after = None + if retry_after is not None and (not math.isfinite(retry_after) or retry_after < 0): + retry_after = None + return cls( + response.status_code, + body if isinstance(body, dict) else None, + retry_after=retry_after, + ) # Adaptive Stealth Mode (wire `mode=auto`) picks js_render / premium_proxy @@ -168,7 +245,7 @@ def extract( params: dict | None = None, headers: dict | None = None, mode: str = "auto", - fallback_to_autoparse: bool = True, + fallback_to_autoparse: bool = False, adaptive_stealth: bool = True, **kwargs: Any, ) -> requests.Response: @@ -177,13 +254,28 @@ def extract( or "standard". Thin wrapper over `fetch()` with the `extract` param set: no separate endpoint or auth. - Extract works on any domain once Zenrows has prepared it, with any - active API key. A domain that isn't prepared yet answers 403 with - `code: "REQS007"` (prepare it with `POST /v1/extract/prepared-domains`); - the legacy domain-not-enabled code is a 402 with `code: "AUTH010"`. - On either, `mode="auto"` retries once with `autoparse=True` instead of - returning the error response. Pass `fallback_to_autoparse=False` to - disable that and get the raw error response back. + Extract works on any prepared domain (you can prepare one yourself + with `POST /v1/extract/prepared-domains`), with any active API key. + A 403 with `code: "REQS007"` means Extract can't extract this page. + Check `reason` in the problem body (`response.json().get("reason")`) + when present; older responses may not carry it: + + - `domain_not_prepared`: prepare the domain with `prepare_domain()`, + poll `prepared_domain_status()` until `status` is `"ready"`, then + retry. + - `page_not_recognised`: the domain is served, but this page isn't + one Extract extracts (a home, search, login or category page). + Try a product, article or listing page, or retry once in a few + minutes. Preparing the domain again changes nothing. + - `domain_not_enabled`: preparing doesn't help. Use + `css_extractor`, or contact support. + + The legacy domain-not-enabled code is a 402 with `code: "AUTH010"`. + Both error responses are returned to you as is, in one request. + + `fallback_to_autoparse` is deprecated and has no effect: extract() + never retries with autoparse. Passing True emits a + `DeprecationWarning`. A 422 with `code: "no_data_extracted"` means Extract knows this kind of page but found no data on it. It is not charged and is never @@ -191,25 +283,18 @@ def extract( `adaptive_stealth=True` (default) also sends Adaptive Stealth Mode (`mode: "auto"` at the wire-param level - unrelated to this method's own - `mode` argument) on both the extract attempt and the Autoparse fallback, - so a target needing `js_render`/`premium_proxy` gets escalated + `mode` argument), so a target needing `js_render`/`premium_proxy` gets escalated automatically instead of failing with REQS002. It is skipped when `params` sets `js_render` or `premium_proxy` (the API rejects either combined with Adaptive Stealth Mode) or its own wire `mode`. Pass `adaptive_stealth=False` to turn it off entirely. """ + if fallback_to_autoparse: + warnings.warn(_FALLBACK_DEPRECATED, DeprecationWarning, stacklevel=2) final_params = dict(params) if params else {} final_params["extract"] = mode _with_adaptive_stealth(final_params, adaptive_stealth) - response = self.fetch(url, final_params, headers, **kwargs) - - if mode == "auto" and fallback_to_autoparse and _is_extract_domain_unavailable(response): - autoparse_params = dict(params) if params else {} - autoparse_params["autoparse"] = True - _with_adaptive_stealth(autoparse_params, adaptive_stealth) - return self.fetch(url, autoparse_params, headers, **kwargs) - - return response + return self.fetch(url, final_params, headers, **kwargs) def post( self, @@ -231,6 +316,64 @@ def put( ) -> requests.Response: return self._worker("PUT", url, params, headers, data=data, **kwargs) + # ---- Extract prepared domains ---- + + def prepare_domain(self, domain: str, example_urls: list[str]) -> dict: + """Submit `domain` for Extract preparation + (`POST /v1/extract/prepared-domains`) and return the parsed status + object the API answers with `202`: `domain`, `status` (`preparing`, + `ready` or `failed`), `state`, `submitted_at`, `updated_at`. + + `domain` is the domain without a scheme. `example_urls` are 1 to 5 + URLs on that domain, one per kind of page you want to extract. + Submitting a domain that is already prepared returns its current + status instead of starting a new run. + + Preparation takes minutes. This method does not wait for it: call + `prepared_domain_status(domain)` until `status` is no longer + `"preparing"`. These endpoints don't fetch pages and cost no + credits. + + The client's `retries` setting doesn't apply to these endpoints. + Raises `ValueError` for an empty `domain`, before any request, and + `PreparedDomainError` on a non-2xx answer, for example 400 + `invalid_argument`, 409 `in_progress`, 429 `out_of_allowance` or + 503 `overloaded` / `disabled`. + """ + _check_domain(domain) + response = _prepared_domains_request( + "POST", + self._prepared_domains_url(), + json={"domain": domain, "example_urls": list(example_urls)}, + headers=self._prepared_domains_headers(), + ) + return _prepared_domain_body(response) + + def prepared_domain_status(self, domain: str) -> dict: + """Read the preparation status of `domain` + (`GET /v1/extract/prepared-domains/{domain}`) and return the parsed + status object. `status` is `preparing`, `ready` or `failed`; a + failed preparation carries a `failure` object with `code`, `detail` + and `slot_consumed`. `state` and `roles` report progress. + + Raises `ValueError` for an empty `domain`, before any request, and + `PreparedDomainError` on a non-2xx answer, for example 404 + `not_found` when your account never submitted `domain`. + """ + _check_domain(domain) + response = _prepared_domains_request( + "GET", + f"{self._prepared_domains_url()}/{quote(domain, safe='')}", + headers=self._prepared_domains_headers(), + ) + return _prepared_domain_body(response) + + def _prepared_domains_url(self) -> str: + return self.api_url.rstrip("/") + "/extract/prepared-domains" + + def _prepared_domains_headers(self) -> dict: + return {"X-API-Key": self.apikey, "User-Agent": DEFAULT_USER_AGENT} + # ---- async-flavoured wrappers (thread-pool offload) ---- async def fetch_async( @@ -258,25 +401,34 @@ async def extract_async( params: dict | None = None, headers: dict | None = None, mode: str = "auto", - fallback_to_autoparse: bool = True, + fallback_to_autoparse: bool = False, adaptive_stealth: bool = True, **kwargs: Any, ) -> requests.Response: """Async counterpart of `extract()` - see its docstring for the - REQS007 / AUTH010 -> Autoparse fallback behavior, - `fallback_to_autoparse`, and `adaptive_stealth`.""" + REQS007 `reason` cases, the deprecated `fallback_to_autoparse`, and + `adaptive_stealth`.""" + if fallback_to_autoparse: + warnings.warn(_FALLBACK_DEPRECATED, DeprecationWarning, stacklevel=2) final_params = dict(params) if params else {} final_params["extract"] = mode _with_adaptive_stealth(final_params, adaptive_stealth) - response = await self.fetch_async(url, final_params, headers, **kwargs) + return await self.fetch_async(url, final_params, headers, **kwargs) - if mode == "auto" and fallback_to_autoparse and _is_extract_domain_unavailable(response): - autoparse_params = dict(params) if params else {} - autoparse_params["autoparse"] = True - _with_adaptive_stealth(autoparse_params, adaptive_stealth) - return await self.fetch_async(url, autoparse_params, headers, **kwargs) + async def prepare_domain_async(self, domain: str, example_urls: list[str]) -> dict: + """Async counterpart of `prepare_domain()`, run on the client's thread pool.""" + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + self.executor, partial(self.prepare_domain, domain, example_urls) + ) - return response + async def prepared_domain_status_async(self, domain: str) -> dict: + """Async counterpart of `prepared_domain_status()`, run on the client's + thread pool.""" + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + self.executor, partial(self.prepared_domain_status, domain) + ) async def post_async( self, diff --git a/tests/test_fetch_extract.py b/tests/test_fetch_extract.py index f417928..e1ef11d 100644 --- a/tests/test_fetch_extract.py +++ b/tests/test_fetch_extract.py @@ -7,6 +7,7 @@ `fetch()`/`extract()`/`get()` actually *return* and *raise* (or don't). """ +import warnings from unittest import IsolatedAsyncioTestCase, TestCase, mock from requests import Response, Session @@ -123,126 +124,62 @@ def test_extract_does_not_mutate_caller_supplied_params_dict(self, mock_request) self.assertEqual(caller_params, {"js_render": True}) -class TestExtractAutoparseFallback(TestCase): - """`extract(mode="auto")`: 403 REQS007 means the target domain isn't - prepared for Extract yet, and the legacy 402 AUTH010 that it isn't - enabled. On either, by default this retries once with Autoparse instead - of returning the error - same behavior as the CLI's extract adapter.""" +_DOMAIN_UNAVAILABLE = ( + (403, b'{"code": "REQS007", "reason": "domain_not_prepared"}'), + (402, b'{"code": "AUTH010", "title": "Domain not enabled"}'), +) + + +class TestExtractNeverFallsBackToAutoparse(TestCase): + """`extract()` returns 403 REQS007 and the legacy 402 AUTH010 as is, in + one request. The deprecated `fallback_to_autoparse` has no effect beyond + a DeprecationWarning.""" def setUp(self): self.client = ZenRowsClient(apikey) @mock.patch.object(Session, "request") - def test_falls_back_to_autoparse_on_auth010(self, mock_request): - mock_request.side_effect = [ - _fake_response(402, b'{"code": "AUTH010", "title": "Domain not enabled"}'), - _fake_response(200, b'[{"found": "via autoparse"}]'), - ] + def test_returned_as_is_in_one_request(self, mock_request): + for status, body in _DOMAIN_UNAVAILABLE: + with self.subTest(status=status): + mock_request.reset_mock() + mock_request.side_effect = [_fake_response(status, body), _fake_response(200)] - response = self.client.extract(url) + with warnings.catch_warnings(): + warnings.simplefilter("error") + response = self.client.extract(url) - self.assertEqual(response.status_code, 200) - self.assertEqual(mock_request.call_count, 2) - _, fallback_kwargs = mock_request.call_args - self.assertTrue(fallback_kwargs["params"].get("autoparse")) - self.assertNotIn("extract", fallback_kwargs["params"]) + self.assertEqual(response.status_code, status) + self.assertEqual(mock_request.call_count, 1) + _, kwargs = mock_request.call_args + self.assertNotIn("autoparse", kwargs["params"]) @mock.patch.object(Session, "request") - def test_falls_back_to_autoparse_on_reqs007(self, mock_request): - mock_request.side_effect = [ - _fake_response(403, b'{"code": "REQS007", "title": "Domain not prepared"}'), - _fake_response(200, b'[{"found": "via autoparse"}]'), - ] - - response = self.client.extract(url) - - self.assertEqual(response.status_code, 200) - self.assertEqual(mock_request.call_count, 2) - _, fallback_kwargs = mock_request.call_args - self.assertTrue(fallback_kwargs["params"].get("autoparse")) - self.assertNotIn("extract", fallback_kwargs["params"]) - - @mock.patch.object(Session, "request") - def test_reqs007_fallback_disabled_returns_error_response(self, mock_request): - mock_request.return_value = _fake_response(403, b'{"code": "REQS007"}') - - response = self.client.extract(url, fallback_to_autoparse=False) - - self.assertEqual(response.status_code, 403) - self.assertEqual(mock_request.call_count, 1) + def test_deprecated_option_has_no_effect_and_warns(self, mock_request): + for status, body in _DOMAIN_UNAVAILABLE: + with self.subTest(status=status): + mock_request.reset_mock() + mock_request.side_effect = [_fake_response(status, body), _fake_response(200)] + + with self.assertWarns(DeprecationWarning) as caught: + response = self.client.extract(url, fallback_to_autoparse=True) + + self.assertEqual(response.status_code, status) + self.assertEqual(mock_request.call_count, 1) + _, kwargs = mock_request.call_args + self.assertNotIn("autoparse", kwargs["params"]) + self.assertIn("prepare_domain()", str(caught.warning)) + self.assertEqual(caught.filename, __file__) @mock.patch.object(Session, "request") - def test_403_without_reqs007_does_not_fall_back(self, mock_request): - """A Zenrows auth 403 must not be mistaken for a domain that isn't - prepared yet.""" + def test_reason_is_readable_from_the_response(self, mock_request): mock_request.return_value = _fake_response( - 403, b'{"code": "AUTH002", "title": "Invalid API key"}' + 403, b'{"code": "REQS007", "reason": "page_not_recognised"}' ) response = self.client.extract(url) - self.assertEqual(response.status_code, 403) - self.assertEqual(mock_request.call_count, 1) - - @mock.patch.object(Session, "request") - def test_code_on_the_wrong_status_does_not_fall_back(self, mock_request): - """REQS007 is only the not-prepared signal on a 403.""" - mock_request.return_value = _fake_response(400, b'{"code": "REQS007"}') - - response = self.client.extract(url) - - self.assertEqual(response.status_code, 400) - self.assertEqual(mock_request.call_count, 1) - - @mock.patch.object(Session, "request") - def test_fallback_disabled_returns_error_response(self, mock_request): - mock_request.return_value = _fake_response(402, b'{"code": "AUTH010"}') - - response = self.client.extract(url, fallback_to_autoparse=False) - - self.assertEqual(response.status_code, 402) - self.assertEqual(mock_request.call_count, 1) - - @mock.patch.object(Session, "request") - def test_402_without_auth010_does_not_fall_back(self, mock_request): - """A real credits-exhausted 402 (e.g. AUTH004) must not be mistaken - for the domain-not-enabled error.""" - mock_request.return_value = _fake_response( - 402, b'{"code": "AUTH004", "title": "No credit available"}' - ) - - response = self.client.extract(url) - - self.assertEqual(response.status_code, 402) - self.assertEqual(mock_request.call_count, 1) - - @mock.patch.object(Session, "request") - def test_no_fallback_for_non_auto_mode(self, mock_request): - """Neither code triggers a fallback for native/standard modes - - only "auto" takes the fallback path.""" - mock_request.side_effect = [ - _fake_response(402, b'{"code": "AUTH010"}'), - _fake_response(403, b'{"code": "REQS007"}'), - ] - - first = self.client.extract(url, mode="native") - second = self.client.extract(url, mode="standard") - - self.assertEqual(first.status_code, 402) - self.assertEqual(second.status_code, 403) - self.assertEqual(mock_request.call_count, 2) - - @mock.patch.object(Session, "request") - def test_fallback_does_not_mutate_caller_supplied_params_dict(self, mock_request): - mock_request.side_effect = [ - _fake_response(402, b'{"code": "AUTH010"}'), - _fake_response(200, b"{}"), - ] - caller_params = {"js_render": True} - - self.client.extract(url, params=caller_params) - - self.assertEqual(caller_params, {"js_render": True}) + self.assertEqual(response.json().get("reason"), "page_not_recognised") class TestExtractAdaptiveStealth(TestCase): @@ -271,18 +208,6 @@ def test_omits_wire_mode_when_disabled(self, mock_request): _, kwargs = mock_request.call_args self.assertNotIn("mode", kwargs["params"]) - @mock.patch.object(Session, "request") - def test_fallback_request_also_carries_adaptive_stealth(self, mock_request): - mock_request.side_effect = [ - _fake_response(402, b'{"code": "AUTH010"}'), - _fake_response(200, b"{}"), - ] - - self.client.extract(url) - - _, fallback_kwargs = mock_request.call_args - self.assertEqual(fallback_kwargs["params"]["mode"], "auto") - @mock.patch.object(Session, "request") def test_skips_adaptive_stealth_when_js_render_is_set(self, mock_request): """The API rejects mode=auto combined with js_render, so a caller's @@ -322,67 +247,40 @@ def test_keeps_a_wire_mode_the_caller_set(self, mock_request): _, kwargs = mock_request.call_args self.assertEqual(kwargs["params"]["mode"], "custom") - @mock.patch.object(Session, "request") - def test_fallback_request_skips_adaptive_stealth_with_premium_proxy(self, mock_request): - mock_request.side_effect = [ - _fake_response(403, b'{"code": "REQS007"}'), - _fake_response(200, b"{}"), - ] - - self.client.extract(url, params={"premium_proxy": True}) - - first_kwargs = mock_request.call_args_list[0].kwargs - _, fallback_kwargs = mock_request.call_args - self.assertNotIn("mode", first_kwargs["params"]) - self.assertNotIn("mode", fallback_kwargs["params"]) - self.assertTrue(fallback_kwargs["params"]["autoparse"]) - -class TestExtractAsyncAutoparseFallback(IsolatedAsyncioTestCase): - """Async counterpart - same REQS007 / AUTH010 -> Autoparse behavior.""" +class TestExtractAsyncNeverFallsBackToAutoparse(IsolatedAsyncioTestCase): + """Async counterpart: one request, error returned as is, and a + DeprecationWarning for `fallback_to_autoparse=True`.""" def setUp(self): self.client = ZenRowsClient(apikey, concurrency=2) @mock.patch.object(Session, "request") - async def test_falls_back_to_autoparse_on_auth010(self, mock_request): + async def test_returned_as_is_in_one_request(self, mock_request): mock_request.side_effect = [ + _fake_response(403, b'{"code": "REQS007"}'), _fake_response(402, b'{"code": "AUTH010"}'), - _fake_response(200, b'[{"found": "via autoparse"}]'), ] - response = await self.client.extract_async(url) + first = await self.client.extract_async(url) + second = await self.client.extract_async(url) - self.assertEqual(response.status_code, 200) + self.assertEqual(first.status_code, 403) + self.assertEqual(second.status_code, 402) self.assertEqual(mock_request.call_count, 2) @mock.patch.object(Session, "request") - async def test_skips_adaptive_stealth_when_js_render_is_set(self, mock_request): + async def test_deprecated_option_has_no_effect_and_warns(self, mock_request): mock_request.side_effect = [ _fake_response(403, b'{"code": "REQS007"}'), - _fake_response(200, b"{}"), + _fake_response(200), ] - await self.client.extract_async(url, params={"js_render": True}) + with self.assertWarns(DeprecationWarning): + response = await self.client.extract_async(url, fallback_to_autoparse=True) - first_kwargs = mock_request.call_args_list[0].kwargs - _, fallback_kwargs = mock_request.call_args - self.assertNotIn("mode", first_kwargs["params"]) - self.assertNotIn("mode", fallback_kwargs["params"]) - - @mock.patch.object(Session, "request") - async def test_falls_back_to_autoparse_on_reqs007(self, mock_request): - mock_request.side_effect = [ - _fake_response(403, b'{"code": "REQS007"}'), - _fake_response(200, b'[{"found": "via autoparse"}]'), - ] - - response = await self.client.extract_async(url) - - self.assertEqual(response.status_code, 200) - self.assertEqual(mock_request.call_count, 2) - _, fallback_kwargs = mock_request.call_args - self.assertTrue(fallback_kwargs["params"].get("autoparse")) + self.assertEqual(response.status_code, 403) + self.assertEqual(mock_request.call_count, 1) class TestFetchExtractAsync(IsolatedAsyncioTestCase): diff --git a/tests/test_prepared_domains.py b/tests/test_prepared_domains.py new file mode 100644 index 0000000..bf3f1fb --- /dev/null +++ b/tests/test_prepared_domains.py @@ -0,0 +1,301 @@ +"""prepare_domain() / prepared_domain_status() against a stubbed HTTP server: +the request each one sends, the parsed status it returns, and the +PreparedDomainError it raises on every documented error answer.""" + +import json +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer +from unittest import IsolatedAsyncioTestCase, TestCase + +from zenrows import PreparedDomainError, ZenRowsClient + +apikey = "APIKEY" + +PREPARING = { + "domain": "www.example.com", + "status": "preparing", + "state": "received", + "submitted_at": "2026-10-15T10:00:00Z", + "updated_at": "2026-10-15T10:00:00Z", +} + + +def _problem(status: int, code: str, **extra: str) -> dict: + return { + "code": code, + "detail": f"detail for {code}", + "status": status, + "title": f"Title ({code})", + "type": f"https://docs.zenrows.com/api-error-codes#{code}", + **extra, + } + + +class _StubAPI: + """A local HTTP server that records each request and answers with the + next canned (status, body, headers) reply.""" + + def __init__(self) -> None: + self.requests: list[dict] = [] + self.replies: list[tuple[int, object, dict]] = [] + stub = self + + class Handler(BaseHTTPRequestHandler): + def _answer(self) -> None: + length = int(self.headers.get("Content-Length") or 0) + raw = self.rfile.read(length) if length else b"" + stub.requests.append( + { + "method": self.command, + "path": self.path, + "headers": dict(self.headers), + "body": json.loads(raw) if raw else None, + } + ) + status, body, headers = stub.replies.pop(0) + payload = body if isinstance(body, bytes) else json.dumps(body).encode() + self.send_response(status) + content_type = "application/json" if status < 400 else "application/problem+json" + self.send_header("Content-Type", content_type) + for name, value in headers.items(): + self.send_header(name, value) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + do_GET = _answer + do_POST = _answer + + def log_message(self, *args: object) -> None: + pass + + self.server = HTTPServer(("127.0.0.1", 0), Handler) + self.base_url = f"http://127.0.0.1:{self.server.server_address[1]}/v1/" + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + + def reply(self, status: int, body: object, headers: dict | None = None) -> None: + self.replies.append((status, body, headers or {})) + + def close(self) -> None: + self.server.shutdown() + self.server.server_close() + + +class TestPreparedDomains(TestCase): + def setUp(self): + self.api = _StubAPI() + self.addCleanup(self.api.close) + self.client = ZenRowsClient(apikey, base_url=self.api.base_url) + self.addCleanup(self.client.close) + + def test_prepare_domain_posts_domain_and_example_urls(self): + self.api.reply(202, PREPARING) + + status = self.client.prepare_domain( + "www.example.com", ["https://www.example.com/a", "https://www.example.com/b"] + ) + + self.assertEqual(status, PREPARING) + self.assertEqual(len(self.api.requests), 1) + sent = self.api.requests[0] + self.assertEqual(sent["method"], "POST") + self.assertEqual(sent["path"], "/v1/extract/prepared-domains") + self.assertEqual( + sent["body"], + { + "domain": "www.example.com", + "example_urls": ["https://www.example.com/a", "https://www.example.com/b"], + }, + ) + self.assertEqual(sent["headers"]["X-API-Key"], apikey) + self.assertNotIn(apikey, sent["path"]) + + def test_prepared_domain_status_gets_the_domain_and_returns_the_status(self): + ready = { + **PREPARING, + "status": "ready", + "state": "ready", + "roles": [{"role": "listing", "status": "ready", "url": "https://www.example.com/a"}], + } + self.api.reply(200, ready) + + status = self.client.prepared_domain_status("www.example.com") + + self.assertEqual(status, ready) + sent = self.api.requests[0] + self.assertEqual(sent["method"], "GET") + self.assertEqual(sent["path"], "/v1/extract/prepared-domains/www.example.com") + self.assertEqual(sent["headers"]["X-API-Key"], apikey) + + def test_status_returns_a_failed_preparation_without_raising(self): + failed = { + **PREPARING, + "status": "failed", + "state": "failed", + "failure": {"code": "unsupported", "detail": "x", "slot_consumed": True}, + } + self.api.reply(200, failed) + + status = self.client.prepared_domain_status("www.example.com") + + self.assertEqual(status["failure"]["code"], "unsupported") + + def test_status_escapes_the_domain_in_the_path(self): + self.api.reply(200, PREPARING) + + self.client.prepared_domain_status("bad/domain?x") + + self.assertEqual( + self.api.requests[0]["path"], "/v1/extract/prepared-domains/bad%2Fdomain%3Fx" + ) + + def test_prepare_domain_does_not_poll(self): + self.api.reply(202, PREPARING) + + self.client.prepare_domain("www.example.com", ["https://www.example.com/"]) + + self.assertEqual(len(self.api.requests), 1) + + def test_error_answers_raise_prepared_domain_error(self): + cases = [ + (400, "invalid_argument", {"reason": "invalid_domain"}), + (409, "in_progress", {}), + (429, "out_of_allowance", {}), + (503, "overloaded", {}), + (503, "disabled", {}), + ] + for status, code, extra in cases: + with self.subTest(code=code): + self.api.reply(status, _problem(status, code, **extra), {"Retry-After": "7"}) + + with self.assertRaises(PreparedDomainError) as caught: + self.client.prepare_domain("www.example.com", ["https://www.example.com/"]) + + err = caught.exception + self.assertEqual(err.status_code, status) + self.assertEqual(err.code, code) + self.assertEqual(err.detail, f"detail for {code}") + self.assertEqual(err.reason, extra.get("reason")) + self.assertEqual(err.retry_after, 7.0) + self.assertEqual(err.problem["status"], status) + self.assertIn(code, str(err)) + + def test_status_of_an_unknown_domain_raises_not_found(self): + self.api.reply(404, _problem(404, "not_found")) + + with self.assertRaises(PreparedDomainError) as caught: + self.client.prepared_domain_status("www.example.com") + + self.assertEqual(caught.exception.status_code, 404) + self.assertEqual(caught.exception.code, "not_found") + + def test_error_without_a_json_body(self): + self.api.reply(502, b"bad gateway") + + with self.assertRaises(PreparedDomainError) as caught: + self.client.prepared_domain_status("www.example.com") + + self.assertEqual(caught.exception.status_code, 502) + self.assertIsNone(caught.exception.code) + self.assertIsNone(caught.exception.problem) + + def test_empty_domain_is_rejected_before_any_request(self): + for domain in ("", " ", "\t\n"): + with self.subTest(domain=domain): + with self.assertRaises(ValueError): + self.client.prepared_domain_status(domain) + with self.assertRaises(ValueError): + self.client.prepare_domain(domain, ["https://www.example.com/"]) + + self.assertEqual(self.api.requests, []) + + +class TestPreparedDomainsIgnoreRetries(TestCase): + """The client's `retries` apply to scraping only: a 429 or 503 from the + prepared-domains endpoints is returned on the first answer.""" + + def setUp(self): + self.api = _StubAPI() + self.addCleanup(self.api.close) + self.client = ZenRowsClient(apikey, retries=1, base_url=self.api.base_url) + self.addCleanup(self.client.close) + + def test_status_503_is_not_retried(self): + self.api.reply(503, _problem(503, "overloaded")) + self.api.reply(200, PREPARING) + + with self.assertRaises(PreparedDomainError) as caught: + self.client.prepared_domain_status("www.example.com") + + self.assertEqual(caught.exception.code, "overloaded") + self.assertEqual(len(self.api.requests), 1) + + def test_prepare_429_is_not_retried(self): + self.api.reply(429, _problem(429, "out_of_allowance")) + self.api.reply(202, PREPARING) + + with self.assertRaises(PreparedDomainError) as caught: + self.client.prepare_domain("www.example.com", ["https://www.example.com/"]) + + self.assertEqual(caught.exception.code, "out_of_allowance") + self.assertEqual(len(self.api.requests), 1) + + def test_scraping_keeps_its_retries(self): + """Control: the same client still retries a 503 on a scraping request.""" + self.api.reply(503, b"busy") + self.api.reply(200, b"ok") + + response = self.client.fetch("https://www.example.com/") + + self.assertEqual(response.status_code, 200) + self.assertEqual(len(self.api.requests), 2) + + +class TestPreparedDomainsAsync(IsolatedAsyncioTestCase): + def setUp(self): + self.api = _StubAPI() + self.addCleanup(self.api.close) + self.client = ZenRowsClient(apikey, concurrency=2, base_url=self.api.base_url) + self.addCleanup(self.client.close) + + async def test_prepare_domain_async_returns_the_status(self): + self.api.reply(202, PREPARING) + + status = await self.client.prepare_domain_async( + "www.example.com", ["https://www.example.com/a"] + ) + + self.assertEqual(status, PREPARING) + sent = self.api.requests[0] + self.assertEqual(sent["method"], "POST") + self.assertEqual( + sent["body"], + {"domain": "www.example.com", "example_urls": ["https://www.example.com/a"]}, + ) + + async def test_prepared_domain_status_async_returns_the_status(self): + self.api.reply(200, {**PREPARING, "status": "ready"}) + + status = await self.client.prepared_domain_status_async("www.example.com") + + self.assertEqual(status["status"], "ready") + self.assertEqual( + self.api.requests[0]["path"], "/v1/extract/prepared-domains/www.example.com" + ) + + async def test_async_errors_raise_prepared_domain_error(self): + self.api.reply(404, _problem(404, "not_found")) + + with self.assertRaises(PreparedDomainError) as caught: + await self.client.prepared_domain_status_async("www.example.com") + + self.assertEqual(caught.exception.code, "not_found") + + async def test_async_rejects_an_empty_domain(self): + with self.assertRaises(ValueError): + await self.client.prepared_domain_status_async(" ") + with self.assertRaises(ValueError): + await self.client.prepare_domain_async("", ["https://www.example.com/"]) + + self.assertEqual(self.api.requests, [])