diff --git a/docs/02_concepts/05_retries.mdx b/docs/02_concepts/05_retries.mdx index 6b290f46..54cf2562 100644 --- a/docs/02_concepts/05_retries.mdx +++ b/docs/02_concepts/05_retries.mdx @@ -28,7 +28,7 @@ By default, the client retries a failed request up to 4 times. The retry interva You can customize this behavior using the following options in the `ApifyClient` constructor: -- `max_retries`: Defines the maximum number of retry attempts. +- `max_retries`: Defines the maximum number of retry attempts. Set it to `0` to disable retries. - `min_delay_between_retries`: Sets the minimum delay between retries as a `timedelta`. Retries with exponential backoff help reduce the load on the server and increase the chances of a successful request. diff --git a/src/apify_client/_apify_client.py b/src/apify_client/_apify_client.py index 73beadef..e0b0a00e 100644 --- a/src/apify_client/_apify_client.py +++ b/src/apify_client/_apify_client.py @@ -140,7 +140,7 @@ def __init__( as well. api_public_url: The globally accessible URL of the Apify API server. Should be set only if `api_url` is an internal URL that is not globally accessible. Defaults to https://api.apify.com. - max_retries: How many times to retry a failed request at most. + max_retries: How many times to retry a failed request at most. Set to 0 to disable retries. min_delay_between_retries: How long will the client wait between retrying requests (increases exponentially from this value). timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...). @@ -150,7 +150,13 @@ def __init__( headers: Additional HTTP headers to include in all API requests. compression: Compression algorithm for request bodies. Pass a string literal to select an algorithm, or an `HttpCompressor` instance for finer-grained control. + + Raises: + ValueError: If `max_retries` is negative. """ + if max_retries < 0: + raise ValueError(f'max_retries must be a non-negative integer, got {max_retries}') + # We need to do this because of mocking in tests and default mutable arguments. api_url = DEFAULT_API_URL if api_url is None else api_url api_public_url = DEFAULT_API_URL if api_public_url is None else api_public_url @@ -504,7 +510,7 @@ def __init__( as well. api_public_url: The globally accessible URL of the Apify API server. Should be set only if `api_url` is an internal URL that is not globally accessible. Defaults to https://api.apify.com. - max_retries: How many times to retry a failed request at most. + max_retries: How many times to retry a failed request at most. Set to 0 to disable retries. min_delay_between_retries: How long will the client wait between retrying requests (increases exponentially from this value). timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...). @@ -514,7 +520,13 @@ def __init__( headers: Additional HTTP headers to include in all API requests. compression: Compression algorithm for request bodies. Pass a string literal to select an algorithm, or an `HttpCompressor` instance for finer-grained control. + + Raises: + ValueError: If `max_retries` is negative. """ + if max_retries < 0: + raise ValueError(f'max_retries must be a non-negative integer, got {max_retries}') + # We need to do this because of mocking in tests and default mutable arguments. api_url = DEFAULT_API_URL if api_url is None else api_url api_public_url = DEFAULT_API_URL if api_public_url is None else api_public_url diff --git a/src/apify_client/http_clients/_base.py b/src/apify_client/http_clients/_base.py index 5ffa6551..6d448689 100644 --- a/src/apify_client/http_clients/_base.py +++ b/src/apify_client/http_clients/_base.py @@ -136,12 +136,18 @@ def __init__( timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...). timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...). timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts. - max_retries: Maximum number of retries for failed requests. + max_retries: Maximum number of retries for failed requests. Set to 0 to disable retries. min_delay_between_retries: Minimum delay between retries. statistics: Statistics tracker for API calls. Created automatically if not provided. headers: Additional HTTP headers to include in all requests. http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`. + + Raises: + ValueError: If `max_retries` is negative. """ + if max_retries < 0: + raise ValueError(f'max_retries must be a non-negative integer, got {max_retries}') + self._http_compressor = http_compressor if http_compressor is not None else GzipHttpCompressor() self._timeout_short = timeout_short self._timeout_medium = timeout_medium @@ -598,9 +604,6 @@ def _retry_with_exp_backoff( random_factor: float = 1, ) -> T: """Retry a function with exponential backoff and jitter.""" - if max_retries < 1: - raise ValueError(f'max_retries must be at least 1, got {max_retries}') - random_factor = min(max(0, random_factor), 1) backoff_factor = min(max(1, backoff_factor), 10) swallow = True @@ -825,9 +828,6 @@ async def _retry_with_exp_backoff( random_factor: float = 1, ) -> T: """Retry an async function with exponential backoff and jitter.""" - if max_retries < 1: - raise ValueError(f'max_retries must be at least 1, got {max_retries}') - random_factor = min(max(0, random_factor), 1) backoff_factor = min(max(1, backoff_factor), 10) swallow = True diff --git a/src/apify_client/http_clients/_httpx2.py b/src/apify_client/http_clients/_httpx2.py index 48eac9d9..ca833955 100644 --- a/src/apify_client/http_clients/_httpx2.py +++ b/src/apify_client/http_clients/_httpx2.py @@ -75,7 +75,7 @@ def __init__( timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...). timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...). timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts. - max_retries: Maximum number of retry attempts for failed requests. + max_retries: Maximum number of retry attempts for failed requests. Set to 0 to disable retries. min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt). statistics: Statistics tracker for API calls. Created automatically if not provided. headers: Additional HTTP headers to include in all requests. @@ -179,7 +179,7 @@ def __init__( timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...). timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...). timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts. - max_retries: Maximum number of retry attempts for failed requests. + max_retries: Maximum number of retry attempts for failed requests. Set to 0 to disable retries. min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt). statistics: Statistics tracker for API calls. Created automatically if not provided. headers: Additional HTTP headers to include in all requests. diff --git a/src/apify_client/http_clients/_impit.py b/src/apify_client/http_clients/_impit.py index 5d01b2c2..6b1c82bb 100644 --- a/src/apify_client/http_clients/_impit.py +++ b/src/apify_client/http_clients/_impit.py @@ -71,7 +71,7 @@ def __init__( timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...). timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...). timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts. - max_retries: Maximum number of retry attempts for failed requests. + max_retries: Maximum number of retry attempts for failed requests. Set to 0 to disable retries. min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt). statistics: Statistics tracker for API calls. Created automatically if not provided. headers: Additional HTTP headers to include in all requests. @@ -171,7 +171,7 @@ def __init__( timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...). timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...). timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts. - max_retries: Maximum number of retry attempts for failed requests. + max_retries: Maximum number of retry attempts for failed requests. Set to 0 to disable retries. min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt). statistics: Statistics tracker for API calls. Created automatically if not provided. headers: Additional HTTP headers to include in all requests. diff --git a/tests/unit/test_client_errors.py b/tests/unit/test_client_errors.py index 39696147..1536ff2d 100644 --- a/tests/unit/test_client_errors.py +++ b/tests/unit/test_client_errors.py @@ -7,6 +7,7 @@ import pytest from werkzeug import Response +from apify_client import ApifyClient, ApifyClientAsync from apify_client.errors import ( ApifyApiError, ConflictError, @@ -25,7 +26,6 @@ from pytest_httpserver import HTTPServer from werkzeug import Request - from apify_client import ApifyClient, ApifyClientAsync from apify_client.http_clients import HttpClient, HttpClientAsync pytestmark = pytest.mark.usefixtures('http_client_classes') @@ -195,7 +195,7 @@ def test_apify_api_error_dispatches_5xx_to_server_error( httpserver.expect_request('/server_error').respond_with_json( {'error': {'type': 'internal-error', 'message': 'boom'}}, status=503 ) - client = http_client_class(max_retries=1) + client = http_client_class(max_retries=0) with pytest.raises(ServerError) as exc: client.call(method='GET', url=str(httpserver.url_for('/server_error'))) @@ -204,6 +204,32 @@ def test_apify_api_error_dispatches_5xx_to_server_error( assert exc.value.status_code == 503 +def test_max_retries_zero_makes_single_attempt(httpserver: HTTPServer) -> None: + """With `max_retries=0`, a retryable server error is requested once and raised as `ServerError`.""" + httpserver.expect_request('/v2/datasets/ds-1').respond_with_json( + {'error': {'type': 'internal-error', 'message': 'boom'}}, status=500 + ) + client = ApifyClient(token='test', api_url=httpserver.url_for('/').removesuffix('/'), max_retries=0) + + with pytest.raises(ServerError): + client.dataset('ds-1').get() + + assert len(httpserver.log) == 1 + + +async def test_max_retries_zero_makes_single_attempt_async(httpserver: HTTPServer) -> None: + """With `max_retries=0`, a retryable server error is requested once and raised as `ServerError`.""" + httpserver.expect_request('/v2/datasets/ds-1').respond_with_json( + {'error': {'type': 'internal-error', 'message': 'boom'}}, status=500 + ) + client = ApifyClientAsync(token='test', api_url=httpserver.url_for('/').removesuffix('/'), max_retries=0) + + with pytest.raises(ServerError): + await client.dataset('ds-1').get() + + assert len(httpserver.log) == 1 + + def test_apify_api_error_falls_back_for_unmapped_status( httpserver: HTTPServer, http_client_class: type[HttpClient] ) -> None: @@ -242,8 +268,8 @@ def test_apify_api_error_dispatches_all_mapped_statuses( httpserver.expect_request('/dispatch_all').respond_with_json( {'error': {'type': 'some-type', 'message': 'msg'}}, status=status_code ) - # Use max_retries=1 so retryable statuses (429) don't loop during the test. - client = http_client_class(max_retries=1) + # `max_retries=0` keeps retryable statuses (429) from being retried. + client = http_client_class(max_retries=0) with pytest.raises(expected_cls) as exc: client.call(method='GET', url=str(httpserver.url_for('/dispatch_all'))) @@ -272,7 +298,7 @@ def test_apify_api_error_falls_back_for_unparsable_body( ) -> None: """When the body can't be parsed, status-based dispatch still applies and `.type` is None.""" httpserver.expect_request('/unparsable').respond_with_data('', status=418, content_type='text/html') - client = http_client_class(max_retries=1) + client = http_client_class(max_retries=0) with pytest.raises(ApifyApiError) as exc: client.call(method='GET', url=str(httpserver.url_for('/unparsable'))) diff --git a/tests/unit/test_http_clients.py b/tests/unit/test_http_clients.py index f00b5da5..00d07b9a 100644 --- a/tests/unit/test_http_clients.py +++ b/tests/unit/test_http_clients.py @@ -17,6 +17,7 @@ import impit import pytest +from apify_client import ApifyClient, ApifyClientAsync from apify_client._consts import MIN_COMPRESSION_SIZE from apify_client._logging import logger_name from apify_client._statistics import ClientStatistics @@ -210,6 +211,24 @@ def test_base_http_client_initialization() -> None: assert isinstance(client2._statistics, ClientStatistics) +@pytest.mark.parametrize( + 'client_class', + [ + pytest.param(ApifyClient, id='ApifyClient'), + pytest.param(ApifyClientAsync, id='ApifyClientAsync'), + pytest.param(ConcreteHttpClient, id='custom HttpClient'), + pytest.param(ImpitHttpClient, id='ImpitHttpClient'), + pytest.param(ImpitHttpClientAsync, id='ImpitHttpClientAsync'), + pytest.param(Httpx2HttpClient, id='Httpx2HttpClient'), + pytest.param(Httpx2HttpClientAsync, id='Httpx2HttpClientAsync'), + ], +) +def test_negative_max_retries_is_rejected_at_construction(client_class: Callable[..., object]) -> None: + """A negative `max_retries` raises `ValueError` from the constructor, before any request is made.""" + with pytest.raises(ValueError, match='max_retries must be a non-negative integer, got -1'): + client_class(max_retries=-1) + + def test_http_client_init_headers_override_defaults_case_insensitively() -> None: """Constructor headers replace same-named default headers even when their casings differ.""" client = ConcreteHttpClient(token='default_token', headers={'authorization': 'Bearer custom'})