diff --git a/src/apify_client/_apify_client.py b/src/apify_client/_apify_client.py index da48ab22..c15add93 100644 --- a/src/apify_client/_apify_client.py +++ b/src/apify_client/_apify_client.py @@ -5,7 +5,6 @@ from apify_client._client_registry import ClientRegistry, ClientRegistryAsync from apify_client._consts import ( - API_VERSION, DEFAULT_API_URL, DEFAULT_MAX_RETRIES, DEFAULT_MIN_DELAY_BETWEEN_RETRIES, @@ -72,7 +71,7 @@ WebhookDispatchCollectionClientAsync, ) from apify_client._statistics import ClientStatistics -from apify_client._utils.http import check_custom_headers +from apify_client._utils.http import check_custom_headers, to_api_base_url from apify_client.http_clients import HttpClient, HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync from apify_client.http_compressors._resolve import resolve_compressor @@ -138,9 +137,10 @@ def __init__( [Integrations](https://console.apify.com/account/integrations) page in the Apify Console. api_url: The URL of the Apify API server to connect to. Defaults to https://api.apify.com. It can be an internal URL that is not globally accessible, in which case `api_public_url` should be set - as well. + as well. The `/v2` API version segment is appended unless the URL path already ends with it. api_public_url: The globally accessible URL of the Apify API server. Should be set only if `api_url` - is an internal URL that is not globally accessible. Defaults to https://api.apify.com. + is an internal URL that is not globally accessible. Defaults to https://api.apify.com. The `/v2` + API version segment is appended unless the URL path already ends with it. max_retries: How many times to retry a failed request at most. Set to 0 to disable retries. min_delay_between_retries: How long will the client wait between retrying requests (increases exponentially from this value). @@ -171,10 +171,10 @@ def __init__( self._token = token """Apify API token for authentication.""" - self._base_url = f'{api_url.rstrip("/")}/{API_VERSION}' + self._base_url = to_api_base_url(api_url) """Base URL of the Apify API.""" - self._public_base_url = f'{api_public_url.rstrip("/")}/{API_VERSION}' + self._public_base_url = to_api_base_url(api_public_url) """Public base URL for CDN access.""" self._statistics = ClientStatistics() @@ -261,8 +261,10 @@ def send_request(self, *, method, url, headers, content, timeout, stream) -> Htt Args: token: The Apify API token. It is set as the `Authorization` header on the custom client, unless the client already has one configured. - api_url: The URL of the Apify API server to connect to. Defaults to https://api.apify.com. + api_url: The URL of the Apify API server to connect to. Defaults to https://api.apify.com. The `/v2` + API version segment is appended unless the URL path already ends with it. api_public_url: The globally accessible URL of the Apify API server. Defaults to https://api.apify.com. + The `/v2` API version segment is appended unless the URL path already ends with it. http_client: A custom HTTP client instance extending `HttpClient`. """ instance = cls(token=token, api_url=api_url, api_public_url=api_public_url) @@ -514,9 +516,10 @@ def __init__( [Integrations](https://console.apify.com/account/integrations) page in the Apify Console. api_url: The URL of the Apify API server to connect to. Defaults to https://api.apify.com. It can be an internal URL that is not globally accessible, in which case `api_public_url` should be set - as well. + as well. The `/v2` API version segment is appended unless the URL path already ends with it. api_public_url: The globally accessible URL of the Apify API server. Should be set only if `api_url` - is an internal URL that is not globally accessible. Defaults to https://api.apify.com. + is an internal URL that is not globally accessible. Defaults to https://api.apify.com. The `/v2` + API version segment is appended unless the URL path already ends with it. max_retries: How many times to retry a failed request at most. Set to 0 to disable retries. min_delay_between_retries: How long will the client wait between retrying requests (increases exponentially from this value). @@ -547,10 +550,10 @@ def __init__( self._token = token """Apify API token for authentication.""" - self._base_url = f'{api_url.rstrip("/")}/{API_VERSION}' + self._base_url = to_api_base_url(api_url) """Base URL of the Apify API.""" - self._public_base_url = f'{api_public_url.rstrip("/")}/{API_VERSION}' + self._public_base_url = to_api_base_url(api_public_url) """Public base URL for CDN access.""" self._statistics = ClientStatistics() @@ -637,8 +640,10 @@ async def send_request(self, *, method, url, headers, content, timeout, stream) Args: token: The Apify API token. It is set as the `Authorization` header on the custom client, unless the client already has one configured. - api_url: The URL of the Apify API server to connect to. Defaults to https://api.apify.com. + api_url: The URL of the Apify API server to connect to. Defaults to https://api.apify.com. The `/v2` + API version segment is appended unless the URL path already ends with it. api_public_url: The globally accessible URL of the Apify API server. Defaults to https://api.apify.com. + The `/v2` API version segment is appended unless the URL path already ends with it. http_client: A custom HTTP client instance extending `HttpClientAsync`. """ instance = cls(token=token, api_url=api_url, api_public_url=api_public_url) diff --git a/src/apify_client/_utils/http.py b/src/apify_client/_utils/http.py index c6d90d95..fab01a78 100644 --- a/src/apify_client/_utils/http.py +++ b/src/apify_client/_utils/http.py @@ -2,11 +2,12 @@ import warnings from typing import TYPE_CHECKING, Any -from urllib.parse import quote +from urllib.parse import quote, urlsplit from apify_client._consts import ( ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES, ALREADY_COMPRESSED_MEDIA_TYPES, + API_VERSION, COMPRESSIBLE_MEDIA_TYPE_SUFFIXES, COMPRESSIBLE_MEDIA_TYPES, OVERRIDABLE_DEFAULT_HEADERS, @@ -29,6 +30,25 @@ def to_safe_id(id: str) -> str: return id.replace('/', '~') +def to_api_base_url(url: str) -> str: + """Turn a user-supplied API server URL into the versioned base URL that request paths are appended to. + + The API version segment is appended only when the URL path does not already end with it, so both + `https://api.apify.com` and `https://api.apify.com/v2` resolve to `https://api.apify.com/v2`. Trailing slashes are + dropped. + + Args: + url: The URL of the Apify API server, with or without the API version segment. + + Returns: + The URL ending with the API version segment and no trailing slash. + """ + base_url = url.rstrip('/') + if urlsplit(base_url).path.endswith(f'/{API_VERSION}'): + return base_url + return f'{base_url}/{API_VERSION}' + + def to_path_segment(value: str) -> str: """Percent-encode a caller-supplied value so it stays a single URL path segment. diff --git a/tests/unit/test_url_generation.py b/tests/unit/test_url_generation.py index 2641ab3a..5e05b66a 100644 --- a/tests/unit/test_url_generation.py +++ b/tests/unit/test_url_generation.py @@ -238,3 +238,43 @@ async def test_kvs_record_public_url_async(api_url: str, api_public_url: str | N f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/key-value-stores/{MOCKED_KVS_ID}/' f'records/{key}{expected_signature}' ) + + +parametrized_versioned_api_urls = pytest.mark.parametrize( + ('api_url', 'expected_base_url'), + [ + pytest.param('https://api.apify.com', 'https://api.apify.com/v2', id='no version'), + pytest.param('https://api.apify.com/', 'https://api.apify.com/v2', id='no version, trailing slash'), + pytest.param('https://api.apify.com/v2', 'https://api.apify.com/v2', id='with version'), + pytest.param('https://api.apify.com/v2/', 'https://api.apify.com/v2', id='with version, trailing slash'), + pytest.param('http://10.0.88.214:8010/v2', 'http://10.0.88.214:8010/v2', id='internal url with version'), + pytest.param('https://proxy.example.com/apify/v2', 'https://proxy.example.com/apify/v2', id='custom path'), + pytest.param('https://v2.example.com', 'https://v2.example.com/v2', id='version only in host'), + ], +) + + +@parametrized_versioned_api_urls +def test_api_url_version_appended_once_sync(api_url: str, expected_base_url: str) -> None: + """The `/v2` segment is appended to `api_url` and `api_public_url` only when missing, with sync client.""" + client = ApifyClient(token='dummy-token', api_url=api_url, api_public_url=api_url) + kvs = client.key_value_store(MOCKED_KVS_ID) + + with mock.patch.object(client._http_client, 'call', return_value=_get_mocked_kvs_response()) as call_mock: + public_url = kvs.get_record_public_url(key='some_key') + + assert call_mock.call_args.kwargs['url'] == f'{expected_base_url}/key-value-stores/{MOCKED_KVS_ID}' + assert public_url == f'{expected_base_url}/key-value-stores/{MOCKED_KVS_ID}/records/some_key' + + +@parametrized_versioned_api_urls +async def test_api_url_version_appended_once_async(api_url: str, expected_base_url: str) -> None: + """The `/v2` segment is appended to `api_url` and `api_public_url` only when missing, with async client.""" + client = ApifyClientAsync(token='dummy-token', api_url=api_url, api_public_url=api_url) + kvs = client.key_value_store(MOCKED_KVS_ID) + + with mock.patch.object(client._http_client, 'call', return_value=_get_mocked_kvs_response()) as call_mock: + public_url = await kvs.get_record_public_url(key='some_key') + + assert call_mock.call_args.kwargs['url'] == f'{expected_base_url}/key-value-stores/{MOCKED_KVS_ID}' + assert public_url == f'{expected_base_url}/key-value-stores/{MOCKED_KVS_ID}/records/some_key'