From 91c229ba8921a987ebadebd8d0272655250d2a05 Mon Sep 17 00:00:00 2001 From: Ignacio Van Droogenbroeck <64545348+xe-nvdk@users.noreply.github.com> Date: Mon, 2 Nov 2026 08:41:28 -0600 Subject: [PATCH 01/20] feat(telemetry): send an opt-out installation id for Arc's client telemetry Requests carry Arc-Client-Installation-Id (random UUID persisted under the user config dir, env override for containers, in-memory fallback when the dir is unwritable) and a User-Agent naming the Python version and platform. Arc relays distinct ids in its own opt-out telemetry, as it does for arcli; the SDK never contacts anything but the configured Arc server. Opt out with DO_NOT_TRACK=1, ARC_CLIENT_SEND_INSTALLATION_ID=0 or send_installation_id=False. Tests isolate the config dir so no test mints an id in the real home directory. --- CHANGELOG.md | 8 ++ README.md | 27 ++++ src/arc_client/_installation.py | 155 +++++++++++++++++++++ src/arc_client/async_client.py | 5 + src/arc_client/client.py | 5 + src/arc_client/config.py | 7 + src/arc_client/http/base.py | 11 +- tests/conftest.py | 14 +- tests/unit/http/test_headers_and_errors.py | 4 +- tests/unit/http/test_installation_id.py | 136 ++++++++++++++++++ 10 files changed, 367 insertions(+), 5 deletions(-) create mode 100644 src/arc_client/_installation.py create mode 100644 tests/unit/http/test_installation_id.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e49e90..a1a20b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,14 @@ plus bug fixes and the first real HTTP-layer test coverage. ### Added +- **Installation ID for Arc's client telemetry.** Requests carry a random + per-installation `Arc-Client-Installation-Id` and a `User-Agent` that names + the Python version and platform, so Arc's opt-out telemetry can count SDK + installations the same way it counts arcli. The SDK itself still talks only + to the configured Arc server. Opt out with `DO_NOT_TRACK=1`, + `ARC_CLIENT_SEND_INSTALLATION_ID=0` or `send_installation_id=False`; pin the + ID in containers with `ARC_CLIENT_INSTALLATION_ID`. See README, "Privacy". + - Arrow completion metadata in the existing `pyarrow.Table` schema for sync and async queries: policy cap and execution time are retained; failed, malformed, conflicting or unobservable announced completion metadata is rejected. diff --git a/README.md b/README.md index 862974b..5355452 100644 --- a/README.md +++ b/README.md @@ -509,6 +509,33 @@ except ArcConnectionError as e: print(f"Connection failed: {e}") ``` +## Privacy + +The SDK connects only to the Arc server you configure. It never contacts +Basekick or any other service on its own. + +Requests identify the SDK version, the Python version and the OS/architecture +in `User-Agent`. They also carry a random installation ID +(`Arc-Client-Installation-Id`), created on first use and stored in +`~/.config/arc-client-python/installation-id` (`%APPDATA%` on Windows). Arc may +include this ID in its own opt-out telemetry so Basekick can count SDK +installations across Arc servers. The ID is not derived from your machine, user +or token, and nothing about your queries, databases or data is included. + +Disable the ID with any of: + +```bash +DO_NOT_TRACK=1 +ARC_CLIENT_SEND_INSTALLATION_ID=0 +``` + +```python +ArcClient(host="localhost", token="...", send_installation_id=False) +``` + +In ephemeral containers, set `ARC_CLIENT_INSTALLATION_ID` to a fixed UUID so +each start is not counted as a new installation. + ## Documentation See the [full documentation](https://docs.basekick.net/arc/sdks/python]) for detailed guides and API reference. diff --git a/src/arc_client/_installation.py b/src/arc_client/_installation.py new file mode 100644 index 0000000..01b2655 --- /dev/null +++ b/src/arc_client/_installation.py @@ -0,0 +1,155 @@ +"""Client identity sent to Arc: the User-Agent and an installation id. + +Every request carries ``User-Agent: arc-client-python/ +(python/; /)``. Unless the caller opts out, it also +carries ``Arc-Client-Installation-Id``: a random UUID minted once per +installation. Arc includes the distinct ids it served in its own opt-out +telemetry so Basekick can count SDK installations across Arc servers. The +SDK itself never contacts anything but the Arc server it is configured for, +and the id is not derived from the machine, the user, or the token. + +Opt out with ``DO_NOT_TRACK=1``, ``ARC_CLIENT_SEND_INSTALLATION_ID=0``, or +``send_installation_id=False`` on the client. + +The id lives in ``$XDG_CONFIG_HOME/arc-client-python/installation-id`` +(``~/.config/...``; ``%APPDATA%`` on Windows). ``ARC_CLIENT_INSTALLATION_ID`` +overrides it, which suits containers that would otherwise mint a new id per +start. Failing to read or write the file never fails a request: the process +falls back to an id held in memory. +""" + +from __future__ import annotations + +import contextlib +import os +import platform +import re +import sys +import tempfile +import threading +import uuid +from pathlib import Path +from typing import Optional + +from arc_client import __version__ + +HEADER_INSTALLATION_ID = "Arc-Client-Installation-Id" + +ENV_INSTALLATION_ID = "ARC_CLIENT_INSTALLATION_ID" +ENV_SEND_INSTALLATION_ID = "ARC_CLIENT_SEND_INSTALLATION_ID" + +_FILE_NAME = "installation-id" +_DIR_NAME = "arc-client-python" + +_UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") +_TRUTHY = {"1", "true", "yes", "on"} +_FALSY = {"0", "false", "no", "off"} + +# Go's GOOS/GOARCH spelling, so SDK and arcli platforms read the same. +_ARCH = {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64"} + +_lock = threading.Lock() +_cached_id: Optional[str] = None + + +def _os_name() -> str: + if sys.platform.startswith("win"): + return "windows" + if sys.platform.startswith("linux"): + return "linux" + return sys.platform + + +def _arch() -> str: + machine = platform.machine().lower() + return _ARCH.get(machine, machine or "unknown") + + +def user_agent() -> str: + """``arc-client-python/0.2.0 (python/3.12.4; linux/amd64)``.""" + runtime = f"python/{platform.python_version()}" + return f"arc-client-python/{__version__} ({runtime}; {_os_name()}/{_arch()})" + + +def _env_flag(name: str) -> Optional[bool]: + value = os.environ.get(name, "").strip().lower() + if value in _TRUTHY: + return True + if value in _FALSY: + return False + return None + + +def sends_installation_id(enabled: bool = True) -> bool: + """Whether the id goes on requests: the client flag and both env opt-outs.""" + if not enabled or _env_flag("DO_NOT_TRACK") is True: + return False + return _env_flag(ENV_SEND_INSTALLATION_ID) is not False + + +def _config_dir() -> Path: + if sys.platform.startswith("win"): + base = os.environ.get("APPDATA") + if base: + return Path(base) / _DIR_NAME + xdg = os.environ.get("XDG_CONFIG_HOME") + base_path = Path(xdg) if xdg else Path.home() / ".config" + return base_path / _DIR_NAME + + +def _valid(value: str) -> Optional[str]: + value = value.strip().lower() + return value if _UUID_RE.match(value) else None + + +def _load_or_create() -> str: + path = _config_dir() / _FILE_NAME + try: + existing = _valid(path.read_text(encoding="ascii")) + if existing: + return existing + except (OSError, ValueError): + pass + new_id = str(uuid.uuid4()) + try: + path.parent.mkdir(parents=True, exist_ok=True) + # Write-then-rename so concurrent first runs never leave a torn file; + # the last writer wins and later reads agree with it. + fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=".installation-id.") + try: + with os.fdopen(fd, "w", encoding="ascii") as f: + f.write(new_id + "\n") + os.replace(tmp, path) + except BaseException: + with contextlib.suppress(OSError): + os.unlink(tmp) + raise + # Re-read: if another process won the rename, use its id. + return _valid(path.read_text(encoding="ascii")) or new_id + except (OSError, ValueError): + return new_id + + +def installation_id() -> str: + """The installation id, minted and persisted on first use.""" + override = _valid(os.environ.get(ENV_INSTALLATION_ID, "")) + if override: + return override + global _cached_id + with _lock: + if _cached_id is None: + _cached_id = _load_or_create() + return _cached_id + + +def outbound_installation_id(enabled: bool = True) -> Optional[str]: + """The id to put on a request, or None when the caller opted out.""" + if not sends_installation_id(enabled): + return None + return installation_id() + + +def _reset_for_tests() -> None: + global _cached_id + with _lock: + _cached_id = None diff --git a/src/arc_client/async_client.py b/src/arc_client/async_client.py index 969a421..38e9318 100644 --- a/src/arc_client/async_client.py +++ b/src/arc_client/async_client.py @@ -42,6 +42,7 @@ def __init__( verify_ssl: bool = True, max_retries: int = 3, retry_backoff: float = 0.5, + send_installation_id: bool = True, ) -> None: """Initialize the async Arc client. @@ -58,6 +59,9 @@ def __init__( max_retries: Max automatic retries for transient failures (connection errors, 429, 503). 0 disables retries. retry_backoff: Base backoff (seconds) for exponential retry. + send_installation_id: Send a random per-installation id so Arc's + opt-out telemetry can count SDK installations. Nothing about + requests or data is included. DO_NOT_TRACK=1 also disables it. """ self._config = ClientConfig( host=host, @@ -70,6 +74,7 @@ def __init__( verify_ssl=verify_ssl, max_retries=max_retries, retry_backoff=retry_backoff, + send_installation_id=send_installation_id, ) self._http: Optional[AsyncHTTPClient] = None diff --git a/src/arc_client/client.py b/src/arc_client/client.py index 9bc623f..c5d6a73 100644 --- a/src/arc_client/client.py +++ b/src/arc_client/client.py @@ -42,6 +42,7 @@ def __init__( verify_ssl: bool = True, max_retries: int = 3, retry_backoff: float = 0.5, + send_installation_id: bool = True, ) -> None: """Initialize the Arc client. @@ -58,6 +59,9 @@ def __init__( max_retries: Max automatic retries for transient failures (connection errors, 429, 503). 0 disables retries. retry_backoff: Base backoff (seconds) for exponential retry. + send_installation_id: Send a random per-installation id so Arc's + opt-out telemetry can count SDK installations. Nothing about + requests or data is included. DO_NOT_TRACK=1 also disables it. """ self._config = ClientConfig( host=host, @@ -70,6 +74,7 @@ def __init__( verify_ssl=verify_ssl, max_retries=max_retries, retry_backoff=retry_backoff, + send_installation_id=send_installation_id, ) self._http: Optional[SyncHTTPClient] = None diff --git a/src/arc_client/config.py b/src/arc_client/config.py index 0e4caae..89b62ec 100644 --- a/src/arc_client/config.py +++ b/src/arc_client/config.py @@ -36,6 +36,13 @@ class ClientConfig(BaseModel): description="Base backoff in seconds for exponential retry with jitter.", ) + send_installation_id: bool = Field( + default=True, + description="Send this installation's random id (Arc-Client-Installation-Id) so " + "Arc's telemetry can count SDK installations. DO_NOT_TRACK=1 or " + "ARC_CLIENT_SEND_INSTALLATION_ID=0 also disable it.", + ) + @field_validator("compression") @classmethod def _coerce_compression(cls, value: Union[Compression, str, bool]) -> Compression: diff --git a/src/arc_client/http/base.py b/src/arc_client/http/base.py index 1cb8a3f..0ab5326 100644 --- a/src/arc_client/http/base.py +++ b/src/arc_client/http/base.py @@ -8,7 +8,11 @@ import httpx -from arc_client import __version__ +from arc_client._installation import ( + HEADER_INSTALLATION_ID, + outbound_installation_id, + user_agent, +) from arc_client.config import ClientConfig from arc_client.exceptions import ( ArcAuthenticationError, @@ -24,7 +28,7 @@ ArcServerError, ) -USER_AGENT = f"arc-client-python/{__version__}" +USER_AGENT = user_agent() def build_headers( @@ -38,6 +42,9 @@ def build_headers( headers["Authorization"] = f"Bearer {config.token}" if config.database: headers["x-arc-database"] = config.database + installation_id = outbound_installation_id(config.send_installation_id) + if installation_id: + headers[HEADER_INSTALLATION_ID] = installation_id if extra_headers: headers.update(extra_headers) return headers diff --git a/tests/conftest.py b/tests/conftest.py index b8167b3..d635fdb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,7 +6,19 @@ import pytest -from arc_client import ArcClient, AsyncArcClient +from arc_client import ArcClient, AsyncArcClient, _installation + + +@pytest.fixture(autouse=True) +def _isolated_installation_id(tmp_path, monkeypatch): + """Keep tests from reading or minting the real installation id in $HOME.""" + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "xdg")) + monkeypatch.setenv("APPDATA", str(tmp_path / "appdata")) + for name in ("DO_NOT_TRACK", "ARC_CLIENT_SEND_INSTALLATION_ID", "ARC_CLIENT_INSTALLATION_ID"): + monkeypatch.delenv(name, raising=False) + _installation._reset_for_tests() + yield + _installation._reset_for_tests() @pytest.fixture diff --git a/tests/unit/http/test_headers_and_errors.py b/tests/unit/http/test_headers_and_errors.py index 3d266ae..5e8dfc9 100644 --- a/tests/unit/http/test_headers_and_errors.py +++ b/tests/unit/http/test_headers_and_errors.py @@ -35,13 +35,13 @@ def test_headers_include_auth_db_and_versioned_user_agent( assert request.headers["Authorization"] == "Bearer secret-token" assert request.headers["x-arc-database"] == "mydb" assert request.headers["User-Agent"] == USER_AGENT - assert USER_AGENT == "arc-client-python/0.2.0" + assert USER_AGENT.startswith("arc-client-python/0.2.0 (python/") def test_user_agent_matches_package_version(httpx_mock: HTTPXMock, client: ArcClient) -> None: import arc_client - assert f"arc-client-python/{arc_client.__version__}" == USER_AGENT + assert USER_AGENT.startswith(f"arc-client-python/{arc_client.__version__} ") @pytest.mark.parametrize( diff --git a/tests/unit/http/test_installation_id.py b/tests/unit/http/test_installation_id.py new file mode 100644 index 0000000..b591a9e --- /dev/null +++ b/tests/unit/http/test_installation_id.py @@ -0,0 +1,136 @@ +"""Installation id and User-Agent sent for Arc's client telemetry.""" + +from __future__ import annotations + +import platform +import re +import stat +import sys +from pathlib import Path + +import pytest +from pytest_httpx import HTTPXMock + +from arc_client import ArcClient, AsyncArcClient, _installation +from arc_client._installation import HEADER_INSTALLATION_ID + +UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$") + + +def _id_file(tmp_path: Path) -> Path: + if sys.platform.startswith("win"): + return tmp_path / "appdata" / "arc-client-python" / "installation-id" + return tmp_path / "xdg" / "arc-client-python" / "installation-id" + + +async def _request_headers(httpx_mock: HTTPXMock, async_mode: bool, **kwargs: object) -> dict: + httpx_mock.add_response(json={"status": "ok"}) + if async_mode: + async with AsyncArcClient(host="h", token="t", max_retries=0, **kwargs) as client: + await client.health() + else: + with ArcClient(host="h", token="t", max_retries=0, **kwargs) as client: + client.health() + return dict(httpx_mock.get_requests()[-1].headers) + + +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_sends_persisted_id(httpx_mock: HTTPXMock, async_mode: bool, tmp_path: Path) -> None: + headers = await _request_headers(httpx_mock, async_mode) + + sent = headers[HEADER_INSTALLATION_ID.lower()] + assert UUID_RE.match(sent) + assert _id_file(tmp_path).read_text().strip() == sent + + +async def test_id_is_stable_across_processes(httpx_mock: HTTPXMock, tmp_path: Path) -> None: + first = (await _request_headers(httpx_mock, False))[HEADER_INSTALLATION_ID.lower()] + _installation._reset_for_tests() # a new process reads the file instead of minting + second = (await _request_headers(httpx_mock, False))[HEADER_INSTALLATION_ID.lower()] + assert first == second + + +@pytest.mark.skipif(sys.platform.startswith("win"), reason="POSIX permissions") +def test_id_file_is_private(tmp_path: Path) -> None: + _installation.installation_id() + assert stat.S_IMODE(_id_file(tmp_path).stat().st_mode) == 0o600 + + +def test_corrupt_file_is_replaced(tmp_path: Path) -> None: + path = _id_file(tmp_path) + path.parent.mkdir(parents=True) + path.write_text("not-a-uuid\n") + new_id = _installation.installation_id() + assert UUID_RE.match(new_id) + assert path.read_text().strip() == new_id + + +def test_env_override_wins(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + fixed = "123e4567-e89b-42d3-a456-426614174000" + monkeypatch.setenv("ARC_CLIENT_INSTALLATION_ID", fixed.upper()) + assert _installation.installation_id() == fixed + assert not _id_file(tmp_path).exists() + + +def test_invalid_env_override_is_ignored(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("ARC_CLIENT_INSTALLATION_ID", "my-container") + assert UUID_RE.match(_installation.installation_id()) + + +def test_unwritable_config_dir_falls_back_to_memory( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + blocker = tmp_path / "file-not-dir" + blocker.write_text("") + monkeypatch.setenv("XDG_CONFIG_HOME", str(blocker)) + monkeypatch.setenv("APPDATA", str(blocker)) + + first = _installation.installation_id() + assert UUID_RE.match(first) + assert _installation.installation_id() == first # stable for the process + + +@pytest.mark.parametrize( + ("kwargs", "env"), + [ + ({"send_installation_id": False}, {}), + ({}, {"DO_NOT_TRACK": "1"}), + ({}, {"DO_NOT_TRACK": "true"}), + ({}, {"ARC_CLIENT_SEND_INSTALLATION_ID": "0"}), + ({}, {"ARC_CLIENT_SEND_INSTALLATION_ID": "false"}), + ], +) +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_opt_outs_send_nothing_and_write_nothing( + httpx_mock: HTTPXMock, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + async_mode: bool, + kwargs: dict, + env: dict, +) -> None: + for name, value in env.items(): + monkeypatch.setenv(name, value) + + headers = await _request_headers(httpx_mock, async_mode, **kwargs) + + assert HEADER_INSTALLATION_ID.lower() not in headers + assert not _id_file(tmp_path).exists() + assert headers["user-agent"].startswith("arc-client-python/") + + +def test_do_not_track_zero_does_not_opt_out() -> None: + assert _installation.sends_installation_id(True) + + +def test_user_agent_shape() -> None: + import arc_client + + ua = _installation.user_agent() + assert ua.startswith(f"arc-client-python/{arc_client.__version__} (python/") + assert f"python/{platform.python_version()};" in ua + m = re.match(r"^arc-client-python/\S+ \(python/(\S+); (\S+)/(\S+)\)$", ua) + assert m, ua + # Arc keeps only printable ASCII up to 32 chars per field. + for part in m.groups(): + assert 0 < len(part) <= 32 and part.isascii() and part.isprintable() From 97850a8c713024f1f425a1ac6209a4a389f40bf8 Mon Sep 17 00:00:00 2001 From: Ignacio Van Droogenbroeck <64545348+xe-nvdk@users.noreply.github.com> Date: Mon, 2 Nov 2026 08:43:53 -0600 Subject: [PATCH 02/20] feat(compaction): add cycles, too-wide and scoped trigger/history Add trigger(measurement=) (Arc 2026.09.2+), history(cycle_id=), cycles(), cycle(id), too_wide() and clear_too_wide() (Arc 2027.01.1+) to the sync and async compaction clients. Correct the module docstring: Arc made every compaction route admin-only, reads included. --- src/arc_client/management/async_compaction.py | 102 ++++++++++++-- src/arc_client/management/compaction.py | 100 ++++++++++++-- tests/unit/http/test_compaction_contracts.py | 125 ++++++++++++++++++ 3 files changed, 307 insertions(+), 20 deletions(-) create mode 100644 tests/unit/http/test_compaction_contracts.py diff --git a/src/arc_client/management/async_compaction.py b/src/arc_client/management/async_compaction.py index 040f642..192346a 100644 --- a/src/arc_client/management/async_compaction.py +++ b/src/arc_client/management/async_compaction.py @@ -1,7 +1,13 @@ """Asynchronous compaction-management client for Arc. -Wraps ``/api/v1/compaction``. Reads are available to any valid token; ``trigger`` -requires an admin token. +Wraps ``/api/v1/compaction``. Every route requires an admin token, including +the read-only ones: Arc made the reads admin-only because ``candidates``, +``jobs`` and ``history`` name the databases and measurements of every tenant +(Arc ``internal/api/compaction.go`` ``RegisterRoutes``). A non-admin token +raises :class:`~arc_client.exceptions.ArcPermissionError`. + +Responses are returned as raw dicts: they are operator diagnostics whose +fields Arc extends between releases, not a promised contract. """ from __future__ import annotations @@ -9,12 +15,17 @@ from typing import Any, Optional from arc_client.config import ClientConfig -from arc_client.exceptions import ArcError +from arc_client.exceptions import ArcError, ArcValidationError from arc_client.http.async_http import AsyncHTTPClient +def _check_positive(value: Optional[int], name: str) -> None: + if value is not None and (isinstance(value, bool) or not isinstance(value, int) or value < 1): + raise ArcValidationError(f"{name} must be a positive integer") + + class AsyncCompactionClient: - """Inspect and trigger compaction.""" + """Inspect and trigger compaction (admin).""" def __init__(self, http: AsyncHTTPClient, config: ClientConfig) -> None: self._http = http @@ -45,17 +56,88 @@ async def jobs(self) -> Any: """List active compaction jobs.""" return await self._json("get", "/api/v1/compaction/jobs") - async def history(self, limit: Optional[int] = None) -> Any: - """Get recent compaction jobs (default 10).""" + async def history(self, limit: Optional[int] = None, cycle_id: Optional[int] = None) -> Any: + """Get recent compaction job attempts (default 10). + + Args: + limit: Maximum records to return. Arc caps it at its history size. + cycle_id: Only the attempts of this compaction cycle (the id + :meth:`trigger` returns). Needs Arc 2027.01.1+; older servers + ignore it and return every recent job. + + Raises: + ArcValidationError: ``cycle_id`` is not a positive integer. + """ + _check_positive(cycle_id, "cycle_id") + params = {k: v for k, v in {"limit": limit, "cycle_id": cycle_id}.items() if v is not None} + return await self._json("get", "/api/v1/compaction/history", params=params or None) + + async def cycles(self, limit: Optional[int] = None) -> Any: + """List the retained compaction cycles, newest first. + + History is held in memory on the node that ran the cycles (the + compactor lease holder on a cluster) and is lost on restart. + Needs Arc 2027.01.1+. + """ params = {"limit": limit} if limit is not None else None - return await self._json("get", "/api/v1/compaction/history", params=params) + return await self._json("get", "/api/v1/compaction/cycles", params=params) - async def trigger(self, tier: Optional[str] = None, database: Optional[str] = None) -> Any: - """Trigger a compaction cycle (admin). + async def cycle(self, cycle_id: int) -> Any: + """Get one compaction cycle by id. Needs Arc 2027.01.1+. + + Raises: + ArcValidationError: ``cycle_id`` is not a positive integer. + ArcNotFoundError: This node retains no record of the cycle. Arc's + 404 body names the retained window and, on a cluster, the + compactor ``lease_holder`` where the cycle ran. + """ + _check_positive(cycle_id, "cycle_id") + return await self._json("get", f"/api/v1/compaction/cycles/{cycle_id}") + + async def too_wide(self) -> Any: + """List partitions this node remembers as too wide to compact. + + Compaction skips them until the settings change or they are cleared + with :meth:`clear_too_wide`. On a cluster, the compactor node's list + is the one cycles use. Needs Arc 2027.01.1+. + """ + return await self._json("get", "/api/v1/compaction/too-wide") + + async def clear_too_wide( + self, tier: Optional[str] = None, partition: Optional[str] = None + ) -> Any: + """Forget remembered too-wide partitions so the next cycle retries them. + + Clears every entry, or only those matching ``tier`` and/or + ``partition``. No data is deleted: this resets a skip list, so no + ``confirm`` is needed. Returns ``{"cleared": n}``. + Needs Arc 2027.01.1+. + """ + params = {k: v for k, v in {"tier": tier, "partition": partition}.items() if v} + return await self._json("delete", "/api/v1/compaction/too-wide", params=params or None) + + async def trigger( + self, + tier: Optional[str] = None, + database: Optional[str] = None, + measurement: Optional[str] = None, + ) -> Any: + """Trigger a compaction cycle. Args: tier: Comma-separated tiers (e.g. "hourly,daily"). Default all enabled. database: Restrict to a single database. + measurement: Restrict to one measurement of ``database``, which + must also be given. Needs Arc 2026.09.2+. + + Raises: + ArcValidationError: ``measurement`` was given without ``database``. """ - params = {k: v for k, v in {"tier": tier, "database": database}.items() if v} + if measurement and not database: + raise ArcValidationError("measurement requires database") + params = { + k: v + for k, v in {"tier": tier, "database": database, "measurement": measurement}.items() + if v + } return await self._json("post", "/api/v1/compaction/trigger", params=params or None) diff --git a/src/arc_client/management/compaction.py b/src/arc_client/management/compaction.py index 2add4bb..450fa2e 100644 --- a/src/arc_client/management/compaction.py +++ b/src/arc_client/management/compaction.py @@ -1,7 +1,13 @@ """Synchronous compaction-management client for Arc. -Wraps ``/api/v1/compaction``. Reads are available to any valid token; ``trigger`` -requires an admin token. +Wraps ``/api/v1/compaction``. Every route requires an admin token, including +the read-only ones: Arc made the reads admin-only because ``candidates``, +``jobs`` and ``history`` name the databases and measurements of every tenant +(Arc ``internal/api/compaction.go`` ``RegisterRoutes``). A non-admin token +raises :class:`~arc_client.exceptions.ArcPermissionError`. + +Responses are returned as raw dicts: they are operator diagnostics whose +fields Arc extends between releases, not a promised contract. """ from __future__ import annotations @@ -9,12 +15,17 @@ from typing import Any, Optional from arc_client.config import ClientConfig -from arc_client.exceptions import ArcError +from arc_client.exceptions import ArcError, ArcValidationError from arc_client.http.sync_http import SyncHTTPClient +def _check_positive(value: Optional[int], name: str) -> None: + if value is not None and (isinstance(value, bool) or not isinstance(value, int) or value < 1): + raise ArcValidationError(f"{name} must be a positive integer") + + class CompactionClient: - """Inspect and trigger compaction.""" + """Inspect and trigger compaction (admin).""" def __init__(self, http: SyncHTTPClient, config: ClientConfig) -> None: self._http = http @@ -45,17 +56,86 @@ def jobs(self) -> Any: """List active compaction jobs.""" return self._json("get", "/api/v1/compaction/jobs") - def history(self, limit: Optional[int] = None) -> Any: - """Get recent compaction jobs (default 10).""" + def history(self, limit: Optional[int] = None, cycle_id: Optional[int] = None) -> Any: + """Get recent compaction job attempts (default 10). + + Args: + limit: Maximum records to return. Arc caps it at its history size. + cycle_id: Only the attempts of this compaction cycle (the id + :meth:`trigger` returns). Needs Arc 2027.01.1+; older servers + ignore it and return every recent job. + + Raises: + ArcValidationError: ``cycle_id`` is not a positive integer. + """ + _check_positive(cycle_id, "cycle_id") + params = {k: v for k, v in {"limit": limit, "cycle_id": cycle_id}.items() if v is not None} + return self._json("get", "/api/v1/compaction/history", params=params or None) + + def cycles(self, limit: Optional[int] = None) -> Any: + """List the retained compaction cycles, newest first. + + History is held in memory on the node that ran the cycles (the + compactor lease holder on a cluster) and is lost on restart. + Needs Arc 2027.01.1+. + """ params = {"limit": limit} if limit is not None else None - return self._json("get", "/api/v1/compaction/history", params=params) + return self._json("get", "/api/v1/compaction/cycles", params=params) - def trigger(self, tier: Optional[str] = None, database: Optional[str] = None) -> Any: - """Trigger a compaction cycle (admin). + def cycle(self, cycle_id: int) -> Any: + """Get one compaction cycle by id. Needs Arc 2027.01.1+. + + Raises: + ArcValidationError: ``cycle_id`` is not a positive integer. + ArcNotFoundError: This node retains no record of the cycle. Arc's + 404 body names the retained window and, on a cluster, the + compactor ``lease_holder`` where the cycle ran. + """ + _check_positive(cycle_id, "cycle_id") + return self._json("get", f"/api/v1/compaction/cycles/{cycle_id}") + + def too_wide(self) -> Any: + """List partitions this node remembers as too wide to compact. + + Compaction skips them until the settings change or they are cleared + with :meth:`clear_too_wide`. On a cluster, the compactor node's list + is the one cycles use. Needs Arc 2027.01.1+. + """ + return self._json("get", "/api/v1/compaction/too-wide") + + def clear_too_wide(self, tier: Optional[str] = None, partition: Optional[str] = None) -> Any: + """Forget remembered too-wide partitions so the next cycle retries them. + + Clears every entry, or only those matching ``tier`` and/or + ``partition``. No data is deleted: this resets a skip list, so no + ``confirm`` is needed. Returns ``{"cleared": n}``. + Needs Arc 2027.01.1+. + """ + params = {k: v for k, v in {"tier": tier, "partition": partition}.items() if v} + return self._json("delete", "/api/v1/compaction/too-wide", params=params or None) + + def trigger( + self, + tier: Optional[str] = None, + database: Optional[str] = None, + measurement: Optional[str] = None, + ) -> Any: + """Trigger a compaction cycle. Args: tier: Comma-separated tiers (e.g. "hourly,daily"). Default all enabled. database: Restrict to a single database. + measurement: Restrict to one measurement of ``database``, which + must also be given. Needs Arc 2026.09.2+. + + Raises: + ArcValidationError: ``measurement`` was given without ``database``. """ - params = {k: v for k, v in {"tier": tier, "database": database}.items() if v} + if measurement and not database: + raise ArcValidationError("measurement requires database") + params = { + k: v + for k, v in {"tier": tier, "database": database, "measurement": measurement}.items() + if v + } return self._json("post", "/api/v1/compaction/trigger", params=params or None) diff --git a/tests/unit/http/test_compaction_contracts.py b/tests/unit/http/test_compaction_contracts.py new file mode 100644 index 0000000..9904bcc --- /dev/null +++ b/tests/unit/http/test_compaction_contracts.py @@ -0,0 +1,125 @@ +"""Compaction wire contracts shared by sync and async public clients.""" + +from __future__ import annotations + +import inspect +from contextlib import AsyncExitStack + +import pytest + +from arc_client import ArcClient, AsyncArcClient +from arc_client.exceptions import ArcNotFoundError, ArcPermissionError, ArcValidationError + +BASE = "http://localhost:8000/api/v1/compaction" + + +async def call(client, method, *args, **kwargs): + result = getattr(client.compaction, method)(*args, **kwargs) + return await result if inspect.isawaitable(result) else result + + +@pytest.fixture(params=[False, True], ids=["sync", "async"]) +def async_mode(request): + return request.param + + +async def enter_client(stack, async_mode): + if async_mode: + return await stack.enter_async_context(AsyncArcClient(token="t", max_retries=0)) + return stack.enter_context(ArcClient(token="t", max_retries=0)) + + +@pytest.mark.parametrize( + ("kwargs", "params"), + [ + ({}, {}), + ({"database": "db"}, {"database": "db"}), + ( + {"tier": "hourly", "database": "db", "measurement": "cpu"}, + {"tier": "hourly", "database": "db", "measurement": "cpu"}, + ), + ], +) +async def test_trigger_params(httpx_mock, async_mode, kwargs, params): + httpx_mock.add_response(method="POST", json={"message": "ok", "cycle_id": 3}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert (await call(client, "trigger", **kwargs))["cycle_id"] == 3 + request = httpx_mock.get_request() + assert request.url.path == "/api/v1/compaction/trigger" + assert dict(request.url.params) == params + + +async def test_trigger_measurement_requires_database(httpx_mock, async_mode): + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcValidationError): + await call(client, "trigger", measurement="cpu") + assert httpx_mock.get_requests() == [] + + +@pytest.mark.parametrize( + ("kwargs", "params"), + [({}, {}), ({"limit": 5}, {"limit": "5"}), ({"cycle_id": 7}, {"cycle_id": "7"})], +) +async def test_history_params(httpx_mock, async_mode, kwargs, params): + httpx_mock.add_response(method="GET", json={"recent_jobs": [], "total_jobs": 0}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + await call(client, "history", **kwargs) + assert dict(httpx_mock.get_request().url.params) == params + + +@pytest.mark.parametrize("bad", [0, -1, True]) +async def test_cycle_id_must_be_positive(httpx_mock, async_mode, bad): + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcValidationError): + await call(client, "history", cycle_id=bad) + with pytest.raises(ArcValidationError): + await call(client, "cycle", bad) + assert httpx_mock.get_requests() == [] + + +async def test_cycles_and_cycle(httpx_mock, async_mode): + httpx_mock.add_response(method="GET", url=f"{BASE}/cycles?limit=2", json={"cycles": []}) + httpx_mock.add_response(method="GET", url=f"{BASE}/cycles/4", json={"cycle_id": 4}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert (await call(client, "cycles", limit=2)) == {"cycles": []} + assert (await call(client, "cycle", 4))["cycle_id"] == 4 + + +async def test_cycle_not_found_keeps_diagnostics(httpx_mock, async_mode): + httpx_mock.add_response( + method="GET", + url=f"{BASE}/cycles/99", + status_code=404, + json={"error": "no retained record", "cycle_id": 99, "lease_holder": "n2"}, + ) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcNotFoundError): + await call(client, "cycle", 99) + + +async def test_too_wide_list_and_clear(httpx_mock, async_mode): + httpx_mock.add_response(method="GET", url=f"{BASE}/too-wide", json={"partitions": []}) + httpx_mock.add_response(method="DELETE", url=f"{BASE}/too-wide", json={"cleared": 3}) + httpx_mock.add_response( + method="DELETE", url=f"{BASE}/too-wide?tier=daily&partition=db%2Fcpu", json={"cleared": 1} + ) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert (await call(client, "too_wide"))["partitions"] == [] + assert (await call(client, "clear_too_wide"))["cleared"] == 3 + result = await call(client, "clear_too_wide", tier="daily", partition="db/cpu") + assert result["cleared"] == 1 + + +async def test_reads_are_admin_only(httpx_mock, async_mode): + httpx_mock.add_response(method="GET", status_code=403, json={"error": "admin required"}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcPermissionError): + await call(client, "cycles") From 1fff5185767097d52512f6cb77e569117410867b Mon Sep 17 00:00:00 2001 From: Ignacio Van Droogenbroeck <64545348+xe-nvdk@users.noreply.github.com> Date: Mon, 2 Nov 2026 08:44:09 -0600 Subject: [PATCH 03/20] feat(reconciliation): add status, trigger and run inspection (#17) Wrap /api/v1/reconciliation as client.reconciliation (sync and async). trigger() defaults to a dry run; act mode requires dry_run=False plus confirm=True and sends Arc's dry_run=false&act=true pair. Runs are returned as ReconciliationRun models; status() stays a raw dict. --- src/arc_client/async_client.py | 10 + src/arc_client/client.py | 10 + src/arc_client/management/__init__.py | 4 + .../management/async_reconciliation.py | 87 +++++++++ src/arc_client/management/reconciliation.py | 106 +++++++++++ src/arc_client/models/__init__.py | 2 + src/arc_client/models/reconciliation.py | 54 ++++++ .../http/test_reconciliation_contracts.py | 179 ++++++++++++++++++ 8 files changed, 452 insertions(+) create mode 100644 src/arc_client/management/async_reconciliation.py create mode 100644 src/arc_client/management/reconciliation.py create mode 100644 src/arc_client/models/reconciliation.py create mode 100644 tests/unit/http/test_reconciliation_contracts.py diff --git a/src/arc_client/async_client.py b/src/arc_client/async_client.py index 969a421..2dd36a1 100644 --- a/src/arc_client/async_client.py +++ b/src/arc_client/async_client.py @@ -91,6 +91,7 @@ def __init__( self._tiering: Any = None self._backup: Any = None self._mqtt: Any = None + self._reconciliation: Any = None def _get_http(self) -> AsyncHTTPClient: """Get or create the HTTP client.""" @@ -258,6 +259,15 @@ def mqtt(self) -> Any: self._mqtt = AsyncMQTTClient(self._get_http(), self._config) return self._mqtt + @property + def reconciliation(self) -> Any: + """Get the async manifest/storage reconciliation client (enterprise).""" + if self._reconciliation is None: + from arc_client.management.async_reconciliation import AsyncReconciliationClient + + self._reconciliation = AsyncReconciliationClient(self._get_http(), self._config) + return self._reconciliation + async def health(self) -> HealthResponse: """Check server health. diff --git a/src/arc_client/client.py b/src/arc_client/client.py index 9bc623f..acad506 100644 --- a/src/arc_client/client.py +++ b/src/arc_client/client.py @@ -91,6 +91,7 @@ def __init__( self._tiering: Any = None self._backup: Any = None self._mqtt: Any = None + self._reconciliation: Any = None def _get_http(self) -> SyncHTTPClient: """Get or create the HTTP client.""" @@ -256,6 +257,15 @@ def mqtt(self) -> Any: self._mqtt = MQTTClient(self._get_http(), self._config) return self._mqtt + @property + def reconciliation(self) -> Any: + """Get the manifest/storage reconciliation client (enterprise).""" + if self._reconciliation is None: + from arc_client.management.reconciliation import ReconciliationClient + + self._reconciliation = ReconciliationClient(self._get_http(), self._config) + return self._reconciliation + def health(self) -> HealthResponse: """Check server health. diff --git a/src/arc_client/management/__init__.py b/src/arc_client/management/__init__.py index f48e3af..2a09c3b 100644 --- a/src/arc_client/management/__init__.py +++ b/src/arc_client/management/__init__.py @@ -16,6 +16,7 @@ from arc_client.management.async_mqtt import AsyncMQTTClient from arc_client.management.async_query_management import AsyncQueryManagementClient from arc_client.management.async_rbac import AsyncRBACClient +from arc_client.management.async_reconciliation import AsyncReconciliationClient from arc_client.management.async_retention import AsyncRetentionClient from arc_client.management.async_tiering import AsyncTieringClient from arc_client.management.audit import AuditClient @@ -29,6 +30,7 @@ from arc_client.management.mqtt import MQTTClient from arc_client.management.query_management import QueryManagementClient from arc_client.management.rbac import RBACClient +from arc_client.management.reconciliation import ReconciliationClient from arc_client.management.retention import RetentionClient from arc_client.management.tiering import TieringClient @@ -61,4 +63,6 @@ "AsyncBackupClient", "MQTTClient", "AsyncMQTTClient", + "ReconciliationClient", + "AsyncReconciliationClient", ] diff --git a/src/arc_client/management/async_reconciliation.py b/src/arc_client/management/async_reconciliation.py new file mode 100644 index 0000000..c06317d --- /dev/null +++ b/src/arc_client/management/async_reconciliation.py @@ -0,0 +1,87 @@ +"""Asynchronous reconciliation client for Arc (enterprise clustering). + +Wraps ``/api/v1/reconciliation``. Reconciliation compares the cluster manifest +with what is in storage and, in act mode, deletes orphans on both sides. + +Gates (Arc ``internal/api/reconciliation.go``): + +* ``status`` — any token with read permission; not license-gated, so a + cluster without the license still reports ``{"enabled": false, ...}``. +* ``trigger`` and ``get_run`` — admin token plus the ``clustering`` license + feature (403 :class:`~arc_client.exceptions.ArcLicenseError` otherwise). + 503 when reconciliation is not running on the node. +""" + +from __future__ import annotations + +from typing import Any + +from arc_client.config import ClientConfig +from arc_client.exceptions import ArcError, ArcValidationError +from arc_client.http.async_http import AsyncHTTPClient +from arc_client.management.reconciliation import _run_from, _trigger_params +from arc_client.models.reconciliation import ReconciliationRun + + +class AsyncReconciliationClient: + """Inspect and trigger manifest/storage reconciliation (enterprise).""" + + def __init__(self, http: AsyncHTTPClient, config: ClientConfig) -> None: + self._http = http + self._config = config + + async def _json(self, method: str, path: str, **kwargs: Any) -> Any: + try: + response = await getattr(self._http, method)(path, **kwargs) + return response.json() + except ArcError: + raise + except Exception as e: + raise ArcError(f"Reconciliation request failed ({method.upper()} {path}): {e}") from e + + async def status(self) -> dict[str, Any]: + """Get the reconciliation scheduler status. + + Returned as a raw dict (operator diagnostics): ``enabled``, ``running``, + ``schedule``, ``backend_kind``, limits, ``in_flight``, and when + present ``next_run``, ``last_run`` and ``recent_runs``. When + reconciliation is unavailable on the node the server answers 200 with + ``{"enabled": false, "reason": ...}``. + """ + data = await self._json("get", "/api/v1/reconciliation/status") + if not isinstance(data, dict): + raise ArcError(f"Unexpected reconciliation status response: {data!r}") + return data + + async def trigger(self, dry_run: bool = True, confirm: bool = False) -> ReconciliationRun: + """Run one reconciliation cycle now and wait for its report (admin). + + Dry run (the default) only reports what would be deleted. Act mode + (``dry_run=False``) deletes orphans, bounded by the server's + ``max_deletes_per_run`` and grace window, and requires + ``confirm=True``. + + Raises: + ArcValidationError: ``dry_run=False`` without ``confirm=True`` + (raised before any request is sent). + ArcServerError: 409 when a run is already in progress, 422 when + the manifest is too large, 503 when reconciliation is + disabled. For 422/500 the partial run is kept in + ``error.response_body["run"]``. + ArcPermissionError: 403 when the node is role-gated. + ArcLicenseError: the ``clustering`` license feature is missing. + """ + params = _trigger_params(dry_run, confirm) + path = "/api/v1/reconciliation/trigger" + return _run_from(await self._json("post", path, params=params), path) + + async def get_run(self, run_id: str) -> ReconciliationRun: + """Get a recent run by id from the server's in-memory buffer (admin). + + Raises: + ArcNotFoundError: The run is not (or no longer) in the buffer. + """ + if not run_id: + raise ArcValidationError("run_id is required") + path = f"/api/v1/reconciliation/runs/{run_id}" + return _run_from(await self._json("get", path), path) diff --git a/src/arc_client/management/reconciliation.py b/src/arc_client/management/reconciliation.py new file mode 100644 index 0000000..3e8c8fd --- /dev/null +++ b/src/arc_client/management/reconciliation.py @@ -0,0 +1,106 @@ +"""Synchronous reconciliation client for Arc (enterprise clustering). + +Wraps ``/api/v1/reconciliation``. Reconciliation compares the cluster manifest +with what is in storage and, in act mode, deletes orphans on both sides. + +Gates (Arc ``internal/api/reconciliation.go``): + +* ``status`` — any token with read permission; not license-gated, so a + cluster without the license still reports ``{"enabled": false, ...}``. +* ``trigger`` and ``get_run`` — admin token plus the ``clustering`` license + feature (403 :class:`~arc_client.exceptions.ArcLicenseError` otherwise). + 503 when reconciliation is not running on the node. +""" + +from __future__ import annotations + +from typing import Any + +from arc_client.config import ClientConfig +from arc_client.exceptions import ArcError, ArcValidationError +from arc_client.http.sync_http import SyncHTTPClient +from arc_client.models.reconciliation import ReconciliationRun + + +def _trigger_params(dry_run: bool, confirm: bool) -> dict[str, str]: + if dry_run: + return {"dry_run": "true"} + if not confirm: + raise ArcValidationError( + "Refusing to run reconciliation in act mode without confirm=True. " + "Act mode deletes orphaned files and manifest entries cluster-wide." + ) + # Arc requires both flags for act mode. + return {"dry_run": "false", "act": "true"} + + +def _run_from(data: Any, path: str) -> ReconciliationRun: + if isinstance(data, dict) and isinstance(data.get("run"), dict): + data = data["run"] + if not isinstance(data, dict): + raise ArcError(f"Unexpected reconciliation response from {path}: {data!r}") + return ReconciliationRun.model_validate(data) + + +class ReconciliationClient: + """Inspect and trigger manifest/storage reconciliation (enterprise).""" + + def __init__(self, http: SyncHTTPClient, config: ClientConfig) -> None: + self._http = http + self._config = config + + def _json(self, method: str, path: str, **kwargs: Any) -> Any: + try: + response = getattr(self._http, method)(path, **kwargs) + return response.json() + except ArcError: + raise + except Exception as e: + raise ArcError(f"Reconciliation request failed ({method.upper()} {path}): {e}") from e + + def status(self) -> dict[str, Any]: + """Get the reconciliation scheduler status. + + Returned as a raw dict (operator diagnostics): ``enabled``, ``running``, + ``schedule``, ``backend_kind``, limits, ``in_flight``, and when + present ``next_run``, ``last_run`` and ``recent_runs``. When + reconciliation is unavailable on the node the server answers 200 with + ``{"enabled": false, "reason": ...}``. + """ + data = self._json("get", "/api/v1/reconciliation/status") + if not isinstance(data, dict): + raise ArcError(f"Unexpected reconciliation status response: {data!r}") + return data + + def trigger(self, dry_run: bool = True, confirm: bool = False) -> ReconciliationRun: + """Run one reconciliation cycle now and wait for its report (admin). + + Dry run (the default) only reports what would be deleted. Act mode + (``dry_run=False``) deletes orphans, bounded by the server's + ``max_deletes_per_run`` and grace window, and requires + ``confirm=True``. + + Raises: + ArcValidationError: ``dry_run=False`` without ``confirm=True`` + (raised before any request is sent). + ArcServerError: 409 when a run is already in progress, 422 when + the manifest is too large, 503 when reconciliation is + disabled. For 422/500 the partial run is kept in + ``error.response_body["run"]``. + ArcPermissionError: 403 when the node is role-gated. + ArcLicenseError: the ``clustering`` license feature is missing. + """ + params = _trigger_params(dry_run, confirm) + path = "/api/v1/reconciliation/trigger" + return _run_from(self._json("post", path, params=params), path) + + def get_run(self, run_id: str) -> ReconciliationRun: + """Get a recent run by id from the server's in-memory buffer (admin). + + Raises: + ArcNotFoundError: The run is not (or no longer) in the buffer. + """ + if not run_id: + raise ArcValidationError("run_id is required") + path = f"/api/v1/reconciliation/runs/{run_id}" + return _run_from(self._json("get", path), path) diff --git a/src/arc_client/models/__init__.py b/src/arc_client/models/__init__.py index bd10416..975c3ae 100644 --- a/src/arc_client/models/__init__.py +++ b/src/arc_client/models/__init__.py @@ -27,6 +27,7 @@ TLEImportResult, ) from arc_client.models.query import EstimateResponse, MeasurementInfo, QueryResponse +from arc_client.models.reconciliation import ReconciliationRun from arc_client.models.retention import ( ExecuteRetentionResponse, RetentionExecution, @@ -53,6 +54,7 @@ "HealthResponse", "MeasurementInfo", "QueryResponse", + "ReconciliationRun", "ReadyResponse", "RetentionExecution", "RetentionPolicy", diff --git a/src/arc_client/models/reconciliation.py b/src/arc_client/models/reconciliation.py new file mode 100644 index 0000000..27129b6 --- /dev/null +++ b/src/arc_client/models/reconciliation.py @@ -0,0 +1,54 @@ +"""Models for the reconciliation API (enterprise clustering).""" + +from __future__ import annotations + +from datetime import datetime +from typing import List, Optional + +from pydantic import BaseModel, ConfigDict, Field + + +class ReconciliationRun(BaseModel): + """One reconciliation run, as Arc reports it. + + Mirrors ``reconciliation.Run`` in Arc. Unknown fields are kept (``extra``) + so newer servers' counters are not lost. ``finished_at`` is Go's zero time + (year 1) for a run that has not finished. + """ + + model_config = ConfigDict(extra="allow") + + id: str + started_at: Optional[datetime] = None + finished_at: Optional[datetime] = None + dry_run: bool = True + backend_kind: Optional[str] = None + role: Optional[str] = None + + manifest_file_count: int = 0 + storage_file_count: int = 0 + orphan_manifest_count: int = 0 + orphan_storage_count: int = 0 + manifest_deletes: int = 0 + storage_deletes: int = 0 + skipped_grace: int = 0 + skipped_recheck: int = 0 + skipped_pending: int = 0 + skipped_held: int = 0 + skipped_pending_remote: int = 0 + skipped_adoptable: int = 0 + skipped_transient: int = 0 + skipped_invalid_path: int = 0 + + orphan_manifest_sample: List[str] = Field(default_factory=list) + orphan_storage_sample: List[str] = Field(default_factory=list) + errors: List[str] = Field(default_factory=list) + + aborted: bool = False + abort_reason: Optional[str] = None + abort_message: Optional[str] = None + manifest_sweep_held: bool = False + storage_sweep_held: bool = False + storage_sweep_held_reason: Optional[str] = None + cap_hit: bool = False + walk_partial: bool = False diff --git a/tests/unit/http/test_reconciliation_contracts.py b/tests/unit/http/test_reconciliation_contracts.py new file mode 100644 index 0000000..cfefa42 --- /dev/null +++ b/tests/unit/http/test_reconciliation_contracts.py @@ -0,0 +1,179 @@ +"""Reconciliation wire contracts (Arc internal/api/reconciliation.go).""" + +from __future__ import annotations + +import inspect +from contextlib import AsyncExitStack +from typing import Any + +import pytest +from pytest_httpx import HTTPXMock + +from arc_client import ArcClient, AsyncArcClient +from arc_client.exceptions import ( + ArcLicenseError, + ArcNotFoundError, + ArcPermissionError, + ArcServerError, + ArcValidationError, +) +from arc_client.models.reconciliation import ReconciliationRun + +BASE = "http://localhost:8000/api/v1/reconciliation" +RUN = { + "id": "run-1", + "started_at": "2027-01-05T10:00:00Z", + "finished_at": "0001-01-01T00:00:00Z", + "dry_run": True, + "backend_kind": "s3", + "role": "compactor", + "manifest_file_count": 10, + "storage_file_count": 12, + "orphan_manifest_count": 0, + "orphan_storage_count": 2, + "manifest_deletes": 0, + "storage_deletes": 0, + "skipped_grace": 1, + "skipped_recheck": 0, + "skipped_pending": 0, + "skipped_transient": 0, + "skipped_invalid_path": 0, + "orphan_storage_sample": ["db/cpu/a.parquet"], + "aborted": False, + "manifest_sweep_held": False, + "cap_hit": False, + "walk_partial": False, + "future_counter": 3, +} + + +async def call(client: Any, method: str, *args: Any, **kwargs: Any) -> Any: + result = getattr(client.reconciliation, method)(*args, **kwargs) + return await result if inspect.isawaitable(result) else result + + +@pytest.fixture(params=[False, True], ids=["sync", "async"]) +def async_mode(request: Any) -> bool: + return bool(request.param) + + +async def enter_client(stack: AsyncExitStack, async_mode: bool) -> Any: + if async_mode: + return await stack.enter_async_context(AsyncArcClient(token="t", max_retries=0)) + return stack.enter_context(ArcClient(token="t", max_retries=0)) + + +async def test_status_returns_raw_dict(httpx_mock: HTTPXMock, async_mode: bool) -> None: + body = {"enabled": True, "running": True, "schedule": "0 */6 * * *", "last_run": RUN} + httpx_mock.add_response(method="GET", url=f"{BASE}/status", json=body) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert await call(client, "status") == body + + +async def test_status_disabled_is_not_an_error(httpx_mock: HTTPXMock, async_mode: bool) -> None: + body = {"enabled": False, "reason": "Reconciliation requires Enterprise clustering"} + httpx_mock.add_response(method="GET", url=f"{BASE}/status", json=body) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert (await call(client, "status"))["enabled"] is False + + +async def test_trigger_defaults_to_dry_run(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(method="POST", url=f"{BASE}/trigger?dry_run=true", json={"run": RUN}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + run = await call(client, "trigger") + assert isinstance(run, ReconciliationRun) + assert run.id == "run-1" + assert run.dry_run is True + assert run.orphan_storage_sample == ["db/cpu/a.parquet"] + assert run.finished_at is not None and run.finished_at.year == 1 + assert run.model_extra == {"future_counter": 3} + + +async def test_act_mode_requires_confirm(httpx_mock: HTTPXMock, async_mode: bool) -> None: + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcValidationError): + await call(client, "trigger", dry_run=False) + assert httpx_mock.get_requests() == [] + + +async def test_act_mode_sends_both_flags(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response( + method="POST", + url=f"{BASE}/trigger?dry_run=false&act=true", + json={"run": {**RUN, "dry_run": False, "storage_deletes": 2}}, + ) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + run = await call(client, "trigger", dry_run=False, confirm=True) + assert run.dry_run is False + assert run.storage_deletes == 2 + + +async def test_confirm_alone_stays_dry_run(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(method="POST", url=f"{BASE}/trigger?dry_run=true", json={"run": RUN}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert (await call(client, "trigger", confirm=True)).dry_run is True + + +@pytest.mark.parametrize( + ("status", "body", "exc"), + [ + (409, {"error": "a reconciliation run is already in progress"}, ArcServerError), + (422, {"error": "manifest too large", "run": RUN}, ArcServerError), + (503, {"error": "reconciliation feature is disabled"}, ArcServerError), + (403, {"error": "this node is not permitted to run reconciliation"}, ArcPermissionError), + ( + 403, + { + "error": "Reconciliation requires an enterprise license with the " + "'clustering' feature enabled" + }, + ArcLicenseError, + ), + ], +) +async def test_trigger_errors_keep_diagnostics( + httpx_mock: HTTPXMock, async_mode: bool, status: int, body: dict[str, Any], exc: type +) -> None: + httpx_mock.add_response(method="POST", status_code=status, json=body) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(exc) as info: + await call(client, "trigger") + assert info.value.response_body == body + if isinstance(info.value, ArcServerError): + assert info.value.status_code == status + + +async def test_get_run(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(method="GET", url=f"{BASE}/runs/run-1", json=RUN) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + run = await call(client, "get_run", "run-1") + assert run.id == "run-1" + assert run.storage_file_count == 12 + + +async def test_get_run_not_found(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response( + method="GET", + url=f"{BASE}/runs/gone", + status_code=404, + json={"error": "run not found in recent-runs buffer"}, + ) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcNotFoundError): + await call(client, "get_run", "gone") + + +async def test_get_run_requires_id(httpx_mock: HTTPXMock, async_mode: bool) -> None: + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcValidationError): + await call(client, "get_run", "") From f1c324676f2d52f64598c9c60e73b4a3a5f2663a Mon Sep 17 00:00:00 2001 From: Ignacio Van Droogenbroeck <64545348+xe-nvdk@users.noreply.github.com> Date: Mon, 2 Nov 2026 08:44:35 -0600 Subject: [PATCH 04/20] feat(auth): expose token-cache stats and invalidation (#15) Add AuthClient.cache_stats() and invalidate_cache() (sync and async) for the admin-only /api/v1/auth/cache routes. Stats are returned as a raw dict; invalidation loses no data, so it takes no confirm flag. --- src/arc_client/auth/async_manager.py | 39 +++++++++- src/arc_client/auth/manager.py | 39 +++++++++- tests/unit/http/test_auth_cache_contracts.py | 77 ++++++++++++++++++++ 3 files changed, 153 insertions(+), 2 deletions(-) create mode 100644 tests/unit/http/test_auth_cache_contracts.py diff --git a/src/arc_client/auth/async_manager.py b/src/arc_client/auth/async_manager.py index 87958f8..15f6188 100644 --- a/src/arc_client/auth/async_manager.py +++ b/src/arc_client/auth/async_manager.py @@ -5,7 +5,7 @@ from typing import Any, List, Optional from arc_client.config import ClientConfig -from arc_client.exceptions import ArcAuthenticationError, ArcNotFoundError +from arc_client.exceptions import ArcAuthenticationError, ArcError, ArcNotFoundError from arc_client.http.async_http import AsyncHTTPClient from arc_client.models.auth import ( CreateTokenResponse, @@ -172,3 +172,40 @@ async def revoke_token(self, token_id: int) -> None: raise except Exception as e: raise ArcAuthenticationError(f"Failed to revoke token: {e}") from e + + async def cache_stats(self) -> dict[str, Any]: + """Get the server's token-validation cache statistics (admin). + + Returns the inner ``stats`` object as a raw dict (operator + diagnostics): ``cache_size``, ``max_cache_size``, + ``cache_ttl_seconds``, ``utilization_percent``, ``total_requests``, + ``cache_hits``, ``cache_misses``, ``cache_evictions`` and + ``hit_rate_percent``. The cache is per node. + """ + data = await self._admin_json("get", "/api/v1/auth/cache/stats") + stats = data.get("stats") + if not isinstance(stats, dict): + raise ArcError(f"Unexpected auth cache stats response: {data!r}") + return stats + + async def invalidate_cache(self) -> dict[str, Any]: + """Clear the server's token-validation cache on this node (admin). + + Loses no data and needs no ``confirm``: the next request for each + token is validated against the token store again, which costs one + lookup per token. Use it to make a token change take effect before + the cache TTL expires. Returns ``{"success": true, "message": ...}``. + """ + return await self._admin_json("post", "/api/v1/auth/cache/invalidate") + + async def _admin_json(self, method: str, path: str) -> dict[str, Any]: + try: + response = await getattr(self._http, method)(path) + data = response.json() + except ArcError: + raise + except Exception as e: + raise ArcError(f"Auth cache request failed ({method.upper()} {path}): {e}") from e + if not isinstance(data, dict): + raise ArcError(f"Unexpected response from {path}: {data!r}") + return data diff --git a/src/arc_client/auth/manager.py b/src/arc_client/auth/manager.py index 2c5861b..209ab5c 100644 --- a/src/arc_client/auth/manager.py +++ b/src/arc_client/auth/manager.py @@ -5,7 +5,7 @@ from typing import Any, List, Optional from arc_client.config import ClientConfig -from arc_client.exceptions import ArcAuthenticationError, ArcNotFoundError +from arc_client.exceptions import ArcAuthenticationError, ArcError, ArcNotFoundError from arc_client.http.sync_http import SyncHTTPClient from arc_client.models.auth import ( CreateTokenResponse, @@ -184,3 +184,40 @@ def revoke_token(self, token_id: int) -> None: raise except Exception as e: raise ArcAuthenticationError(f"Failed to revoke token: {e}") from e + + def cache_stats(self) -> dict[str, Any]: + """Get the server's token-validation cache statistics (admin). + + Returns the inner ``stats`` object as a raw dict (operator + diagnostics): ``cache_size``, ``max_cache_size``, + ``cache_ttl_seconds``, ``utilization_percent``, ``total_requests``, + ``cache_hits``, ``cache_misses``, ``cache_evictions`` and + ``hit_rate_percent``. The cache is per node. + """ + data = self._admin_json("get", "/api/v1/auth/cache/stats") + stats = data.get("stats") + if not isinstance(stats, dict): + raise ArcError(f"Unexpected auth cache stats response: {data!r}") + return stats + + def invalidate_cache(self) -> dict[str, Any]: + """Clear the server's token-validation cache on this node (admin). + + Loses no data and needs no ``confirm``: the next request for each + token is validated against the token store again, which costs one + lookup per token. Use it to make a token change take effect before + the cache TTL expires. Returns ``{"success": true, "message": ...}``. + """ + return self._admin_json("post", "/api/v1/auth/cache/invalidate") + + def _admin_json(self, method: str, path: str) -> dict[str, Any]: + try: + response = getattr(self._http, method)(path) + data = response.json() + except ArcError: + raise + except Exception as e: + raise ArcError(f"Auth cache request failed ({method.upper()} {path}): {e}") from e + if not isinstance(data, dict): + raise ArcError(f"Unexpected response from {path}: {data!r}") + return data diff --git a/tests/unit/http/test_auth_cache_contracts.py b/tests/unit/http/test_auth_cache_contracts.py new file mode 100644 index 0000000..b8549ee --- /dev/null +++ b/tests/unit/http/test_auth_cache_contracts.py @@ -0,0 +1,77 @@ +"""Auth token-cache wire contracts (Arc internal/api/auth_routes.go).""" + +from __future__ import annotations + +import inspect +from contextlib import AsyncExitStack +from typing import Any + +import pytest +from pytest_httpx import HTTPXMock + +from arc_client import ArcClient, AsyncArcClient +from arc_client.exceptions import ArcError, ArcPermissionError + +BASE = "http://localhost:8000/api/v1/auth/cache" +STATS = { + "cache_size": 3, + "max_cache_size": 1000, + "cache_ttl_seconds": 30.0, + "utilization_percent": 0.3, + "total_requests": 10, + "cache_hits": 7, + "cache_misses": 3, + "cache_evictions": 0, + "hit_rate_percent": 70.0, +} + + +async def call(client: Any, method: str, *args: Any, **kwargs: Any) -> Any: + result = getattr(client.auth, method)(*args, **kwargs) + return await result if inspect.isawaitable(result) else result + + +@pytest.fixture(params=[False, True], ids=["sync", "async"]) +def async_mode(request: Any) -> bool: + return bool(request.param) + + +async def enter_client(stack: AsyncExitStack, async_mode: bool) -> Any: + if async_mode: + return await stack.enter_async_context(AsyncArcClient(token="t", max_retries=0)) + return stack.enter_context(ArcClient(token="t", max_retries=0)) + + +async def test_cache_stats_unwraps_stats(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response( + method="GET", url=f"{BASE}/stats", json={"success": True, "stats": STATS} + ) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert await call(client, "cache_stats") == STATS + + +async def test_cache_stats_malformed(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(method="GET", url=f"{BASE}/stats", json={"success": True}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcError): + await call(client, "cache_stats") + + +async def test_invalidate_cache(httpx_mock: HTTPXMock, async_mode: bool) -> None: + body = {"success": True, "message": "Cache invalidated successfully"} + httpx_mock.add_response(method="POST", url=f"{BASE}/invalidate", json=body) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert await call(client, "invalidate_cache") == body + assert httpx_mock.get_request().content == b"" + + +@pytest.mark.parametrize("method", ["cache_stats", "invalidate_cache"]) +async def test_non_admin_refused(httpx_mock: HTTPXMock, async_mode: bool, method: str) -> None: + httpx_mock.add_response(status_code=403, json={"error": "Admin permission required"}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcPermissionError): + await call(client, method) From d7f69637d82391e0f5bc18dde5ecd0ef4c175237 Mon Sep 17 00:00:00 2001 From: Ignacio Van Droogenbroeck <64545348+xe-nvdk@users.noreply.github.com> Date: Mon, 2 Nov 2026 08:44:37 -0600 Subject: [PATCH 05/20] feat(backup): scope backups, list foreign backups and pick restore mode create(databases=) scopes a backup, list(include_foreign=) includes backups another instance wrote, and restore(mode=) selects merge or replace. All three need Arc 2027.01.1+ and are omitted unless set, so older servers behave as before. Restore still requires confirm=True; mode is validated client-side and left to the server's effective default when omitted. --- src/arc_client/management/async_backup.py | 57 ++++++++++-- src/arc_client/management/backup.py | 82 +++++++++++++++-- tests/unit/http/test_backup_contracts.py | 103 ++++++++++++++++++++++ 3 files changed, 228 insertions(+), 14 deletions(-) create mode 100644 tests/unit/http/test_backup_contracts.py diff --git a/src/arc_client/management/async_backup.py b/src/arc_client/management/async_backup.py index 16bb797..2eaf458 100644 --- a/src/arc_client/management/async_backup.py +++ b/src/arc_client/management/async_backup.py @@ -1,16 +1,18 @@ """Asynchronous backup/restore client for Arc (enterprise). Wraps ``/api/v1/backup``. Requires an admin token. Backup/restore run -asynchronously server-side; poll :meth:`status` for progress. +asynchronously server-side; poll :meth:`status` for progress. Responses are +returned as raw dicts (operator diagnostics, not a promised contract). """ from __future__ import annotations -from typing import Any, Optional +from typing import Any, Optional, Sequence from arc_client.config import ClientConfig from arc_client.exceptions import ArcError, ArcValidationError from arc_client.http.async_http import AsyncHTTPClient +from arc_client.management.backup import _restore_mode, _scope class AsyncBackupClient: @@ -33,18 +35,45 @@ async def create( self, include_metadata: Optional[bool] = None, include_config: Optional[bool] = None, + databases: Optional[Sequence[str]] = None, ) -> Any: - """Start a backup (async). Returns 202 with a running status.""" + """Start a backup (async). Returns 202 with a running status. + + Args: + include_metadata: Include the SQLite metadata database. Default + true for a whole-instance backup; a scoped backup cannot + carry it (Arc answers 400 to ``True`` with ``databases``). + include_config: Include ``arc.toml``. Default true, except false + for a scoped backup or a remote backup target. + databases: Scope the backup to these databases (at most 256). + Omit for a whole-instance backup. Needs Arc 2027.01.1+; an + older server ignores the field and backs up everything. + + Raises: + ArcValidationError: ``databases`` is empty or holds an empty name. + """ + scope = _scope(databases) body: dict[str, Any] = {} if include_metadata is not None: body["include_metadata"] = include_metadata if include_config is not None: body["include_config"] = include_config + if scope is not None: + body["databases"] = scope return await self._json("post", "/api/v1/backup/", json=body or None) - async def list(self) -> Any: - """List available backups.""" - return await self._json("get", "/api/v1/backup/") + async def list(self, include_foreign: Optional[bool] = None) -> Any: + """List available backups. + + Args: + include_foreign: Also list backups another Arc instance wrote to + the same destination, each marked ``foreign_owner``. Needed to + find a backup after restoring onto fresh hardware, where every + backup reads as foreign; the response's ``filtered_foreign`` + counts the ones left out by default. Needs Arc 2027.01.1+. + """ + params = {"include_foreign": "true"} if include_foreign else None + return await self._json("get", "/api/v1/backup/", params=params) async def status(self) -> Any: """Get the current backup/restore progress (or idle).""" @@ -65,16 +94,28 @@ async def restore( restore_data: Optional[bool] = None, restore_metadata: Optional[bool] = None, restore_config: Optional[bool] = None, + mode: Optional[str] = None, ) -> Any: """Restore from a backup (async). Requires confirm=True. + Args: + mode: ``"merge"`` adds the backup's files to the current ones + (files deleted since the backup come back). ``"replace"`` + first removes the current files of each restored database + through the cluster manifest; cluster nodes only (a standalone + server answers 400). Omitted, Arc picks: merge, except that a + scoped backup restored on a cluster node defaults to replace. + The response echoes the effective mode. Needs Arc 2027.01.1+. + Raises: - ArcValidationError: If confirm is not True (restore is destructive). + ArcValidationError: If confirm is not True (restore is destructive) + or ``mode`` is not ``"merge"`` or ``"replace"``. """ if not confirm: raise ArcValidationError( "Refusing to restore without confirm=True. This overwrites data." ) + restore_mode = _restore_mode(mode) body: dict[str, Any] = {"backup_id": backup_id, "confirm": True} if restore_data is not None: body["restore_data"] = restore_data @@ -82,4 +123,6 @@ async def restore( body["restore_metadata"] = restore_metadata if restore_config is not None: body["restore_config"] = restore_config + if restore_mode is not None: + body["mode"] = restore_mode return await self._json("post", "/api/v1/backup/restore", json=body) diff --git a/src/arc_client/management/backup.py b/src/arc_client/management/backup.py index 119cd00..f1ddf9c 100644 --- a/src/arc_client/management/backup.py +++ b/src/arc_client/management/backup.py @@ -1,17 +1,44 @@ """Synchronous backup/restore client for Arc (enterprise). Wraps ``/api/v1/backup``. Requires an admin token. Backup/restore run -asynchronously server-side; poll :meth:`status` for progress. +asynchronously server-side; poll :meth:`status` for progress. Responses are +returned as raw dicts (operator diagnostics, not a promised contract). """ from __future__ import annotations -from typing import Any, Optional +from typing import Any, Optional, Sequence from arc_client.config import ClientConfig from arc_client.exceptions import ArcError, ArcValidationError from arc_client.http.sync_http import SyncHTTPClient +RESTORE_MODES = ("merge", "replace") + + +def _scope(databases: Optional[Sequence[str]]) -> Optional[list[str]]: + if databases is None: + return None + if isinstance(databases, str): + raise ArcValidationError("databases must be a list of names, not a string") + names = list(databases) + if not names: + # Arc reads an empty list as a whole-instance backup; refuse it rather + # than silently widen a scope the caller meant to narrow. + raise ArcValidationError("databases must name at least one database (omit it for all)") + if any(not isinstance(n, str) or not n for n in names): + raise ArcValidationError("databases must contain non-empty names") + return names + + +def _restore_mode(mode: Optional[str]) -> Optional[str]: + if mode is None: + return None + normalized = mode.strip().lower() + if normalized not in RESTORE_MODES: + raise ArcValidationError(f"mode must be one of {RESTORE_MODES}, got {mode!r}") + return normalized + class BackupClient: """Create, list, inspect, delete backups and restore from them (enterprise).""" @@ -33,18 +60,45 @@ def create( self, include_metadata: Optional[bool] = None, include_config: Optional[bool] = None, + databases: Optional[Sequence[str]] = None, ) -> Any: - """Start a backup (async). Returns 202 with a running status.""" + """Start a backup (async). Returns 202 with a running status. + + Args: + include_metadata: Include the SQLite metadata database. Default + true for a whole-instance backup; a scoped backup cannot + carry it (Arc answers 400 to ``True`` with ``databases``). + include_config: Include ``arc.toml``. Default true, except false + for a scoped backup or a remote backup target. + databases: Scope the backup to these databases (at most 256). + Omit for a whole-instance backup. Needs Arc 2027.01.1+; an + older server ignores the field and backs up everything. + + Raises: + ArcValidationError: ``databases`` is empty or holds an empty name. + """ + scope = _scope(databases) body: dict[str, Any] = {} if include_metadata is not None: body["include_metadata"] = include_metadata if include_config is not None: body["include_config"] = include_config + if scope is not None: + body["databases"] = scope return self._json("post", "/api/v1/backup/", json=body or None) - def list(self) -> Any: - """List available backups.""" - return self._json("get", "/api/v1/backup/") + def list(self, include_foreign: Optional[bool] = None) -> Any: + """List available backups. + + Args: + include_foreign: Also list backups another Arc instance wrote to + the same destination, each marked ``foreign_owner``. Needed to + find a backup after restoring onto fresh hardware, where every + backup reads as foreign; the response's ``filtered_foreign`` + counts the ones left out by default. Needs Arc 2027.01.1+. + """ + params = {"include_foreign": "true"} if include_foreign else None + return self._json("get", "/api/v1/backup/", params=params) def status(self) -> Any: """Get the current backup/restore progress (or idle).""" @@ -65,16 +119,28 @@ def restore( restore_data: Optional[bool] = None, restore_metadata: Optional[bool] = None, restore_config: Optional[bool] = None, + mode: Optional[str] = None, ) -> Any: """Restore from a backup (async). Requires confirm=True. + Args: + mode: ``"merge"`` adds the backup's files to the current ones + (files deleted since the backup come back). ``"replace"`` + first removes the current files of each restored database + through the cluster manifest; cluster nodes only (a standalone + server answers 400). Omitted, Arc picks: merge, except that a + scoped backup restored on a cluster node defaults to replace. + The response echoes the effective mode. Needs Arc 2027.01.1+. + Raises: - ArcValidationError: If confirm is not True (restore is destructive). + ArcValidationError: If confirm is not True (restore is destructive) + or ``mode`` is not ``"merge"`` or ``"replace"``. """ if not confirm: raise ArcValidationError( "Refusing to restore without confirm=True. This overwrites data." ) + restore_mode = _restore_mode(mode) body: dict[str, Any] = {"backup_id": backup_id, "confirm": True} if restore_data is not None: body["restore_data"] = restore_data @@ -82,4 +148,6 @@ def restore( body["restore_metadata"] = restore_metadata if restore_config is not None: body["restore_config"] = restore_config + if restore_mode is not None: + body["mode"] = restore_mode return self._json("post", "/api/v1/backup/restore", json=body) diff --git a/tests/unit/http/test_backup_contracts.py b/tests/unit/http/test_backup_contracts.py new file mode 100644 index 0000000..a1ccac4 --- /dev/null +++ b/tests/unit/http/test_backup_contracts.py @@ -0,0 +1,103 @@ +"""Backup wire contracts shared by sync and async public clients.""" + +from __future__ import annotations + +import inspect +import json +from contextlib import AsyncExitStack + +import pytest + +from arc_client import ArcClient, AsyncArcClient +from arc_client.exceptions import ArcValidationError + + +async def call(client, method, *args, **kwargs): + result = getattr(client.backup, method)(*args, **kwargs) + return await result if inspect.isawaitable(result) else result + + +@pytest.fixture(params=[False, True], ids=["sync", "async"]) +def async_mode(request): + return request.param + + +async def enter_client(stack, async_mode): + if async_mode: + return await stack.enter_async_context(AsyncArcClient(token="t", max_retries=0)) + return stack.enter_context(ArcClient(token="t", max_retries=0)) + + +def body_of(request): + return json.loads(request.content) if request.content else None + + +@pytest.mark.parametrize( + ("kwargs", "body"), + [ + ({}, None), + ({"databases": ["b", "a"]}, {"databases": ["b", "a"]}), + ( + {"databases": ("a",), "include_config": False}, + {"include_config": False, "databases": ["a"]}, + ), + ], +) +async def test_create_scope(httpx_mock, async_mode, kwargs, body): + httpx_mock.add_response(method="POST", status_code=202, json={"status": "running"}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + await call(client, "create", **kwargs) + request = httpx_mock.get_request() + assert request.url.path == "/api/v1/backup/" + assert body_of(request) == body + + +@pytest.mark.parametrize("bad", [[], "metrics", [""]]) +async def test_create_rejects_bad_scope(httpx_mock, async_mode, bad): + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcValidationError): + await call(client, "create", databases=bad) + assert httpx_mock.get_requests() == [] + + +@pytest.mark.parametrize( + ("kwargs", "params"), + [ + ({}, {}), + ({"include_foreign": False}, {}), + ({"include_foreign": True}, {"include_foreign": "true"}), + ], +) +async def test_list_include_foreign(httpx_mock, async_mode, kwargs, params): + httpx_mock.add_response(method="GET", json={"backups": [], "count": 0, "filtered_foreign": 2}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + result = await call(client, "list", **kwargs) + assert result["filtered_foreign"] == 2 + assert dict(httpx_mock.get_request().url.params) == params + + +@pytest.mark.parametrize( + ("mode", "sent"), [(None, None), ("merge", "merge"), (" Replace ", "replace")] +) +async def test_restore_mode(httpx_mock, async_mode, mode, sent): + httpx_mock.add_response(method="POST", status_code=202, json={"mode": sent or "merge"}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + await call(client, "restore", "b1", confirm=True, mode=mode) + expected = {"backup_id": "b1", "confirm": True} + if sent: + expected["mode"] = sent + assert body_of(httpx_mock.get_request()) == expected + + +async def test_restore_rejects_unknown_mode_and_missing_confirm(httpx_mock, async_mode): + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcValidationError): + await call(client, "restore", "b1", confirm=True, mode="overwrite") + with pytest.raises(ArcValidationError): + await call(client, "restore", "b1", mode="merge") + assert httpx_mock.get_requests() == [] From bb0c38e3e364f528c11cdc2d614acf41b81cac52 Mon Sep 17 00:00:00 2001 From: Ignacio Van Droogenbroeck <64545348+xe-nvdk@users.noreply.github.com> Date: Mon, 2 Nov 2026 08:45:59 -0600 Subject: [PATCH 06/20] feat(delete): add allow_unreachable to row and database deletes delete.delete(allow_unreachable=True) sends the body field and databases.delete(allow_unreachable=True) sends ?allow_unreachable=true (Arc 2027.01.1+). Both are omitted by default. Docstrings spell out the relaxed cluster safety check: the delete runs and rows in unreachable files or on unflushed nodes are left in place. --- src/arc_client/management/async_databases.py | 26 +++++-- src/arc_client/management/async_delete.py | 26 ++++++- src/arc_client/management/databases.py | 21 +++++- src/arc_client/management/delete.py | 26 ++++++- .../http/test_delete_allow_unreachable.py | 67 +++++++++++++++++++ 5 files changed, 155 insertions(+), 11 deletions(-) create mode 100644 tests/unit/http/test_delete_allow_unreachable.py diff --git a/src/arc_client/management/async_databases.py b/src/arc_client/management/async_databases.py index c3c6bb0..1290b07 100644 --- a/src/arc_client/management/async_databases.py +++ b/src/arc_client/management/async_databases.py @@ -99,17 +99,33 @@ async def create(self, name: str) -> DatabaseInfo: except Exception as e: raise ArcError(f"Failed to create database {name!r}: {e}") from e - async def delete(self, name: str, confirm: bool = False) -> dict[str, Any]: - """Delete a database (admin). Requires confirm=True.""" + async def delete( + self, name: str, confirm: bool = False, allow_unreachable: bool = False + ) -> dict[str, Any]: + """Delete a database (admin). Requires confirm=True. + + Args: + name: Database to delete. + confirm: Must be True; deletion is irreversible. + allow_unreachable: Shared-storage cluster only; relaxes a safety + check. Without it, Arc refuses the deletion (503, naming + ``unflushed_nodes``) when other nodes could not flush the rows + they buffer for this database first. With it, the deletion + runs and the rows those nodes still buffer are left behind; + the response lists the nodes in ``unflushed_nodes``. Sent as + ``?allow_unreachable=true`` only when True. Needs Arc + 2027.01.1+; older servers ignore it. + """ validate_name(name, "database") if not confirm: raise ArcValidationError( "Refusing to delete database without confirm=True. This is destructive." ) try: - response = await self._http.delete( - f"/api/v1/databases/{name}", params={"confirm": "true"} - ) + params = {"confirm": "true"} + if allow_unreachable: + params["allow_unreachable"] = "true" + response = await self._http.delete(f"/api/v1/databases/{name}", params=params) body: dict[str, Any] = response.json() return body except ArcError: diff --git a/src/arc_client/management/async_delete.py b/src/arc_client/management/async_delete.py index 99a7f74..c1dc5de 100644 --- a/src/arc_client/management/async_delete.py +++ b/src/arc_client/management/async_delete.py @@ -2,6 +2,8 @@ from __future__ import annotations +from typing import Any + from arc_client.config import ClientConfig from arc_client.exceptions import ArcError, ArcValidationError from arc_client.http.async_http import AsyncHTTPClient @@ -22,8 +24,26 @@ async def delete( where: str, dry_run: bool = True, confirm: bool = False, + allow_unreachable: bool = False, ) -> DeleteResponse: - """Delete data matching the WHERE clause.""" + """Delete data matching the WHERE clause. + + Args: + database: Database holding the measurement. + measurement: Measurement to delete from. + where: SQL WHERE clause selecting the rows. + dry_run: Preview only (default). Set False to delete. + confirm: Required by Arc for any delete that is not a dry run. + allow_unreachable: Cluster only; relaxes a safety check. Without + it, Arc refuses the DELETE (503) when files the cluster + manifest lists are not on this node yet (stream takeover), or + when other nodes of a shared-storage cluster could not flush + the rows they buffer for this measurement first. With it, the + DELETE runs anyway and the rows in those files or on those + nodes are NOT deleted; the response names them in + ``unreachable_files`` and ``unflushed_nodes``. Sent only when + True. Needs Arc 2027.01.1+; older servers ignore it. + """ if not database: raise ArcValidationError("database is required") if not measurement: @@ -33,13 +53,15 @@ async def delete( "where clause is required. Use '1=1' with confirm=True for full delete" ) - payload = { + payload: dict[str, Any] = { "database": database, "measurement": measurement, "where": where, "dry_run": dry_run, "confirm": confirm, } + if allow_unreachable: + payload["allow_unreachable"] = True try: response = await self._http.post("/api/v1/delete", json=payload) diff --git a/src/arc_client/management/databases.py b/src/arc_client/management/databases.py index 262b138..ff3d46d 100644 --- a/src/arc_client/management/databases.py +++ b/src/arc_client/management/databases.py @@ -122,12 +122,26 @@ def create(self, name: str) -> DatabaseInfo: except Exception as e: raise ArcError(f"Failed to create database {name!r}: {e}") from e - def delete(self, name: str, confirm: bool = False) -> dict[str, Any]: + def delete( + self, name: str, confirm: bool = False, allow_unreachable: bool = False + ) -> dict[str, Any]: """Delete a database (admin). Requires ``confirm=True`` and server-side delete operations to be enabled (otherwise the server returns 403). + Args: + name: Database to delete. + confirm: Must be True; deletion is irreversible. + allow_unreachable: Shared-storage cluster only; relaxes a safety + check. Without it, Arc refuses the deletion (503, naming + ``unflushed_nodes``) when other nodes could not flush the rows + they buffer for this database first. With it, the deletion + runs and the rows those nodes still buffer are left behind; + the response lists the nodes in ``unflushed_nodes``. Sent as + ``?allow_unreachable=true`` only when True. Needs Arc + 2027.01.1+; older servers ignore it. + Returns: The server's response dict, e.g. ``{"message": ..., "files_deleted": N}``. """ @@ -137,7 +151,10 @@ def delete(self, name: str, confirm: bool = False) -> dict[str, Any]: "Refusing to delete database without confirm=True. This is destructive." ) try: - response = self._http.delete(f"/api/v1/databases/{name}", params={"confirm": "true"}) + params = {"confirm": "true"} + if allow_unreachable: + params["allow_unreachable"] = "true" + response = self._http.delete(f"/api/v1/databases/{name}", params=params) body: dict[str, Any] = response.json() return body except ArcError: diff --git a/src/arc_client/management/delete.py b/src/arc_client/management/delete.py index 0208e54..8f22bcc 100644 --- a/src/arc_client/management/delete.py +++ b/src/arc_client/management/delete.py @@ -2,6 +2,8 @@ from __future__ import annotations +from typing import Any + from arc_client.config import ClientConfig from arc_client.exceptions import ArcError, ArcValidationError from arc_client.http.sync_http import SyncHTTPClient @@ -25,8 +27,26 @@ def delete( where: str, dry_run: bool = True, confirm: bool = False, + allow_unreachable: bool = False, ) -> DeleteResponse: - """Delete data matching the WHERE clause.""" + """Delete data matching the WHERE clause. + + Args: + database: Database holding the measurement. + measurement: Measurement to delete from. + where: SQL WHERE clause selecting the rows. + dry_run: Preview only (default). Set False to delete. + confirm: Required by Arc for any delete that is not a dry run. + allow_unreachable: Cluster only; relaxes a safety check. Without + it, Arc refuses the DELETE (503) when files the cluster + manifest lists are not on this node yet (stream takeover), or + when other nodes of a shared-storage cluster could not flush + the rows they buffer for this measurement first. With it, the + DELETE runs anyway and the rows in those files or on those + nodes are NOT deleted; the response names them in + ``unreachable_files`` and ``unflushed_nodes``. Sent only when + True. Needs Arc 2027.01.1+; older servers ignore it. + """ if not database: raise ArcValidationError("database is required") if not measurement: @@ -36,13 +56,15 @@ def delete( "where clause is required. Use '1=1' with confirm=True for full delete" ) - payload = { + payload: dict[str, Any] = { "database": database, "measurement": measurement, "where": where, "dry_run": dry_run, "confirm": confirm, } + if allow_unreachable: + payload["allow_unreachable"] = True try: response = self._http.post("/api/v1/delete", json=payload) diff --git a/tests/unit/http/test_delete_allow_unreachable.py b/tests/unit/http/test_delete_allow_unreachable.py new file mode 100644 index 0000000..bf1e8f2 --- /dev/null +++ b/tests/unit/http/test_delete_allow_unreachable.py @@ -0,0 +1,67 @@ +"""allow_unreachable on row and database deletes, shared by sync and async clients.""" + +from __future__ import annotations + +import inspect +import json +from contextlib import AsyncExitStack + +import pytest + +from arc_client import ArcClient, AsyncArcClient + +DELETE_ARGS = {"database": "db", "measurement": "cpu", "where": "value < 0"} + + +async def call(obj, method, *args, **kwargs): + result = getattr(obj, method)(*args, **kwargs) + return await result if inspect.isawaitable(result) else result + + +@pytest.fixture(params=[False, True], ids=["sync", "async"]) +def async_mode(request): + return request.param + + +async def enter_client(stack, async_mode): + if async_mode: + return await stack.enter_async_context(AsyncArcClient(token="t", max_retries=0)) + return stack.enter_context(ArcClient(token="t", max_retries=0)) + + +@pytest.mark.parametrize("allow", [None, False, True]) +async def test_row_delete_allow_unreachable(httpx_mock, async_mode, allow): + httpx_mock.add_response( + method="POST", + json={"success": True, "unreachable_files": ["a.parquet"], "unflushed_nodes": ["w2"]}, + ) + kwargs = {} if allow is None else {"allow_unreachable": allow} + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + result = await call( + client.delete, "delete", **DELETE_ARGS, dry_run=False, confirm=True, **kwargs + ) + assert result.unreachable_files == ["a.parquet"] + assert result.unflushed_nodes == ["w2"] + expected = {**DELETE_ARGS, "dry_run": False, "confirm": True} + if allow: + expected["allow_unreachable"] = True + assert json.loads(httpx_mock.get_request().content) == expected + + +@pytest.mark.parametrize("allow", [None, False, True]) +async def test_database_delete_allow_unreachable(httpx_mock, async_mode, allow): + httpx_mock.add_response( + method="DELETE", json={"message": "deleted", "files_deleted": 2, "unflushed_nodes": ["w2"]} + ) + kwargs = {} if allow is None else {"allow_unreachable": allow} + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + result = await call(client.databases, "delete", "db", confirm=True, **kwargs) + assert result["unflushed_nodes"] == ["w2"] + request = httpx_mock.get_request() + assert request.url.path == "/api/v1/databases/db" + expected = {"confirm": "true"} + if allow: + expected["allow_unreachable"] = "true" + assert dict(request.url.params) == expected From cdec77700bf94781889fbe9b497cbdd1fb22d280 Mon Sep 17 00:00:00 2001 From: Ignacio Van Droogenbroeck <64545348+xe-nvdk@users.noreply.github.com> Date: Mon, 2 Nov 2026 08:46:18 -0600 Subject: [PATCH 07/20] feat(diagnostics): add readiness, metrics, logs and memory diagnostics (#15) Add client.diagnostics (sync and async): - readiness(write=...) for /ready and /ready/write; a 503 probe answer is returned as ReadinessStatus (not_ready / not_write_target + reason) - JSON metrics, timeseries, and Prometheus text (public routes) - admin logs with client-side range checks for values Arc would silently replace with defaults - admin memstats / duckdb-memory, and free_os_memory gated by confirm=True because it forces a stop-the-world GC --- src/arc_client/async_client.py | 10 + src/arc_client/client.py | 10 + src/arc_client/management/__init__.py | 4 + .../management/async_diagnostics.py | 178 +++++++++++++ src/arc_client/management/diagnostics.py | 222 ++++++++++++++++ src/arc_client/models/__init__.py | 2 + src/arc_client/models/diagnostics.py | 31 +++ tests/unit/http/test_diagnostics_contracts.py | 245 ++++++++++++++++++ 8 files changed, 702 insertions(+) create mode 100644 src/arc_client/management/async_diagnostics.py create mode 100644 src/arc_client/management/diagnostics.py create mode 100644 src/arc_client/models/diagnostics.py create mode 100644 tests/unit/http/test_diagnostics_contracts.py diff --git a/src/arc_client/async_client.py b/src/arc_client/async_client.py index 2dd36a1..99e0485 100644 --- a/src/arc_client/async_client.py +++ b/src/arc_client/async_client.py @@ -91,6 +91,7 @@ def __init__( self._tiering: Any = None self._backup: Any = None self._mqtt: Any = None + self._diagnostics: Any = None self._reconciliation: Any = None def _get_http(self) -> AsyncHTTPClient: @@ -268,6 +269,15 @@ def reconciliation(self) -> Any: self._reconciliation = AsyncReconciliationClient(self._get_http(), self._config) return self._reconciliation + @property + def diagnostics(self) -> Any: + """Get the async operational diagnostics client (readiness, metrics, logs, memory).""" + if self._diagnostics is None: + from arc_client.management.async_diagnostics import AsyncDiagnosticsClient + + self._diagnostics = AsyncDiagnosticsClient(self._get_http(), self._config) + return self._diagnostics + async def health(self) -> HealthResponse: """Check server health. diff --git a/src/arc_client/client.py b/src/arc_client/client.py index acad506..f2c17b1 100644 --- a/src/arc_client/client.py +++ b/src/arc_client/client.py @@ -91,6 +91,7 @@ def __init__( self._tiering: Any = None self._backup: Any = None self._mqtt: Any = None + self._diagnostics: Any = None self._reconciliation: Any = None def _get_http(self) -> SyncHTTPClient: @@ -266,6 +267,15 @@ def reconciliation(self) -> Any: self._reconciliation = ReconciliationClient(self._get_http(), self._config) return self._reconciliation + @property + def diagnostics(self) -> Any: + """Get the operational diagnostics client (readiness, metrics, logs, memory).""" + if self._diagnostics is None: + from arc_client.management.diagnostics import DiagnosticsClient + + self._diagnostics = DiagnosticsClient(self._get_http(), self._config) + return self._diagnostics + def health(self) -> HealthResponse: """Check server health. diff --git a/src/arc_client/management/__init__.py b/src/arc_client/management/__init__.py index 2a09c3b..ee760fc 100644 --- a/src/arc_client/management/__init__.py +++ b/src/arc_client/management/__init__.py @@ -12,6 +12,7 @@ from arc_client.management.async_continuous_query import AsyncContinuousQueryClient from arc_client.management.async_databases import AsyncDatabaseClient from arc_client.management.async_delete import AsyncDeleteClient +from arc_client.management.async_diagnostics import AsyncDiagnosticsClient from arc_client.management.async_governance import AsyncGovernanceClient from arc_client.management.async_mqtt import AsyncMQTTClient from arc_client.management.async_query_management import AsyncQueryManagementClient @@ -26,6 +27,7 @@ from arc_client.management.continuous_query import ContinuousQueryClient from arc_client.management.databases import DatabaseClient from arc_client.management.delete import DeleteClient +from arc_client.management.diagnostics import DiagnosticsClient from arc_client.management.governance import GovernanceClient from arc_client.management.mqtt import MQTTClient from arc_client.management.query_management import QueryManagementClient @@ -65,4 +67,6 @@ "AsyncMQTTClient", "ReconciliationClient", "AsyncReconciliationClient", + "DiagnosticsClient", + "AsyncDiagnosticsClient", ] diff --git a/src/arc_client/management/async_diagnostics.py b/src/arc_client/management/async_diagnostics.py new file mode 100644 index 0000000..1ab2b10 --- /dev/null +++ b/src/arc_client/management/async_diagnostics.py @@ -0,0 +1,178 @@ +"""Asynchronous operational-diagnostics client for Arc. + +Wraps Arc's readiness probes, JSON metrics, Prometheus metrics, buffered +application logs, and the admin memory diagnostics under ``/api/v1/debug``. + +Everything except :meth:`DiagnosticsClient.readiness` returns raw dicts: these +are operator diagnostics whose fields follow Arc's internal counters and Go +runtime, not a stable contract. + +Access (Arc ``internal/api/server.go``, ``cmd/arc/main.go``, +``internal/api/debug.go``): + +* ``/ready``, ``/ready/write``, ``/metrics`` and ``/api/v1/metrics/*`` are + public; no token is needed (they are meant for load balancers, scrapers and + dashboards). +* ``/api/v1/logs`` and ``/api/v1/debug/*`` require an admin token. +""" + +from __future__ import annotations + +from typing import Any, Optional + +from arc_client.config import ClientConfig +from arc_client.exceptions import ArcError, ArcServerError +from arc_client.http.async_http import AsyncHTTPClient +from arc_client.management.diagnostics import ( + _free_os_memory_guard, + _logs_params, + _readiness_from_error, + _readiness_path, + _timeseries_request, +) +from arc_client.models.diagnostics import ReadinessStatus + + +class AsyncDiagnosticsClient: + """Readiness, metrics, logs and memory diagnostics.""" + + def __init__(self, http: AsyncHTTPClient, config: ClientConfig) -> None: + self._http = http + self._config = config + + async def _json(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]: + try: + response = await getattr(self._http, method)(path, **kwargs) + data = response.json() + except ArcError: + raise + except Exception as e: + raise ArcError(f"Diagnostics request failed ({method.upper()} {path}): {e}") from e + if not isinstance(data, dict): + raise ArcError(f"Unexpected response from {path}: {data!r}") + return data + + # Readiness ----------------------------------------------------------- + + async def readiness(self, write: bool = False) -> ReadinessStatus: + """Ask whether this node should receive traffic (public). + + ``write=False`` probes ``/ready``: is the node ready at all. + ``write=True`` probes ``/ready/write``: should a load balancer send + *writes* here. It is strictly narrower — a reader, or a non-primary + node in local-storage mode, is ready but answers + ``status="not_write_target"``. + + Unlike :meth:`AsyncArcClient.ready`, the reason is kept: a 503 from the + probe is returned as a :class:`ReadinessStatus` with ``ready == + False``, not raised. Note that the SDK retries a GET that returns 503 + up to ``max_retries`` times before answering; construct the client + with ``max_retries=0`` for a probe that must answer immediately. + """ + try: + data = await self._json("get", _readiness_path(write)) + except ArcServerError as e: + return _readiness_from_error(e) + return ReadinessStatus.model_validate(data) + + # Metrics ------------------------------------------------------------- + + async def metrics(self) -> dict[str, Any]: + """Get every metric counter as JSON, plus ``timestamp`` (public).""" + return await self._json("get", "/api/v1/metrics") + + async def memory_metrics(self) -> dict[str, Any]: + """Get Go runtime memory and GC metrics: ``memory`` and ``runtime`` (public).""" + return await self._json("get", "/api/v1/metrics/memory") + + async def query_pool_metrics(self) -> dict[str, Any]: + """Get DuckDB connection-pool and query counters under ``pool`` (public).""" + return await self._json("get", "/api/v1/metrics/query-pool") + + async def endpoint_metrics(self) -> dict[str, Any]: + """Get HTTP and query request statistics with average latencies (public).""" + return await self._json("get", "/api/v1/metrics/endpoints") + + async def timeseries( + self, metric_type: str, duration_minutes: Optional[int] = None + ) -> dict[str, Any]: + """Get recent sampled metric points (public). + + Args: + metric_type: ``"system"``, ``"application"`` or ``"api"``. + duration_minutes: Window, 1-1440. Server default 30. + + Raises: + ArcValidationError: Unknown type or out-of-range duration (Arc + would silently fall back to 30 minutes). + """ + path, params = _timeseries_request(metric_type, duration_minutes) + return await self._json("get", path, params=params) + + async def prometheus(self) -> str: + """Get ``/metrics`` in Prometheus text exposition format (public).""" + try: + response = await self._http.get("/metrics") + return str(response.text) + except ArcError: + raise + except Exception as e: + raise ArcError(f"Diagnostics request failed (GET /metrics): {e}") from e + + # Logs (admin) -------------------------------------------------------- + + async def logs( + self, + limit: Optional[int] = None, + level: Optional[str] = None, + since_minutes: Optional[int] = None, + ) -> dict[str, Any]: + """Get recent application logs from the node's in-memory buffer (admin). + + Args: + limit: Max entries, 1-1000. Server default 100. + level: Minimum level, case-insensitive: ``debug``, ``info``, + ``warn``, ``error`` or ``fatal`` (entries at or above it). + Any other value matches that level name exactly. + since_minutes: Look-back window, 1-1440. Server default 60. + + Returns a raw dict with ``logs`` (newest first; ``timestamp``, + ``level``, ``component``, ``message``, ``caller``), ``count`` and the + effective filters. + + Raises: + ArcValidationError: ``limit``/``since_minutes`` out of range (Arc + would silently use its default instead). + """ + return await self._json( + "get", "/api/v1/logs", params=_logs_params(limit, level, since_minutes) + ) + + # Memory debugging (admin) ------------------------------------------- + + async def memstats(self) -> dict[str, Any]: + """Attribute process memory to Go heap, glibc and DuckDB (admin). + + Returns ``go``, and when the platform reports them ``process``, + ``glibc_heap`` and ``duckdb_summary``. Intended for support cases + about unexplained memory growth. + """ + return await self._json("get", "/api/v1/debug/memstats") + + async def duckdb_memory(self) -> dict[str, Any]: + """Get DuckDB memory use per tag: ``total_bytes``, ``by_tag`` (admin).""" + return await self._json("get", "/api/v1/debug/duckdb-memory") + + async def free_os_memory(self, confirm: bool = False) -> dict[str, Any]: + """Force a GC and return freed heap to the OS (admin). + + Stop-the-world: it pauses ingestion and queries while it runs, so it + requires ``confirm=True``. Arc allows one call per 30 seconds; a + throttled call raises :class:`~arc_client.exceptions.ArcRateLimitError` + whose ``response_body`` may carry ``retry_after_seconds``. + + Returns ``duration_ms`` and ``heap_released_before`` / + ``heap_released_after`` / ``heap_released_delta``. + """ + _free_os_memory_guard(confirm) + return await self._json("post", "/api/v1/debug/free-os-memory") diff --git a/src/arc_client/management/diagnostics.py b/src/arc_client/management/diagnostics.py new file mode 100644 index 0000000..84d92e8 --- /dev/null +++ b/src/arc_client/management/diagnostics.py @@ -0,0 +1,222 @@ +"""Synchronous operational-diagnostics client for Arc. + +Wraps Arc's readiness probes, JSON metrics, Prometheus metrics, buffered +application logs, and the admin memory diagnostics under ``/api/v1/debug``. + +Everything except :meth:`DiagnosticsClient.readiness` returns raw dicts: these +are operator diagnostics whose fields follow Arc's internal counters and Go +runtime, not a stable contract. + +Access (Arc ``internal/api/server.go``, ``cmd/arc/main.go``, +``internal/api/debug.go``): + +* ``/ready``, ``/ready/write``, ``/metrics`` and ``/api/v1/metrics/*`` are + public; no token is needed (they are meant for load balancers, scrapers and + dashboards). +* ``/api/v1/logs`` and ``/api/v1/debug/*`` require an admin token. +""" + +from __future__ import annotations + +from typing import Any, Optional + +from arc_client.config import ClientConfig +from arc_client.exceptions import ArcError, ArcServerError, ArcValidationError +from arc_client.http.sync_http import SyncHTTPClient +from arc_client.models.diagnostics import ReadinessStatus + +TIMESERIES_TYPES = ("system", "application", "api") + + +def _bounded(name: str, value: Optional[int], low: int, high: int) -> Optional[int]: + """Reject values Arc would silently replace with its default.""" + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int) or not low <= value <= high: + raise ArcValidationError(f"{name} must be an integer from {low} to {high}") + return value + + +def _logs_params( + limit: Optional[int], level: Optional[str], since_minutes: Optional[int] +) -> Optional[dict[str, Any]]: + params: dict[str, Any] = {} + if _bounded("limit", limit, 1, 1000) is not None: + params["limit"] = limit + if level: + params["level"] = level + if _bounded("since_minutes", since_minutes, 1, 1440) is not None: + params["since_minutes"] = since_minutes + return params or None + + +def _timeseries_request( + metric_type: str, duration_minutes: Optional[int] +) -> tuple[str, Optional[dict[str, Any]]]: + if metric_type not in TIMESERIES_TYPES: + raise ArcValidationError(f"metric_type must be one of {', '.join(TIMESERIES_TYPES)}") + duration = _bounded("duration_minutes", duration_minutes, 1, 1440) + params = {"duration_minutes": duration} if duration is not None else None + return f"/api/v1/metrics/timeseries/{metric_type}", params + + +def _readiness_path(write: bool) -> str: + return "/ready/write" if write else "/ready" + + +def _readiness_from_error(error: ArcServerError) -> ReadinessStatus: + """A 503 from a readiness probe is an answer, not a failure.""" + body = error.response_body + if error.status_code == 503 and isinstance(body, dict) and isinstance(body.get("status"), str): + return ReadinessStatus.model_validate(body) + raise error + + +def _free_os_memory_guard(confirm: bool) -> None: + if not confirm: + raise ArcValidationError( + "Refusing to call free-os-memory without confirm=True. It forces a " + "stop-the-world garbage collection that pauses ingestion and queries." + ) + + +class DiagnosticsClient: + """Readiness, metrics, logs and memory diagnostics.""" + + def __init__(self, http: SyncHTTPClient, config: ClientConfig) -> None: + self._http = http + self._config = config + + def _json(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]: + try: + response = getattr(self._http, method)(path, **kwargs) + data = response.json() + except ArcError: + raise + except Exception as e: + raise ArcError(f"Diagnostics request failed ({method.upper()} {path}): {e}") from e + if not isinstance(data, dict): + raise ArcError(f"Unexpected response from {path}: {data!r}") + return data + + # Readiness ----------------------------------------------------------- + + def readiness(self, write: bool = False) -> ReadinessStatus: + """Ask whether this node should receive traffic (public). + + ``write=False`` probes ``/ready``: is the node ready at all. + ``write=True`` probes ``/ready/write``: should a load balancer send + *writes* here. It is strictly narrower — a reader, or a non-primary + node in local-storage mode, is ready but answers + ``status="not_write_target"``. + + Unlike :meth:`ArcClient.ready`, the reason is kept: a 503 from the + probe is returned as a :class:`ReadinessStatus` with ``ready == + False``, not raised. Note that the SDK retries a GET that returns 503 + up to ``max_retries`` times before answering; construct the client + with ``max_retries=0`` for a probe that must answer immediately. + """ + try: + data = self._json("get", _readiness_path(write)) + except ArcServerError as e: + return _readiness_from_error(e) + return ReadinessStatus.model_validate(data) + + # Metrics ------------------------------------------------------------- + + def metrics(self) -> dict[str, Any]: + """Get every metric counter as JSON, plus ``timestamp`` (public).""" + return self._json("get", "/api/v1/metrics") + + def memory_metrics(self) -> dict[str, Any]: + """Get Go runtime memory and GC metrics: ``memory`` and ``runtime`` (public).""" + return self._json("get", "/api/v1/metrics/memory") + + def query_pool_metrics(self) -> dict[str, Any]: + """Get DuckDB connection-pool and query counters under ``pool`` (public).""" + return self._json("get", "/api/v1/metrics/query-pool") + + def endpoint_metrics(self) -> dict[str, Any]: + """Get HTTP and query request statistics with average latencies (public).""" + return self._json("get", "/api/v1/metrics/endpoints") + + def timeseries( + self, metric_type: str, duration_minutes: Optional[int] = None + ) -> dict[str, Any]: + """Get recent sampled metric points (public). + + Args: + metric_type: ``"system"``, ``"application"`` or ``"api"``. + duration_minutes: Window, 1-1440. Server default 30. + + Raises: + ArcValidationError: Unknown type or out-of-range duration (Arc + would silently fall back to 30 minutes). + """ + path, params = _timeseries_request(metric_type, duration_minutes) + return self._json("get", path, params=params) + + def prometheus(self) -> str: + """Get ``/metrics`` in Prometheus text exposition format (public).""" + try: + return str(self._http.get("/metrics").text) + except ArcError: + raise + except Exception as e: + raise ArcError(f"Diagnostics request failed (GET /metrics): {e}") from e + + # Logs (admin) -------------------------------------------------------- + + def logs( + self, + limit: Optional[int] = None, + level: Optional[str] = None, + since_minutes: Optional[int] = None, + ) -> dict[str, Any]: + """Get recent application logs from the node's in-memory buffer (admin). + + Args: + limit: Max entries, 1-1000. Server default 100. + level: Minimum level, case-insensitive: ``debug``, ``info``, + ``warn``, ``error`` or ``fatal`` (entries at or above it). + Any other value matches that level name exactly. + since_minutes: Look-back window, 1-1440. Server default 60. + + Returns a raw dict with ``logs`` (newest first; ``timestamp``, + ``level``, ``component``, ``message``, ``caller``), ``count`` and the + effective filters. + + Raises: + ArcValidationError: ``limit``/``since_minutes`` out of range (Arc + would silently use its default instead). + """ + return self._json("get", "/api/v1/logs", params=_logs_params(limit, level, since_minutes)) + + # Memory debugging (admin) ------------------------------------------- + + def memstats(self) -> dict[str, Any]: + """Attribute process memory to Go heap, glibc and DuckDB (admin). + + Returns ``go``, and when the platform reports them ``process``, + ``glibc_heap`` and ``duckdb_summary``. Intended for support cases + about unexplained memory growth. + """ + return self._json("get", "/api/v1/debug/memstats") + + def duckdb_memory(self) -> dict[str, Any]: + """Get DuckDB memory use per tag: ``total_bytes``, ``by_tag`` (admin).""" + return self._json("get", "/api/v1/debug/duckdb-memory") + + def free_os_memory(self, confirm: bool = False) -> dict[str, Any]: + """Force a GC and return freed heap to the OS (admin). + + Stop-the-world: it pauses ingestion and queries while it runs, so it + requires ``confirm=True``. Arc allows one call per 30 seconds; a + throttled call raises :class:`~arc_client.exceptions.ArcRateLimitError` + whose ``response_body`` may carry ``retry_after_seconds``. + + Returns ``duration_ms`` and ``heap_released_before`` / + ``heap_released_after`` / ``heap_released_delta``. + """ + _free_os_memory_guard(confirm) + return self._json("post", "/api/v1/debug/free-os-memory") diff --git a/src/arc_client/models/__init__.py b/src/arc_client/models/__init__.py index 975c3ae..f4cd526 100644 --- a/src/arc_client/models/__init__.py +++ b/src/arc_client/models/__init__.py @@ -20,6 +20,7 @@ MeasurementListResponse, ) from arc_client.models.delete import DeleteConfigResponse, DeleteResponse +from arc_client.models.diagnostics import ReadinessStatus from arc_client.models.import_ import ( ImportResult, ImportStats, @@ -54,6 +55,7 @@ "HealthResponse", "MeasurementInfo", "QueryResponse", + "ReadinessStatus", "ReconciliationRun", "ReadyResponse", "RetentionExecution", diff --git a/src/arc_client/models/diagnostics.py b/src/arc_client/models/diagnostics.py new file mode 100644 index 0000000..c726574 --- /dev/null +++ b/src/arc_client/models/diagnostics.py @@ -0,0 +1,31 @@ +"""Models for Arc readiness probes.""" + +from __future__ import annotations + +from typing import Optional + +from pydantic import BaseModel, ConfigDict + + +class ReadinessStatus(BaseModel): + """The answer of ``/ready`` or ``/ready/write``. + + ``status`` is ``"ready"``, ``"not_ready"`` (the node is not ready for any + traffic) or, from ``/ready/write`` only, ``"not_write_target"`` (the node + is healthy but its cluster role does not take writes; route writes to a + writer). ``reason`` and, for a storage-credential problem, ``tier`` + explain a not-ready answer. ``uptime_sec`` is sent by ``/ready`` only. + """ + + model_config = ConfigDict(extra="allow") + + status: str + time: Optional[str] = None + reason: Optional[str] = None + tier: Optional[str] = None + uptime_sec: Optional[float] = None + + @property + def ready(self) -> bool: + """True when the node should receive the traffic the probe asked about.""" + return self.status == "ready" diff --git a/tests/unit/http/test_diagnostics_contracts.py b/tests/unit/http/test_diagnostics_contracts.py new file mode 100644 index 0000000..b57ff72 --- /dev/null +++ b/tests/unit/http/test_diagnostics_contracts.py @@ -0,0 +1,245 @@ +"""Operational diagnostics wire contracts. + +Arc sources: internal/api/server.go (readiness, metrics, logs) and +internal/api/debug.go (memory diagnostics). +""" + +from __future__ import annotations + +import inspect +from contextlib import AsyncExitStack +from typing import Any + +import pytest +from pytest_httpx import HTTPXMock + +from arc_client import ArcClient, AsyncArcClient +from arc_client.exceptions import ( + ArcError, + ArcPermissionError, + ArcRateLimitError, + ArcServerError, + ArcValidationError, +) +from arc_client.models.diagnostics import ReadinessStatus + +ROOT = "http://localhost:8000" + + +async def call(client: Any, method: str, *args: Any, **kwargs: Any) -> Any: + result = getattr(client.diagnostics, method)(*args, **kwargs) + return await result if inspect.isawaitable(result) else result + + +@pytest.fixture(params=[False, True], ids=["sync", "async"]) +def async_mode(request: Any) -> bool: + return bool(request.param) + + +async def enter_client(stack: AsyncExitStack, async_mode: bool) -> Any: + if async_mode: + return await stack.enter_async_context(AsyncArcClient(token="t", max_retries=0)) + return stack.enter_context(ArcClient(token="t", max_retries=0)) + + +# Readiness ----------------------------------------------------------------- + + +@pytest.mark.parametrize(("write", "path"), [(False, "/ready"), (True, "/ready/write")]) +async def test_readiness_ready( + httpx_mock: HTTPXMock, async_mode: bool, write: bool, path: str +) -> None: + body = {"status": "ready", "time": "2027-01-05T10:00:00Z", "uptime_sec": 12.5} + httpx_mock.add_response(method="GET", url=f"{ROOT}{path}", json=body) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + result = await call(client, "readiness", write=write) + assert isinstance(result, ReadinessStatus) + assert result.ready is True + assert result.uptime_sec == 12.5 + + +@pytest.mark.parametrize( + ("write", "body"), + [ + (False, {"status": "not_ready", "reason": "storage credentials expired", "tier": "cold"}), + (True, {"status": "not_ready", "reason": "replaying write-ahead log"}), + (True, {"status": "not_write_target", "reason": "route writes to a writer"}), + ], +) +async def test_readiness_503_is_an_answer( + httpx_mock: HTTPXMock, async_mode: bool, write: bool, body: dict[str, Any] +) -> None: + httpx_mock.add_response(status_code=503, json={**body, "time": "2027-01-05T10:00:00Z"}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + result = await call(client, "readiness", write=write) + assert result.ready is False + assert result.status == body["status"] + assert result.reason == body["reason"] + assert result.tier == body.get("tier") + + +async def test_readiness_other_503_raises(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(status_code=503, text="upstream unavailable") + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcServerError): + await call(client, "readiness") + + +# Metrics ------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("method", "path"), + [ + ("metrics", "/api/v1/metrics"), + ("memory_metrics", "/api/v1/metrics/memory"), + ("query_pool_metrics", "/api/v1/metrics/query-pool"), + ("endpoint_metrics", "/api/v1/metrics/endpoints"), + ("memstats", "/api/v1/debug/memstats"), + ("duckdb_memory", "/api/v1/debug/duckdb-memory"), + ], +) +async def test_raw_dict_reads( + httpx_mock: HTTPXMock, async_mode: bool, method: str, path: str +) -> None: + body = {"timestamp": "2027-01-05T10:00:00Z", "value": {"nested": 1}} + httpx_mock.add_response(method="GET", url=f"{ROOT}{path}", json=body) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert await call(client, method) == body + + +async def test_raw_dict_rejects_non_object(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(method="GET", url=f"{ROOT}/api/v1/metrics", json=[1, 2]) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcError): + await call(client, "metrics") + + +async def test_timeseries(httpx_mock: HTTPXMock, async_mode: bool) -> None: + body = {"type": "api", "duration_minutes": 15, "points_count": 0, "data": []} + httpx_mock.add_response( + method="GET", + url=f"{ROOT}/api/v1/metrics/timeseries/api?duration_minutes=15", + json=body, + ) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert await call(client, "timeseries", "api", duration_minutes=15) == body + + +async def test_timeseries_default_window(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response( + method="GET", url=f"{ROOT}/api/v1/metrics/timeseries/system", json={"data": []} + ) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + await call(client, "timeseries", "system") + + +@pytest.mark.parametrize( + ("args", "kwargs"), + [ + (("disk",), {}), + (("api",), {"duration_minutes": 0}), + (("api",), {"duration_minutes": 1441}), + (("api",), {"duration_minutes": True}), + ], +) +async def test_timeseries_validation( + httpx_mock: HTTPXMock, async_mode: bool, args: tuple[Any, ...], kwargs: dict[str, Any] +) -> None: + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcValidationError): + await call(client, "timeseries", *args, **kwargs) + assert httpx_mock.get_requests() == [] + + +async def test_prometheus_text(httpx_mock: HTTPXMock, async_mode: bool) -> None: + text = "# HELP arc_up 1\narc_up 1\n" + httpx_mock.add_response(method="GET", url=f"{ROOT}/metrics", text=text) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert await call(client, "prometheus") == text + + +# Logs ---------------------------------------------------------------------- + + +async def test_logs_params(httpx_mock: HTTPXMock, async_mode: bool) -> None: + body = {"count": 0, "limit": 50, "level_filter": "warn", "since_minutes": 10, "logs": []} + httpx_mock.add_response( + method="GET", + url=f"{ROOT}/api/v1/logs?limit=50&level=warn&since_minutes=10", + json=body, + ) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert await call(client, "logs", limit=50, level="warn", since_minutes=10) == body + + +async def test_logs_defaults_send_no_params(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(method="GET", url=f"{ROOT}/api/v1/logs", json={"logs": []}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + await call(client, "logs") + + +@pytest.mark.parametrize( + "kwargs", + [{"limit": 0}, {"limit": 1001}, {"since_minutes": 0}, {"since_minutes": 1441}], +) +async def test_logs_validation( + httpx_mock: HTTPXMock, async_mode: bool, kwargs: dict[str, Any] +) -> None: + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcValidationError): + await call(client, "logs", **kwargs) + assert httpx_mock.get_requests() == [] + + +@pytest.mark.parametrize("method", ["logs", "memstats", "duckdb_memory"]) +async def test_admin_routes_refuse_non_admin( + httpx_mock: HTTPXMock, async_mode: bool, method: str +) -> None: + httpx_mock.add_response(status_code=403, json={"error": "Admin permission required"}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcPermissionError): + await call(client, method) + + +# free-os-memory ------------------------------------------------------------ + + +async def test_free_os_memory_requires_confirm(httpx_mock: HTTPXMock, async_mode: bool) -> None: + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcValidationError): + await call(client, "free_os_memory") + assert httpx_mock.get_requests() == [] + + +async def test_free_os_memory(httpx_mock: HTTPXMock, async_mode: bool) -> None: + body = {"duration_ms": 1.5, "heap_released_delta": 4096} + httpx_mock.add_response(method="POST", url=f"{ROOT}/api/v1/debug/free-os-memory", json=body) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert await call(client, "free_os_memory", confirm=True) == body + + +async def test_free_os_memory_throttled(httpx_mock: HTTPXMock, async_mode: bool) -> None: + body = {"error": "throttled", "retry_after_seconds": 12} + httpx_mock.add_response(method="POST", status_code=429, json=body) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcRateLimitError) as info: + await call(client, "free_os_memory", confirm=True) + assert info.value.response_body == body + assert len(httpx_mock.get_requests()) == 1 From 64855f36d80c61515f9a9ae42127c113233af4d8 Mon Sep 17 00:00:00 2001 From: Ignacio Van Droogenbroeck <64545348+xe-nvdk@users.noreply.github.com> Date: Mon, 2 Nov 2026 08:46:31 -0600 Subject: [PATCH 08/20] feat(cq): expose watermark_status on manual execution ExecuteCQResponse gains optional watermark_status (set, moved or pending), which Arc 2027.01.1+ returns from a manual run once the cluster has adopted continuous queries. None on older servers. --- src/arc_client/models/continuous_query.py | 12 +++++++++++- tests/unit/http/test_cq_contracts.py | 24 +++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/arc_client/models/continuous_query.py b/src/arc_client/models/continuous_query.py index de435ff..001fef1 100644 --- a/src/arc_client/models/continuous_query.py +++ b/src/arc_client/models/continuous_query.py @@ -39,7 +39,16 @@ def from_response(cls, data: dict[str, Any]) -> ContinuousQuery: class ExecuteCQResponse(BaseModel): - """Response from executing a continuous query.""" + """Response from executing a continuous query. + + Attributes: + watermark_status: In a cluster that adopted continuous queries, what + this manual run did to ``last_processed_time``: ``"set"``, + ``"moved"`` (another run or the primary moved it meanwhile and + theirs is kept) or ``"pending"`` (the cluster has not taken it + yet; the next run settles it). ``None`` otherwise, and always on + servers before Arc 2027.01.1. + """ query_id: int query_name: str @@ -55,6 +64,7 @@ class ExecuteCQResponse(BaseModel): executed_at: Optional[str] = None executed_query: Optional[str] = None error: Optional[str] = None + watermark_status: Optional[str] = None class CQExecution(BaseModel): diff --git a/tests/unit/http/test_cq_contracts.py b/tests/unit/http/test_cq_contracts.py index 5fe63e5..552abcb 100644 --- a/tests/unit/http/test_cq_contracts.py +++ b/tests/unit/http/test_cq_contracts.py @@ -143,3 +143,27 @@ async def test_clear_nonnullable_rejected_before_request(async_mode): client = await enter_client(stack, async_mode) with pytest.raises(ArcValidationError): await call(client, "update", 7, name=CLEAR) + + +EXECUTE_RESPONSE = { + "query_id": 7, + "query_name": "rollup", + "execution_id": "e1", + "status": "completed", + "start_time": "2026-01-01T00:00:00Z", + "end_time": "2026-01-01T01:00:00Z", + "records_written": 3, + "destination_measurement": "cpu_hour", +} + + +@pytest.mark.parametrize("status", [None, "set", "moved", "pending"]) +async def test_execute_watermark_status(httpx_mock, async_mode, status): + body = dict(EXECUTE_RESPONSE) + if status is not None: + body["watermark_status"] = status + httpx_mock.add_response(method="POST", json=body) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + result = await call(client, "execute", 7) + assert result.watermark_status == status From 3253ae882d5d81372aa1054f587fe059d101ae36 Mon Sep 17 00:00:00 2001 From: Ignacio Van Droogenbroeck <64545348+xe-nvdk@users.noreply.github.com> Date: Mon, 2 Nov 2026 08:48:35 -0600 Subject: [PATCH 09/20] feat(cluster): add operator controls for files, roles, voters, claims and policies Add delete_file, demote_writer, assign_compactor and converge_voters (Arc 2026.09.2+) and stream_claims, release_stream_claim, abort_compaction_job, policies and adopt_policies (Arc 2027.01.1+) to the sync and async cluster clients. Destructive controls (delete_file, demote_writer, converge_voters with dry_run=False, release_stream_claim with abandon=True, abort_compaction_job, adopt_policies) require confirm=True. converge_voters plans by default. adopt_policies raises when a standalone server answers 200 with success false. --- src/arc_client/management/async_cluster.py | 218 ++++++++++++++++++- src/arc_client/management/cluster.py | 241 ++++++++++++++++++++- tests/unit/http/test_cluster_contracts.py | 214 ++++++++++++++++++ 3 files changed, 663 insertions(+), 10 deletions(-) create mode 100644 tests/unit/http/test_cluster_contracts.py diff --git a/src/arc_client/management/async_cluster.py b/src/arc_client/management/async_cluster.py index 44e0327..ab104bf 100644 --- a/src/arc_client/management/async_cluster.py +++ b/src/arc_client/management/async_cluster.py @@ -1,17 +1,34 @@ """Asynchronous cluster-management client for Arc (enterprise). -Wraps ``/api/v1/cluster``. Reads are available to any valid token; ``files`` and -node removal require an admin token. When clustering is disabled the server -returns 200 with ``{"enabled": false, ...}`` rather than an error. +Wraps ``/api/v1/cluster``. The status/node/health reads are available to any +valid token; every other route (files, node removal, the operator controls +below) requires an admin token. None of them is gated on a license feature +beyond clustering itself. When clustering is disabled the reads (and +``policies``) return 200 with ``{"enabled": false, ...}``; the operator +controls answer 409 instead, and :meth:`AsyncClusterClient.adopt_policies` raises +:class:`~arc_client.exceptions.ArcError`, so "nothing happened" never reads +as success. + +Responses are returned as raw dicts: they are operator diagnostics whose +fields Arc extends between releases, not a promised contract. Controls that +lose data or leave the cluster without a role holder require ``confirm=True`` +and raise :class:`~arc_client.exceptions.ArcValidationError` otherwise. """ from __future__ import annotations -from typing import Any, Optional +from typing import Any, Optional, Union from arc_client.config import ClientConfig -from arc_client.exceptions import ArcError +from arc_client.exceptions import ArcError, ArcValidationError from arc_client.http.async_http import AsyncHTTPClient +from arc_client.management.cluster import ( + _check_adopted, + _check_kind, + _instance_segment, + _require_confirm, + _segment, +) class AsyncClusterClient: @@ -65,6 +82,197 @@ async def files( } return await self._json("get", "/api/v1/cluster/files", params=params or None) + async def delete_file( + self, path: str, confirm: bool = False, reason: Optional[str] = None + ) -> Any: + """Remove a file from the cluster manifest and delete it cluster-wide (admin). + + Destructive: on local storage every node unlinks the file; on shared + object storage the object is orphaned until the reconciliation + sweeper removes it. ``path`` is matched exactly, so malformed keys + that retention cannot remove can still be targeted. Needs Arc + 2026.09.2+. + + Args: + path: Manifest path of the file (max 4096 bytes). + confirm: Must be True. + reason: Recorded in the audit log and Raft entry (max 256 + characters; Arc defaults to ``"operator"``). + + Raises: + ArcValidationError: ``confirm`` is not True or ``path`` is empty. + ArcNotFoundError: The path is not in the cluster manifest. + """ + if not path: + raise ArcValidationError("path is required") + _require_confirm(confirm, "delete a cluster file") + params = {"path": path, "confirm": "true"} + if reason: + params["reason"] = reason + return await self._json("delete", "/api/v1/cluster/files", params=params) + async def remove_node(self, node_id: str) -> Any: """Remove a node from the cluster via Raft (admin).""" return await self._json("delete", f"/api/v1/cluster/nodes/{node_id}") + + async def demote_writer(self, node_id: str, confirm: bool = False) -> Any: + """Hand the primary-writer role off ``node_id`` so the cluster elects another (admin). + + Use it to drain a writer, or to recover a cluster without automatic + failover whose primary is gone for good. Not gated on the + writer-failover license feature. If no other writer can take the + role, the response has ``new_primary: None`` and the cluster has no + primary writer until one can, hence ``confirm``. Arc answers 409 when + ``node_id`` is not the current primary or this node is not the Raft + leader. Needs Arc 2026.09.2+. + + Raises: + ArcValidationError: ``confirm`` is not True. + """ + segment = _segment(node_id, "node_id") + _require_confirm(confirm, "demote the primary writer") + return await self._json("post", f"/api/v1/cluster/writers/{segment}/demote") + + async def assign_compactor(self, node_id: str) -> Any: + """Hand the compactor lease to ``node_id`` (admin). + + The target is explicit and nothing is deleted, so no ``confirm``. + Arc refuses (409) readers, unhealthy nodes, the current holder and + non-leader nodes. The response's ``preemption_suppressed_seconds`` + says how long the choice is protected from automatic preemption. + Needs Arc 2026.09.2+. + """ + if not node_id: + raise ArcValidationError("node_id is required") + return await self._json( + "post", "/api/v1/cluster/compactor/assign", json={"node_id": node_id} + ) + + async def converge_voters( + self, + dry_run: bool = True, + allow_single_voter: bool = False, + confirm: bool = False, + ) -> Any: + """Converge the Raft voter set onto the role-based rule (admin). + + Plans by default. With ``dry_run=False`` it revokes votes the nodes' + roles do not grant, which changes Raft membership, so it also needs + ``confirm=True``. A partial converge answers 409 with the result. + Needs Arc 2026.09.2+. + + Args: + dry_run: Plan only (default). + allow_single_voter: Permit converging down to one voter. + confirm: Required when ``dry_run`` is False. + + Raises: + ArcValidationError: ``dry_run`` is False and ``confirm`` is not True. + """ + if not dry_run: + _require_confirm(confirm, "change the Raft voter set") + body: dict[str, Any] = {"dry_run": dry_run} + if allow_single_voter: + body["allow_single_voter"] = True + return await self._json("post", "/api/v1/cluster/voters/converge", json=body) + + async def stream_claims(self) -> Any: + """List stream-takeover writer processes, claims and departed writers (admin). + + Read from this node's manifest. Needs Arc 2027.01.1+. + """ + return await self._json("get", "/api/v1/cluster/stream-claims") + + async def release_stream_claim( + self, + instance: Union[str, int], + epoch: int, + abandon: bool = False, + confirm: bool = False, + ) -> Any: + """Release an unfinished stream claim its claimant cannot finish (admin, leader only). + + Another writer (its claimant too) may then take the instance over. + With ``abandon=True`` the claim is finished instead: batches it has + not published are left to their origin and lost when the origin is + gone, so ``abandon`` requires ``confirm=True``. Needs Arc 2027.01.1+. + + Args: + instance: The 16-digit hex id :meth:`stream_claims` lists (an int + is formatted as that hex id). + epoch: The claim's epoch, so a later claim is not released by + mistake. + abandon: Finish the claim instead of releasing it. + confirm: Required when ``abandon`` is True. + + Raises: + ArcValidationError: Bad ``instance``/``epoch``, or ``abandon`` + without ``confirm``. + """ + segment = _instance_segment(instance) + if isinstance(epoch, bool) or not isinstance(epoch, int) or epoch < 1: + raise ArcValidationError("epoch must be a positive integer") + if abandon: + _require_confirm(confirm, "abandon a stream claim") + body: dict[str, Any] = {"epoch": epoch} + if abandon: + body["abandon"] = True + return await self._json( + "post", f"/api/v1/cluster/stream-claims/{segment}/release", json=body + ) + + async def abort_compaction_job(self, job: str, confirm: bool = False) -> Any: + """End a wedged compaction job that waits for an operator (admin). + + Meant for an operator who has checked the job's data: its entry is + dropped and its staged object removed; the manifest and the bucket's + committed files are not changed. ``GET /api/v1/cluster`` lists the + candidates as ``compaction_commit.wedged_jobs``. Arc answers 404 for + an unknown job and 409 for one that is not wedged. + Needs Arc 2027.01.1+. + + Raises: + ArcValidationError: ``confirm`` is not True. + """ + segment = _segment(job, "job") + _require_confirm(confirm, "abort a compaction job") + return await self._json("post", f"/api/v1/cluster/compaction-jobs/{segment}/abort") + + async def policies(self, kind: Optional[str] = None) -> Any: + """List each node's own retention and/or CQ policies with their hashes (admin). + + Run on the Raft leader (others answer 409 naming it). Pass the hash + to :meth:`adopt_policies`. Needs Arc 2027.01.1+. + + Args: + kind: ``"retention"`` or ``"cq"``; both by default. + """ + _check_kind(kind) + params = {"kind": kind} if kind else None + return await self._json("get", "/api/v1/cluster/policies", params=params) + + async def adopt_policies(self, kind: str, node: str, hash: str, confirm: bool = False) -> Any: + """Choose the node whose policies of ``kind`` the cluster adopts (admin, leader only). + + The leader adopts them only if ``node`` still holds exactly the set + ``hash`` names. Every other node's policies of that kind then stop + running for good (kept as dormant rows) and IDs may change, so this + requires ``confirm=True``. Answers 202; ``GET /api/v1/cluster`` shows + the outcome under ``cluster_policies``. Needs Arc 2027.01.1+. + + Raises: + ArcValidationError: Bad ``kind``, missing ``node``/``hash``, or + ``confirm`` is not True. + ArcError: Clustering is not enabled (Arc answers 200 with + ``success: false`` rather than an error status). + """ + _check_kind(kind, required=True) + if not node or not hash or not hash.strip(): + raise ArcValidationError("node and hash are required (see policies())") + _require_confirm(confirm, "adopt cluster policies") + result = await self._json( + "post", + "/api/v1/cluster/policies/adopt", + json={"kind": kind, "node": node, "hash": hash}, + ) + return _check_adopted(result) diff --git a/src/arc_client/management/cluster.py b/src/arc_client/management/cluster.py index 554d173..6806681 100644 --- a/src/arc_client/management/cluster.py +++ b/src/arc_client/management/cluster.py @@ -1,18 +1,64 @@ """Synchronous cluster-management client for Arc (enterprise). -Wraps ``/api/v1/cluster``. Reads are available to any valid token; ``files`` and -node removal require an admin token. When clustering is disabled the server -returns 200 with ``{"enabled": false, ...}`` rather than an error. +Wraps ``/api/v1/cluster``. The status/node/health reads are available to any +valid token; every other route (files, node removal, the operator controls +below) requires an admin token. None of them is gated on a license feature +beyond clustering itself. When clustering is disabled the reads (and +``policies``) return 200 with ``{"enabled": false, ...}``; the operator +controls answer 409 instead, and :meth:`ClusterClient.adopt_policies` raises +:class:`~arc_client.exceptions.ArcError`, so "nothing happened" never reads +as success. + +Responses are returned as raw dicts: they are operator diagnostics whose +fields Arc extends between releases, not a promised contract. Controls that +lose data or leave the cluster without a role holder require ``confirm=True`` +and raise :class:`~arc_client.exceptions.ArcValidationError` otherwise. """ from __future__ import annotations -from typing import Any, Optional +from typing import Any, Optional, Union +from urllib.parse import quote from arc_client.config import ClientConfig -from arc_client.exceptions import ArcError +from arc_client.exceptions import ArcError, ArcValidationError from arc_client.http.sync_http import SyncHTTPClient +POLICY_KINDS = ("retention", "cq") + + +def _segment(value: str, name: str) -> str: + if not isinstance(value, str) or not value: + raise ArcValidationError(f"{name} is required") + return quote(value, safe="") + + +def _instance_segment(instance: Union[str, int]) -> str: + if isinstance(instance, bool): + raise ArcValidationError("instance must be the hex id stream_claims() lists") + if isinstance(instance, int): + if instance <= 0: + raise ArcValidationError("instance must be a positive id") + return f"{instance:016x}" + return _segment(instance, "instance") + + +def _check_kind(kind: Optional[str], required: bool = False) -> None: + if (required or kind is not None) and kind not in POLICY_KINDS: + raise ArcValidationError(f"kind must be one of {POLICY_KINDS}, got {kind!r}") + + +def _check_adopted(result: Any) -> Any: + if isinstance(result, dict) and result.get("success") is False: + reason = result.get("error") or result.get("reason") or "clustering is not enabled" + raise ArcError(f"Policy adoption not accepted: {reason}") + return result + + +def _require_confirm(confirm: bool, what: str) -> None: + if not confirm: + raise ArcValidationError(f"Refusing to {what} without confirm=True.") + class ClusterClient: """Inspect and manage the Arc cluster (enterprise).""" @@ -65,6 +111,191 @@ def files( } return self._json("get", "/api/v1/cluster/files", params=params or None) + def delete_file(self, path: str, confirm: bool = False, reason: Optional[str] = None) -> Any: + """Remove a file from the cluster manifest and delete it cluster-wide (admin). + + Destructive: on local storage every node unlinks the file; on shared + object storage the object is orphaned until the reconciliation + sweeper removes it. ``path`` is matched exactly, so malformed keys + that retention cannot remove can still be targeted. Needs Arc + 2026.09.2+. + + Args: + path: Manifest path of the file (max 4096 bytes). + confirm: Must be True. + reason: Recorded in the audit log and Raft entry (max 256 + characters; Arc defaults to ``"operator"``). + + Raises: + ArcValidationError: ``confirm`` is not True or ``path`` is empty. + ArcNotFoundError: The path is not in the cluster manifest. + """ + if not path: + raise ArcValidationError("path is required") + _require_confirm(confirm, "delete a cluster file") + params = {"path": path, "confirm": "true"} + if reason: + params["reason"] = reason + return self._json("delete", "/api/v1/cluster/files", params=params) + def remove_node(self, node_id: str) -> Any: """Remove a node from the cluster via Raft (admin).""" return self._json("delete", f"/api/v1/cluster/nodes/{node_id}") + + def demote_writer(self, node_id: str, confirm: bool = False) -> Any: + """Hand the primary-writer role off ``node_id`` so the cluster elects another (admin). + + Use it to drain a writer, or to recover a cluster without automatic + failover whose primary is gone for good. Not gated on the + writer-failover license feature. If no other writer can take the + role, the response has ``new_primary: None`` and the cluster has no + primary writer until one can, hence ``confirm``. Arc answers 409 when + ``node_id`` is not the current primary or this node is not the Raft + leader. Needs Arc 2026.09.2+. + + Raises: + ArcValidationError: ``confirm`` is not True. + """ + segment = _segment(node_id, "node_id") + _require_confirm(confirm, "demote the primary writer") + return self._json("post", f"/api/v1/cluster/writers/{segment}/demote") + + def assign_compactor(self, node_id: str) -> Any: + """Hand the compactor lease to ``node_id`` (admin). + + The target is explicit and nothing is deleted, so no ``confirm``. + Arc refuses (409) readers, unhealthy nodes, the current holder and + non-leader nodes. The response's ``preemption_suppressed_seconds`` + says how long the choice is protected from automatic preemption. + Needs Arc 2026.09.2+. + """ + if not node_id: + raise ArcValidationError("node_id is required") + return self._json("post", "/api/v1/cluster/compactor/assign", json={"node_id": node_id}) + + def converge_voters( + self, + dry_run: bool = True, + allow_single_voter: bool = False, + confirm: bool = False, + ) -> Any: + """Converge the Raft voter set onto the role-based rule (admin). + + Plans by default. With ``dry_run=False`` it revokes votes the nodes' + roles do not grant, which changes Raft membership, so it also needs + ``confirm=True``. A partial converge answers 409 with the result. + Needs Arc 2026.09.2+. + + Args: + dry_run: Plan only (default). + allow_single_voter: Permit converging down to one voter. + confirm: Required when ``dry_run`` is False. + + Raises: + ArcValidationError: ``dry_run`` is False and ``confirm`` is not True. + """ + if not dry_run: + _require_confirm(confirm, "change the Raft voter set") + body: dict[str, Any] = {"dry_run": dry_run} + if allow_single_voter: + body["allow_single_voter"] = True + return self._json("post", "/api/v1/cluster/voters/converge", json=body) + + def stream_claims(self) -> Any: + """List stream-takeover writer processes, claims and departed writers (admin). + + Read from this node's manifest. Needs Arc 2027.01.1+. + """ + return self._json("get", "/api/v1/cluster/stream-claims") + + def release_stream_claim( + self, + instance: Union[str, int], + epoch: int, + abandon: bool = False, + confirm: bool = False, + ) -> Any: + """Release an unfinished stream claim its claimant cannot finish (admin, leader only). + + Another writer (its claimant too) may then take the instance over. + With ``abandon=True`` the claim is finished instead: batches it has + not published are left to their origin and lost when the origin is + gone, so ``abandon`` requires ``confirm=True``. Needs Arc 2027.01.1+. + + Args: + instance: The 16-digit hex id :meth:`stream_claims` lists (an int + is formatted as that hex id). + epoch: The claim's epoch, so a later claim is not released by + mistake. + abandon: Finish the claim instead of releasing it. + confirm: Required when ``abandon`` is True. + + Raises: + ArcValidationError: Bad ``instance``/``epoch``, or ``abandon`` + without ``confirm``. + """ + segment = _instance_segment(instance) + if isinstance(epoch, bool) or not isinstance(epoch, int) or epoch < 1: + raise ArcValidationError("epoch must be a positive integer") + if abandon: + _require_confirm(confirm, "abandon a stream claim") + body: dict[str, Any] = {"epoch": epoch} + if abandon: + body["abandon"] = True + return self._json("post", f"/api/v1/cluster/stream-claims/{segment}/release", json=body) + + def abort_compaction_job(self, job: str, confirm: bool = False) -> Any: + """End a wedged compaction job that waits for an operator (admin). + + Meant for an operator who has checked the job's data: its entry is + dropped and its staged object removed; the manifest and the bucket's + committed files are not changed. ``GET /api/v1/cluster`` lists the + candidates as ``compaction_commit.wedged_jobs``. Arc answers 404 for + an unknown job and 409 for one that is not wedged. + Needs Arc 2027.01.1+. + + Raises: + ArcValidationError: ``confirm`` is not True. + """ + segment = _segment(job, "job") + _require_confirm(confirm, "abort a compaction job") + return self._json("post", f"/api/v1/cluster/compaction-jobs/{segment}/abort") + + def policies(self, kind: Optional[str] = None) -> Any: + """List each node's own retention and/or CQ policies with their hashes (admin). + + Run on the Raft leader (others answer 409 naming it). Pass the hash + to :meth:`adopt_policies`. Needs Arc 2027.01.1+. + + Args: + kind: ``"retention"`` or ``"cq"``; both by default. + """ + _check_kind(kind) + params = {"kind": kind} if kind else None + return self._json("get", "/api/v1/cluster/policies", params=params) + + def adopt_policies(self, kind: str, node: str, hash: str, confirm: bool = False) -> Any: + """Choose the node whose policies of ``kind`` the cluster adopts (admin, leader only). + + The leader adopts them only if ``node`` still holds exactly the set + ``hash`` names. Every other node's policies of that kind then stop + running for good (kept as dormant rows) and IDs may change, so this + requires ``confirm=True``. Answers 202; ``GET /api/v1/cluster`` shows + the outcome under ``cluster_policies``. Needs Arc 2027.01.1+. + + Raises: + ArcValidationError: Bad ``kind``, missing ``node``/``hash``, or + ``confirm`` is not True. + ArcError: Clustering is not enabled (Arc answers 200 with + ``success: false`` rather than an error status). + """ + _check_kind(kind, required=True) + if not node or not hash or not hash.strip(): + raise ArcValidationError("node and hash are required (see policies())") + _require_confirm(confirm, "adopt cluster policies") + result = self._json( + "post", + "/api/v1/cluster/policies/adopt", + json={"kind": kind, "node": node, "hash": hash}, + ) + return _check_adopted(result) diff --git a/tests/unit/http/test_cluster_contracts.py b/tests/unit/http/test_cluster_contracts.py new file mode 100644 index 0000000..100a457 --- /dev/null +++ b/tests/unit/http/test_cluster_contracts.py @@ -0,0 +1,214 @@ +"""Cluster operator-control wire contracts shared by sync and async public clients.""" + +from __future__ import annotations + +import inspect +import json +from contextlib import AsyncExitStack + +import pytest + +from arc_client import ArcClient, AsyncArcClient +from arc_client.exceptions import ( + ArcError, + ArcNotFoundError, + ArcPermissionError, + ArcValidationError, +) + + +async def call(client, method, *args, **kwargs): + result = getattr(client.cluster, method)(*args, **kwargs) + return await result if inspect.isawaitable(result) else result + + +@pytest.fixture(params=[False, True], ids=["sync", "async"]) +def async_mode(request): + return request.param + + +async def enter_client(stack, async_mode): + if async_mode: + return await stack.enter_async_context(AsyncArcClient(token="t", max_retries=0)) + return stack.enter_context(ArcClient(token="t", max_retries=0)) + + +def body_of(request): + return json.loads(request.content) if request.content else None + + +# (method, args, kwargs, http method, path, query params, json body) +CASES = [ + ( + "delete_file", + ("db/cpu/f.parquet",), + {"confirm": True}, + "DELETE", + "/api/v1/cluster/files", + {"path": "db/cpu/f.parquet", "confirm": "true"}, + None, + ), + ( + "delete_file", + ("db/cpu/f.parquet",), + {"confirm": True, "reason": "corrupt"}, + "DELETE", + "/api/v1/cluster/files", + {"path": "db/cpu/f.parquet", "confirm": "true", "reason": "corrupt"}, + None, + ), + ( + "demote_writer", + ("writer-1",), + {"confirm": True}, + "POST", + "/api/v1/cluster/writers/writer-1/demote", + {}, + None, + ), + ( + "assign_compactor", + ("node-2",), + {}, + "POST", + "/api/v1/cluster/compactor/assign", + {}, + {"node_id": "node-2"}, + ), + ( + "converge_voters", + (), + {}, + "POST", + "/api/v1/cluster/voters/converge", + {}, + {"dry_run": True}, + ), + ( + "converge_voters", + (), + {"dry_run": False, "allow_single_voter": True, "confirm": True}, + "POST", + "/api/v1/cluster/voters/converge", + {}, + {"dry_run": False, "allow_single_voter": True}, + ), + ("stream_claims", (), {}, "GET", "/api/v1/cluster/stream-claims", {}, None), + ( + "release_stream_claim", + ("00000000000000ab", 3), + {}, + "POST", + "/api/v1/cluster/stream-claims/00000000000000ab/release", + {}, + {"epoch": 3}, + ), + ( + "release_stream_claim", + (0xAB, 3), + {"abandon": True, "confirm": True}, + "POST", + "/api/v1/cluster/stream-claims/00000000000000ab/release", + {}, + {"epoch": 3, "abandon": True}, + ), + ( + "abort_compaction_job", + ("job-9",), + {"confirm": True}, + "POST", + "/api/v1/cluster/compaction-jobs/job-9/abort", + {}, + None, + ), + ("policies", (), {}, "GET", "/api/v1/cluster/policies", {}, None), + ("policies", (), {"kind": "cq"}, "GET", "/api/v1/cluster/policies", {"kind": "cq"}, None), + ( + "adopt_policies", + ("retention", "node-1", "abc123"), + {"confirm": True}, + "POST", + "/api/v1/cluster/policies/adopt", + {}, + {"kind": "retention", "node": "node-1", "hash": "abc123"}, + ), +] + + +@pytest.mark.parametrize( + ("method", "args", "kwargs", "verb", "path", "params", "body"), + CASES, + ids=[f"{c[0]}-{i}" for i, c in enumerate(CASES)], +) +async def test_wire_contract( + httpx_mock, async_mode, method, args, kwargs, verb, path, params, body +): + httpx_mock.add_response(method=verb, json={"success": True}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert (await call(client, method, *args, **kwargs))["success"] is True + request = httpx_mock.get_request() + assert request.method == verb + assert request.url.path == path + assert dict(request.url.params) == params + assert body_of(request) == body + + +@pytest.mark.parametrize( + ("method", "args", "kwargs"), + [ + ("delete_file", ("db/cpu/f.parquet",), {}), + ("delete_file", ("",), {"confirm": True}), + ("demote_writer", ("writer-1",), {}), + ("demote_writer", ("",), {"confirm": True}), + ("assign_compactor", ("",), {}), + ("converge_voters", (), {"dry_run": False}), + ("release_stream_claim", ("ab", 3), {"abandon": True}), + ("release_stream_claim", ("ab", 0), {}), + ("release_stream_claim", (0, 3), {}), + ("release_stream_claim", ("", 3), {}), + ("abort_compaction_job", ("job-9",), {}), + ("policies", (), {"kind": "both"}), + ("adopt_policies", ("retention", "node-1", "abc"), {}), + ("adopt_policies", ("tiering", "node-1", "abc"), {"confirm": True}), + ("adopt_policies", ("cq", "", "abc"), {"confirm": True}), + ("adopt_policies", ("cq", "node-1", " "), {"confirm": True}), + ], +) +async def test_refused_before_request(httpx_mock, async_mode, method, args, kwargs): + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcValidationError): + await call(client, method, *args, **kwargs) + assert httpx_mock.get_requests() == [] + + +async def test_node_id_is_path_escaped(httpx_mock, async_mode): + httpx_mock.add_response(method="POST", json={"success": True}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + await call(client, "demote_writer", "a/b", confirm=True) + assert httpx_mock.get_request().url.raw_path == b"/api/v1/cluster/writers/a%2Fb/demote" + + +async def test_adopt_on_standalone_raises(httpx_mock, async_mode): + httpx_mock.add_response( + method="POST", json={"success": False, "enabled": False, "mode": "standalone"} + ) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcError): + await call(client, "adopt_policies", "cq", "n1", "h", confirm=True) + + +@pytest.mark.parametrize( + ("status", "exc"), [(403, ArcPermissionError), (404, ArcNotFoundError), (409, ArcError)] +) +async def test_server_refusals_surface(httpx_mock, async_mode, status, exc): + httpx_mock.add_response( + method="POST", status_code=status, json={"success": False, "error": "x"} + ) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(exc): + await call(client, "abort_compaction_job", "job-9", confirm=True) From 045bdcf0756aa5996398350af10b7dd92703dd64 Mon Sep 17 00:00:00 2001 From: Ignacio Van Droogenbroeck <64545348+xe-nvdk@users.noreply.github.com> Date: Mon, 2 Nov 2026 08:49:39 -0600 Subject: [PATCH 10/20] feat(edge-sync): add hub spoke/import and spoke sync administration (#14) Add client.edge_sync (sync and async) covering every operator route: - hub: /api/v1/sync-spokes (list/get/register/rotate/enable/disable/ delete) and /api/v1/bundle-import (import, history) - spoke: /api/v1/spoke-sync (status, ledger, run, requeue, dismiss, export, revert, ack) rotate_spoke_secret, delete_spoke and dismiss_failed require confirm=True. Spokes are returned as Spoke/SpokeSecret models; other responses stay raw dicts. The HMAC spoke-to-hub transport routes (/api/v1/sync/file, /api/v1/sync/reconcile) are intentionally not wrapped. --- src/arc_client/async_client.py | 10 + src/arc_client/client.py | 10 + src/arc_client/management/__init__.py | 6 +- src/arc_client/management/async_edge_sync.py | 257 +++++++++++ src/arc_client/management/edge_sync.py | 311 +++++++++++++ src/arc_client/models/__init__.py | 3 + src/arc_client/models/edge_sync.py | 37 ++ tests/unit/http/test_edge_sync_contracts.py | 437 +++++++++++++++++++ 8 files changed, 1070 insertions(+), 1 deletion(-) create mode 100644 src/arc_client/management/async_edge_sync.py create mode 100644 src/arc_client/management/edge_sync.py create mode 100644 src/arc_client/models/edge_sync.py create mode 100644 tests/unit/http/test_edge_sync_contracts.py diff --git a/src/arc_client/async_client.py b/src/arc_client/async_client.py index 99e0485..6b70b59 100644 --- a/src/arc_client/async_client.py +++ b/src/arc_client/async_client.py @@ -91,6 +91,7 @@ def __init__( self._tiering: Any = None self._backup: Any = None self._mqtt: Any = None + self._edge_sync: Any = None self._diagnostics: Any = None self._reconciliation: Any = None @@ -278,6 +279,15 @@ def diagnostics(self) -> Any: self._diagnostics = AsyncDiagnosticsClient(self._get_http(), self._config) return self._diagnostics + @property + def edge_sync(self) -> Any: + """Get the async edge-sync administration client (hub spokes/import, spoke sync/export).""" + if self._edge_sync is None: + from arc_client.management.async_edge_sync import AsyncEdgeSyncClient + + self._edge_sync = AsyncEdgeSyncClient(self._get_http(), self._config) + return self._edge_sync + async def health(self) -> HealthResponse: """Check server health. diff --git a/src/arc_client/client.py b/src/arc_client/client.py index f2c17b1..24dba72 100644 --- a/src/arc_client/client.py +++ b/src/arc_client/client.py @@ -91,6 +91,7 @@ def __init__( self._tiering: Any = None self._backup: Any = None self._mqtt: Any = None + self._edge_sync: Any = None self._diagnostics: Any = None self._reconciliation: Any = None @@ -276,6 +277,15 @@ def diagnostics(self) -> Any: self._diagnostics = DiagnosticsClient(self._get_http(), self._config) return self._diagnostics + @property + def edge_sync(self) -> Any: + """Get the edge-sync administration client (hub spokes/import, spoke sync/export).""" + if self._edge_sync is None: + from arc_client.management.edge_sync import EdgeSyncClient + + self._edge_sync = EdgeSyncClient(self._get_http(), self._config) + return self._edge_sync + def health(self) -> HealthResponse: """Check server health. diff --git a/src/arc_client/management/__init__.py b/src/arc_client/management/__init__.py index ee760fc..f155a87 100644 --- a/src/arc_client/management/__init__.py +++ b/src/arc_client/management/__init__.py @@ -2,7 +2,7 @@ Retention, continuous queries, delete, databases, and the enterprise surface (RBAC, governance, audit, query management, cluster, compaction, tiering, -backup, MQTT). +backup, MQTT) and operations (reconciliation, diagnostics, edge sync). """ from arc_client.management.async_audit import AsyncAuditClient @@ -13,6 +13,7 @@ from arc_client.management.async_databases import AsyncDatabaseClient from arc_client.management.async_delete import AsyncDeleteClient from arc_client.management.async_diagnostics import AsyncDiagnosticsClient +from arc_client.management.async_edge_sync import AsyncEdgeSyncClient from arc_client.management.async_governance import AsyncGovernanceClient from arc_client.management.async_mqtt import AsyncMQTTClient from arc_client.management.async_query_management import AsyncQueryManagementClient @@ -28,6 +29,7 @@ from arc_client.management.databases import DatabaseClient from arc_client.management.delete import DeleteClient from arc_client.management.diagnostics import DiagnosticsClient +from arc_client.management.edge_sync import EdgeSyncClient from arc_client.management.governance import GovernanceClient from arc_client.management.mqtt import MQTTClient from arc_client.management.query_management import QueryManagementClient @@ -69,4 +71,6 @@ "AsyncReconciliationClient", "DiagnosticsClient", "AsyncDiagnosticsClient", + "EdgeSyncClient", + "AsyncEdgeSyncClient", ] diff --git a/src/arc_client/management/async_edge_sync.py b/src/arc_client/management/async_edge_sync.py new file mode 100644 index 0000000..4a46e5e --- /dev/null +++ b/src/arc_client/management/async_edge_sync.py @@ -0,0 +1,257 @@ +"""Asynchronous edge-sync administration client for Arc. + +Edge sync moves data from edge nodes (*spokes*) to a central node (the +*hub*), over the network or on removable media ("air-gap bundles"). Methods +here are grouped by which node they must be sent to; a route that is not +enabled on the node answers 404. + +Hub (``internal/api/edgesync_admin.go``, ``edgesync_import.go``): + +* ``/api/v1/sync-spokes`` — spoke registrations. Mounted when + ``edge_sync.enabled`` or ``edge_sync.import.enabled``. +* ``/api/v1/bundle-import`` — air-gap import. Mounted when + ``edge_sync.import.enabled``. + +Spoke (``internal/api/edgesync_spoke.go``): + +* ``/api/v1/spoke-sync`` — manual sync, ledger and bundle export. Mounted + when ``edge_sync.spoke.enabled`` or ``edge_sync.spoke.bundle.enabled``. + Routes needing the transport that is not enabled answer 503. + +Every route requires an admin token. None is license-gated (only Arc's +automatic spoke scheduler is). The spoke-to-hub transport routes +(``/api/v1/sync/file``, ``/api/v1/sync/reconcile``) are HMAC-authenticated +per spoke and called by Arc itself; they are deliberately not wrapped. + +Spoke registrations are returned as :class:`~arc_client.models.edge_sync.Spoke` +models; the other responses are raw dicts (operator diagnostics). +""" + +from __future__ import annotations + +from typing import Any, List, Optional + +from arc_client.config import ClientConfig +from arc_client.exceptions import ArcError +from arc_client.http.async_http import AsyncHTTPClient +from arc_client.management.edge_sync import ( + BUNDLE_IMPORT, + DELETE_REFUSAL, + DISMISS_REFUSAL, + ROTATE_REFUSAL, + SPOKE_SYNC, + SPOKES, + _ledger_params, + _limit, + _path_body, + _require_confirm, + _segment, + _selector, + _spokes, +) +from arc_client.models.edge_sync import Spoke, SpokeSecret + + +class AsyncEdgeSyncClient: + """Administer edge sync on a hub or a spoke (admin token).""" + + def __init__(self, http: AsyncHTTPClient, config: ClientConfig) -> None: + self._http = http + self._config = config + + async def _json(self, method: str, path: str, **kwargs: Any) -> Any: + try: + response = await getattr(self._http, method)(path, **kwargs) + return response.json() + except ArcError: + raise + except Exception as e: + raise ArcError(f"Edge sync request failed ({method.upper()} {path}): {e}") from e + + async def _dict(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]: + data = await self._json(method, path, **kwargs) + if not isinstance(data, dict): + raise ArcError(f"Unexpected response from {path}: {data!r}") + return data + + # Hub: spoke registrations ------------------------------------------ + + async def list_spokes(self) -> List[Spoke]: + """List spokes registered with this hub (hub).""" + return _spokes(await self._json("get", f"{SPOKES}/")) + + async def get_spoke(self, spoke_id: str) -> Spoke: + """Get one registered spoke (hub). 404 if not registered.""" + return Spoke.model_validate( + await self._dict("get", f"{SPOKES}/{_segment('spoke_id', spoke_id)}") + ) + + async def register_spoke(self, spoke_id: str, name: str = "") -> SpokeSecret: + """Register a spoke and get its HMAC secret (hub). + + The secret is returned only this once; store it before discarding the + result. 409 if the ID is already registered (Arc refuses to reissue). + """ + _segment("spoke_id", spoke_id) + body = {"spoke_id": spoke_id, "name": name} + return SpokeSecret.model_validate(await self._dict("post", f"{SPOKES}/", json=body)) + + async def rotate_spoke_secret(self, spoke_id: str, confirm: bool = False) -> SpokeSecret: + """Replace a spoke's secret (hub). Requires ``confirm=True``. + + The previous secret stops authenticating at once, so a live spoke is + locked out until it is reconfigured with the new one. + """ + segment = _segment("spoke_id", spoke_id) + _require_confirm(confirm, ROTATE_REFUSAL) + return SpokeSecret.model_validate(await self._dict("post", f"{SPOKES}/{segment}/rotate")) + + async def enable_spoke(self, spoke_id: str) -> dict[str, Any]: + """Allow a disabled spoke to sync again (hub).""" + return await self._dict("post", f"{SPOKES}/{_segment('spoke_id', spoke_id)}/enable") + + async def disable_spoke(self, spoke_id: str) -> dict[str, Any]: + """Stop accepting syncs from a spoke (hub). + + Reversible with :meth:`enable_spoke`; counters, history and the secret + survive, so no ``confirm`` is required. + """ + return await self._dict("post", f"{SPOKES}/{_segment('spoke_id', spoke_id)}/disable") + + async def delete_spoke(self, spoke_id: str, confirm: bool = False) -> dict[str, Any]: + """Remove a spoke's registration (hub). Requires ``confirm=True``. + + Files already received from the spoke are kept. Re-adding the spoke + means registering it again and reprovisioning a secret. + """ + segment = _segment("spoke_id", spoke_id) + _require_confirm(confirm, DELETE_REFUSAL) + return await self._dict("delete", f"{SPOKES}/{segment}") + + # Hub: air-gap import ------------------------------------------------ + + async def import_bundle(self, path: str) -> dict[str, Any]: + """Import an air-gap bundle from a directory on the hub (hub). + + ``path`` is resolved on the hub and must be inside + ``edge_sync.import.allowed_dirs``. The bundle is verified in full + before anything is written; paths holding different content are + reported in ``conflicts`` and never overwritten. + + Synchronous and possibly long (Arc allows up to 4 hours): construct + the client with a large ``timeout``. If the client gives up first the + import keeps running, and a repeat call answers 409. + + Raises: + ArcServerError: 409 when another import is running or the bundle + was already imported (``response_body["reason"]`` says which), + 422 when the bundle is invalid. + """ + return await self._dict( + "post", f"{BUNDLE_IMPORT}/", json=_path_body(path, "the bundle directory to import") + ) + + async def import_history(self, spoke_id: str, limit: Optional[int] = None) -> dict[str, Any]: + """List bundles this hub has imported from a spoke, newest first (hub). + + Args: + limit: 1-1000; server default 100. + """ + segment = _segment("spoke_id", spoke_id) + return await self._dict("get", f"{BUNDLE_IMPORT}/history/{segment}", params=_limit(limit)) + + # Spoke: sync control and ledger ------------------------------------ + + async def spoke_status(self) -> dict[str, Any]: + """Get ledger counts and sync lag on this spoke (spoke). + + ``pending``, ``in_flight``, ``exported``, ``synced``, ``failed``, + ``skipped``, ``pending_bytes`` and, once anything has synced, + ``last_synced_at`` / ``seconds_since_last_sync``. + """ + return await self._dict("get", f"{SPOKE_SYNC}/status") + + async def spoke_ledger( + self, limit: Optional[int] = None, state: Optional[str] = None + ) -> dict[str, Any]: + """List ledger entries on this spoke (spoke). + + By default the unfinished entries (including failed ones). ``state`` + lists one state instead: ``pending``, ``in_flight``, ``synced``, + ``exported``, ``failed`` or ``skipped``. ``limit`` is 1-1000 (server + default 100). + """ + return await self._dict("get", f"{SPOKE_SYNC}/ledger", params=_ledger_params(limit, state)) + + async def run_sync(self) -> dict[str, Any]: + """Run one network sync pass now and wait for its result (spoke). + + Requires ``edge_sync.spoke.enabled`` (503 otherwise); 409 if a pass + is already running. Synchronous and possibly long (Arc allows up to + 2 hours): construct the client with a large ``timeout``. + """ + return await self._dict("post", f"{SPOKE_SYNC}/run") + + async def requeue_failed( + self, path: Optional[str] = None, all_failed: bool = False + ) -> dict[str, Any]: + """Return failed or dismissed entries to pending (spoke). + + Give exactly one of ``path`` or ``all_failed=True``. Returns + ``{"requeued": n}``; 404 when a named path matched nothing. + """ + return await self._dict( + "post", f"{SPOKE_SYNC}/ledger/requeue", json=_selector(path, all_failed) + ) + + async def dismiss_failed( + self, path: Optional[str] = None, all_failed: bool = False, confirm: bool = False + ) -> dict[str, Any]: + """Mark failed entries as operator-dismissed (spoke). Requires ``confirm=True``. + + Dismissed entries are no longer delivered and are pruned after the + ledger retention period; until then :meth:`requeue_failed` reverses + it. Give exactly one of ``path`` or ``all_failed=True``. Returns + ``{"dismissed": n}``. + """ + body = _selector(path, all_failed) + _require_confirm(confirm, DISMISS_REFUSAL) + return await self._dict("post", f"{SPOKE_SYNC}/ledger/dismiss", json=body) + + # Spoke: air-gap export --------------------------------------------- + + async def export_bundle(self, path: str, limit: Optional[int] = None) -> dict[str, Any]: + """Write pending files to an air-gap bundle under ``path`` (spoke). + + ``path`` is on the spoke and must be inside the configured allow-list. + ``limit`` caps the number of files (server maximum applies). Requires + ``edge_sync.spoke.bundle.enabled`` (503 otherwise). Exported files + become ``synced`` only after :meth:`apply_ack`. An empty backlog is + ``{"exported": false, "reason": "nothing to export"}``, not an error. + Possibly long (up to 2 hours): use a large client ``timeout``. + """ + body = _path_body(path, "the directory to write the bundle into") + if limit is not None: + body.update(_limit(limit, high=None) or {}) + return await self._dict("post", f"{SPOKE_SYNC}/export", json=body) + + async def revert_export(self, bundle_id: str) -> dict[str, Any]: + """Return a lost or undelivered bundle's files to pending (spoke). + + Scoped to one bundle; the files are carried again by a later bundle or + sync pass (the hub skips copies it already holds). Returns + ``{"bundle_id": ..., "reverted": n}``. + """ + segment = _segment("bundle_id", bundle_id) + return await self._dict("post", f"{SPOKE_SYNC}/export/{segment}/revert") + + async def apply_ack(self, path: str) -> dict[str, Any]: + """Apply the hub's acknowledgment from a returned bundle (spoke). + + Advances the files the hub confirmed from ``exported`` to ``synced``. + A bundle with no acknowledgment yet answers + ``{"applied": false, ...}``, not an error. + """ + return await self._dict( + "post", f"{SPOKE_SYNC}/ack", json=_path_body(path, "the returned bundle directory") + ) diff --git a/src/arc_client/management/edge_sync.py b/src/arc_client/management/edge_sync.py new file mode 100644 index 0000000..379debd --- /dev/null +++ b/src/arc_client/management/edge_sync.py @@ -0,0 +1,311 @@ +"""Synchronous edge-sync administration client for Arc. + +Edge sync moves data from edge nodes (*spokes*) to a central node (the +*hub*), over the network or on removable media ("air-gap bundles"). Methods +here are grouped by which node they must be sent to; a route that is not +enabled on the node answers 404. + +Hub (``internal/api/edgesync_admin.go``, ``edgesync_import.go``): + +* ``/api/v1/sync-spokes`` — spoke registrations. Mounted when + ``edge_sync.enabled`` or ``edge_sync.import.enabled``. +* ``/api/v1/bundle-import`` — air-gap import. Mounted when + ``edge_sync.import.enabled``. + +Spoke (``internal/api/edgesync_spoke.go``): + +* ``/api/v1/spoke-sync`` — manual sync, ledger and bundle export. Mounted + when ``edge_sync.spoke.enabled`` or ``edge_sync.spoke.bundle.enabled``. + Routes needing the transport that is not enabled answer 503. + +Every route requires an admin token. None is license-gated (only Arc's +automatic spoke scheduler is). The spoke-to-hub transport routes +(``/api/v1/sync/file``, ``/api/v1/sync/reconcile``) are HMAC-authenticated +per spoke and called by Arc itself; they are deliberately not wrapped. + +Spoke registrations are returned as :class:`~arc_client.models.edge_sync.Spoke` +models; the other responses are raw dicts (operator diagnostics). +""" + +from __future__ import annotations + +from typing import Any, List, Optional +from urllib.parse import quote + +from arc_client.config import ClientConfig +from arc_client.exceptions import ArcError, ArcValidationError +from arc_client.http.sync_http import SyncHTTPClient +from arc_client.models.edge_sync import Spoke, SpokeSecret + +SPOKES = "/api/v1/sync-spokes" +BUNDLE_IMPORT = "/api/v1/bundle-import" +SPOKE_SYNC = "/api/v1/spoke-sync" +LEDGER_STATES = ("pending", "in_flight", "synced", "exported", "failed", "skipped") + + +def _segment(name: str, value: str) -> str: + if not value: + raise ArcValidationError(f"{name} is required") + return quote(value, safe="") + + +def _limit(value: Optional[int], high: Optional[int] = 1000) -> Optional[dict[str, Any]]: + if value is None: + return None + if ( + isinstance(value, bool) + or not isinstance(value, int) + or value < 1 + or (high is not None and value > high) + ): + bound = f"from 1 to {high}" if high is not None else "of at least 1" + raise ArcValidationError(f"limit must be an integer {bound}") + return {"limit": value} + + +def _require_confirm(confirm: bool, message: str) -> None: + if not confirm: + raise ArcValidationError(message) + + +def _ledger_params(limit: Optional[int], state: Optional[str]) -> Optional[dict[str, Any]]: + params = dict(_limit(limit) or {}) + if state is not None: + if state not in LEDGER_STATES: + raise ArcValidationError(f"state must be one of {', '.join(LEDGER_STATES)}") + params["state"] = state + return params or None + + +def _selector(path: Optional[str], all_failed: bool) -> dict[str, Any]: + if bool(path) == bool(all_failed): + raise ArcValidationError("Provide exactly one of path or all_failed=True") + return {"path": path} if path else {"all": True} + + +def _path_body(path: str, what: str) -> dict[str, Any]: + if not path: + raise ArcValidationError(f"path is required: {what}") + return {"path": path} + + +def _spokes(data: Any) -> List[Spoke]: + spokes = data.get("spokes") if isinstance(data, dict) else None + if not isinstance(spokes, list): + raise ArcError(f"Unexpected spoke list response: {data!r}") + return [Spoke.model_validate(s) for s in spokes] + + +ROTATE_REFUSAL = ( + "Refusing to rotate without confirm=True. The current secret stops " + "authenticating immediately; the spoke cannot sync until reconfigured." +) +DELETE_REFUSAL = ( + "Refusing to delete the spoke registration without confirm=True. Its " + "counters and secret are removed and the spoke can no longer sync." +) +DISMISS_REFUSAL = ( + "Refusing to dismiss without confirm=True. Dismissed entries are no longer " + "delivered and are pruned after the ledger retention period." +) + + +class EdgeSyncClient: + """Administer edge sync on a hub or a spoke (admin token).""" + + def __init__(self, http: SyncHTTPClient, config: ClientConfig) -> None: + self._http = http + self._config = config + + def _json(self, method: str, path: str, **kwargs: Any) -> Any: + try: + response = getattr(self._http, method)(path, **kwargs) + return response.json() + except ArcError: + raise + except Exception as e: + raise ArcError(f"Edge sync request failed ({method.upper()} {path}): {e}") from e + + def _dict(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]: + data = self._json(method, path, **kwargs) + if not isinstance(data, dict): + raise ArcError(f"Unexpected response from {path}: {data!r}") + return data + + # Hub: spoke registrations ------------------------------------------ + + def list_spokes(self) -> List[Spoke]: + """List spokes registered with this hub (hub).""" + return _spokes(self._json("get", f"{SPOKES}/")) + + def get_spoke(self, spoke_id: str) -> Spoke: + """Get one registered spoke (hub). 404 if not registered.""" + return Spoke.model_validate(self._dict("get", f"{SPOKES}/{_segment('spoke_id', spoke_id)}")) + + def register_spoke(self, spoke_id: str, name: str = "") -> SpokeSecret: + """Register a spoke and get its HMAC secret (hub). + + The secret is returned only this once; store it before discarding the + result. 409 if the ID is already registered (Arc refuses to reissue). + """ + _segment("spoke_id", spoke_id) + body = {"spoke_id": spoke_id, "name": name} + return SpokeSecret.model_validate(self._dict("post", f"{SPOKES}/", json=body)) + + def rotate_spoke_secret(self, spoke_id: str, confirm: bool = False) -> SpokeSecret: + """Replace a spoke's secret (hub). Requires ``confirm=True``. + + The previous secret stops authenticating at once, so a live spoke is + locked out until it is reconfigured with the new one. + """ + segment = _segment("spoke_id", spoke_id) + _require_confirm(confirm, ROTATE_REFUSAL) + return SpokeSecret.model_validate(self._dict("post", f"{SPOKES}/{segment}/rotate")) + + def enable_spoke(self, spoke_id: str) -> dict[str, Any]: + """Allow a disabled spoke to sync again (hub).""" + return self._dict("post", f"{SPOKES}/{_segment('spoke_id', spoke_id)}/enable") + + def disable_spoke(self, spoke_id: str) -> dict[str, Any]: + """Stop accepting syncs from a spoke (hub). + + Reversible with :meth:`enable_spoke`; counters, history and the secret + survive, so no ``confirm`` is required. + """ + return self._dict("post", f"{SPOKES}/{_segment('spoke_id', spoke_id)}/disable") + + def delete_spoke(self, spoke_id: str, confirm: bool = False) -> dict[str, Any]: + """Remove a spoke's registration (hub). Requires ``confirm=True``. + + Files already received from the spoke are kept. Re-adding the spoke + means registering it again and reprovisioning a secret. + """ + segment = _segment("spoke_id", spoke_id) + _require_confirm(confirm, DELETE_REFUSAL) + return self._dict("delete", f"{SPOKES}/{segment}") + + # Hub: air-gap import ------------------------------------------------ + + def import_bundle(self, path: str) -> dict[str, Any]: + """Import an air-gap bundle from a directory on the hub (hub). + + ``path`` is resolved on the hub and must be inside + ``edge_sync.import.allowed_dirs``. The bundle is verified in full + before anything is written; paths holding different content are + reported in ``conflicts`` and never overwritten. + + Synchronous and possibly long (Arc allows up to 4 hours): construct + the client with a large ``timeout``. If the client gives up first the + import keeps running, and a repeat call answers 409. + + Raises: + ArcServerError: 409 when another import is running or the bundle + was already imported (``response_body["reason"]`` says which), + 422 when the bundle is invalid. + """ + return self._dict( + "post", f"{BUNDLE_IMPORT}/", json=_path_body(path, "the bundle directory to import") + ) + + def import_history(self, spoke_id: str, limit: Optional[int] = None) -> dict[str, Any]: + """List bundles this hub has imported from a spoke, newest first (hub). + + Args: + limit: 1-1000; server default 100. + """ + segment = _segment("spoke_id", spoke_id) + return self._dict("get", f"{BUNDLE_IMPORT}/history/{segment}", params=_limit(limit)) + + # Spoke: sync control and ledger ------------------------------------ + + def spoke_status(self) -> dict[str, Any]: + """Get ledger counts and sync lag on this spoke (spoke). + + ``pending``, ``in_flight``, ``exported``, ``synced``, ``failed``, + ``skipped``, ``pending_bytes`` and, once anything has synced, + ``last_synced_at`` / ``seconds_since_last_sync``. + """ + return self._dict("get", f"{SPOKE_SYNC}/status") + + def spoke_ledger( + self, limit: Optional[int] = None, state: Optional[str] = None + ) -> dict[str, Any]: + """List ledger entries on this spoke (spoke). + + By default the unfinished entries (including failed ones). ``state`` + lists one state instead: ``pending``, ``in_flight``, ``synced``, + ``exported``, ``failed`` or ``skipped``. ``limit`` is 1-1000 (server + default 100). + """ + return self._dict("get", f"{SPOKE_SYNC}/ledger", params=_ledger_params(limit, state)) + + def run_sync(self) -> dict[str, Any]: + """Run one network sync pass now and wait for its result (spoke). + + Requires ``edge_sync.spoke.enabled`` (503 otherwise); 409 if a pass + is already running. Synchronous and possibly long (Arc allows up to + 2 hours): construct the client with a large ``timeout``. + """ + return self._dict("post", f"{SPOKE_SYNC}/run") + + def requeue_failed( + self, path: Optional[str] = None, all_failed: bool = False + ) -> dict[str, Any]: + """Return failed or dismissed entries to pending (spoke). + + Give exactly one of ``path`` or ``all_failed=True``. Returns + ``{"requeued": n}``; 404 when a named path matched nothing. + """ + return self._dict("post", f"{SPOKE_SYNC}/ledger/requeue", json=_selector(path, all_failed)) + + def dismiss_failed( + self, path: Optional[str] = None, all_failed: bool = False, confirm: bool = False + ) -> dict[str, Any]: + """Mark failed entries as operator-dismissed (spoke). Requires ``confirm=True``. + + Dismissed entries are no longer delivered and are pruned after the + ledger retention period; until then :meth:`requeue_failed` reverses + it. Give exactly one of ``path`` or ``all_failed=True``. Returns + ``{"dismissed": n}``. + """ + body = _selector(path, all_failed) + _require_confirm(confirm, DISMISS_REFUSAL) + return self._dict("post", f"{SPOKE_SYNC}/ledger/dismiss", json=body) + + # Spoke: air-gap export --------------------------------------------- + + def export_bundle(self, path: str, limit: Optional[int] = None) -> dict[str, Any]: + """Write pending files to an air-gap bundle under ``path`` (spoke). + + ``path`` is on the spoke and must be inside the configured allow-list. + ``limit`` caps the number of files (server maximum applies). Requires + ``edge_sync.spoke.bundle.enabled`` (503 otherwise). Exported files + become ``synced`` only after :meth:`apply_ack`. An empty backlog is + ``{"exported": false, "reason": "nothing to export"}``, not an error. + Possibly long (up to 2 hours): use a large client ``timeout``. + """ + body = _path_body(path, "the directory to write the bundle into") + if limit is not None: + body.update(_limit(limit, high=None) or {}) + return self._dict("post", f"{SPOKE_SYNC}/export", json=body) + + def revert_export(self, bundle_id: str) -> dict[str, Any]: + """Return a lost or undelivered bundle's files to pending (spoke). + + Scoped to one bundle; the files are carried again by a later bundle or + sync pass (the hub skips copies it already holds). Returns + ``{"bundle_id": ..., "reverted": n}``. + """ + segment = _segment("bundle_id", bundle_id) + return self._dict("post", f"{SPOKE_SYNC}/export/{segment}/revert") + + def apply_ack(self, path: str) -> dict[str, Any]: + """Apply the hub's acknowledgment from a returned bundle (spoke). + + Advances the files the hub confirmed from ``exported`` to ``synced``. + A bundle with no acknowledgment yet answers + ``{"applied": false, ...}``, not an error. + """ + return self._dict( + "post", f"{SPOKE_SYNC}/ack", json=_path_body(path, "the returned bundle directory") + ) diff --git a/src/arc_client/models/__init__.py b/src/arc_client/models/__init__.py index f4cd526..4c3079b 100644 --- a/src/arc_client/models/__init__.py +++ b/src/arc_client/models/__init__.py @@ -21,6 +21,7 @@ ) from arc_client.models.delete import DeleteConfigResponse, DeleteResponse from arc_client.models.diagnostics import ReadinessStatus +from arc_client.models.edge_sync import Spoke, SpokeSecret from arc_client.models.import_ import ( ImportResult, ImportStats, @@ -61,6 +62,8 @@ "RetentionExecution", "RetentionPolicy", "RotateTokenResponse", + "Spoke", + "SpokeSecret", "TokenInfo", "TokenListResponse", "VerifyResponse", diff --git a/src/arc_client/models/edge_sync.py b/src/arc_client/models/edge_sync.py new file mode 100644 index 0000000..25a0ad8 --- /dev/null +++ b/src/arc_client/models/edge_sync.py @@ -0,0 +1,37 @@ +"""Models for edge-sync spoke administration (hub side).""" + +from __future__ import annotations + +from datetime import datetime +from typing import Optional + +from pydantic import BaseModel, ConfigDict + + +class Spoke(BaseModel): + """A spoke registered with this hub. Never carries the secret.""" + + model_config = ConfigDict(extra="allow") + + spoke_id: str + name: str = "" + enabled: bool = True + files_received: int = 0 + bytes_received: int = 0 + registered_at: Optional[datetime] = None + last_seen_at: Optional[datetime] = None + + +class SpokeSecret(BaseModel): + """A spoke's HMAC secret, returned once by register and rotate. + + Arc cannot show the secret again. Store it before discarding this object; + a lost secret can only be replaced by rotating. + """ + + model_config = ConfigDict(extra="allow") + + spoke_id: str + secret: str + name: Optional[str] = None + warning: Optional[str] = None diff --git a/tests/unit/http/test_edge_sync_contracts.py b/tests/unit/http/test_edge_sync_contracts.py new file mode 100644 index 0000000..06446ff --- /dev/null +++ b/tests/unit/http/test_edge_sync_contracts.py @@ -0,0 +1,437 @@ +"""Edge-sync administration wire contracts. + +Arc sources: internal/api/edgesync_admin.go (hub spokes), +edgesync_import.go (hub bundle import), edgesync_spoke.go (spoke controls). +""" + +from __future__ import annotations + +import inspect +import json +from contextlib import AsyncExitStack +from typing import Any + +import pytest +from pytest_httpx import HTTPXMock + +from arc_client import ArcClient, AsyncArcClient +from arc_client.exceptions import ( + ArcError, + ArcNotFoundError, + ArcPermissionError, + ArcServerError, + ArcValidationError, +) +from arc_client.models.edge_sync import Spoke, SpokeSecret + +ROOT = "http://localhost:8000" +SPOKES = f"{ROOT}/api/v1/sync-spokes" +IMPORT = f"{ROOT}/api/v1/bundle-import" +SPOKE_SYNC = f"{ROOT}/api/v1/spoke-sync" +SPOKE = { + "spoke_id": "rocket-01", + "name": "Rocket", + "enabled": True, + "files_received": 4, + "bytes_received": 2048, + "registered_at": "2027-01-05T10:00:00Z", + "last_seen_at": "2027-01-06T10:00:00Z", +} + + +async def call(client: Any, method: str, *args: Any, **kwargs: Any) -> Any: + result = getattr(client.edge_sync, method)(*args, **kwargs) + return await result if inspect.isawaitable(result) else result + + +@pytest.fixture(params=[False, True], ids=["sync", "async"]) +def async_mode(request: Any) -> bool: + return bool(request.param) + + +async def enter_client(stack: AsyncExitStack, async_mode: bool) -> Any: + if async_mode: + return await stack.enter_async_context(AsyncArcClient(token="t", max_retries=0)) + return stack.enter_context(ArcClient(token="t", max_retries=0)) + + +def body_of(httpx_mock: HTTPXMock) -> Any: + content = httpx_mock.get_request().content + return json.loads(content) if content else None + + +# Hub: spokes ----------------------------------------------------------------- + + +async def test_list_spokes(httpx_mock: HTTPXMock, async_mode: bool) -> None: + second = {k: v for k, v in SPOKE.items() if k != "last_seen_at"} + second["spoke_id"] = "rocket-02" + httpx_mock.add_response(method="GET", url=f"{SPOKES}/", json={"spokes": [SPOKE, second]}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + spokes = await call(client, "list_spokes") + assert [s.spoke_id for s in spokes] == ["rocket-01", "rocket-02"] + assert all(isinstance(s, Spoke) for s in spokes) + assert spokes[0].last_seen_at is not None + assert spokes[1].last_seen_at is None + + +@pytest.mark.parametrize("body", [{}, {"spokes": None}, []]) +async def test_list_spokes_malformed(httpx_mock: HTTPXMock, async_mode: bool, body: Any) -> None: + httpx_mock.add_response(method="GET", url=f"{SPOKES}/", json=body) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcError): + await call(client, "list_spokes") + + +async def test_get_spoke(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(method="GET", url=f"{SPOKES}/rocket-01", json=SPOKE) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + spoke = await call(client, "get_spoke", "rocket-01") + assert spoke.files_received == 4 + + +async def test_get_spoke_not_found(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(status_code=404, json={"error": "spoke not registered"}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcNotFoundError): + await call(client, "get_spoke", "nope") + + +async def test_register_spoke_returns_secret(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response( + method="POST", + url=f"{SPOKES}/", + status_code=201, + json={"spoke_id": "rocket-01", "name": "Rocket", "secret": "s3cr3t", "warning": "once"}, + ) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + result = await call(client, "register_spoke", "rocket-01", name="Rocket") + assert isinstance(result, SpokeSecret) + assert result.secret == "s3cr3t" + assert body_of(httpx_mock) == {"spoke_id": "rocket-01", "name": "Rocket"} + + +async def test_register_conflict(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(status_code=409, json={"error": "spoke already registered"}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcServerError) as info: + await call(client, "register_spoke", "rocket-01") + assert info.value.status_code == 409 + + +@pytest.mark.parametrize( + ("method", "args", "kwargs"), + [ + ("rotate_spoke_secret", ("rocket-01",), {}), + ("delete_spoke", ("rocket-01",), {}), + ("dismiss_failed", (), {"path": "db/cpu/a.parquet"}), + ("dismiss_failed", (), {"all_failed": True}), + ], +) +async def test_confirm_required( + httpx_mock: HTTPXMock, + async_mode: bool, + method: str, + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> None: + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcValidationError): + await call(client, method, *args, **kwargs) + assert httpx_mock.get_requests() == [] + + +async def test_rotate_spoke_secret(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response( + method="POST", + url=f"{SPOKES}/rocket-01/rotate", + json={"spoke_id": "rocket-01", "secret": "new", "warning": "old secret is dead"}, + ) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + result = await call(client, "rotate_spoke_secret", "rocket-01", confirm=True) + assert result.secret == "new" + + +@pytest.mark.parametrize(("method", "enabled"), [("enable_spoke", True), ("disable_spoke", False)]) +async def test_enable_disable( + httpx_mock: HTTPXMock, async_mode: bool, method: str, enabled: bool +) -> None: + verb = "enable" if enabled else "disable" + body = {"spoke_id": "rocket-01", "enabled": enabled} + httpx_mock.add_response(method="POST", url=f"{SPOKES}/rocket-01/{verb}", json=body) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert await call(client, method, "rocket-01") == body + + +async def test_delete_spoke(httpx_mock: HTTPXMock, async_mode: bool) -> None: + body = {"spoke_id": "rocket-01", "deleted": True, "note": "files retained"} + httpx_mock.add_response(method="DELETE", url=f"{SPOKES}/rocket-01", json=body) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert await call(client, "delete_spoke", "rocket-01", confirm=True) == body + + +async def test_spoke_id_is_path_escaped(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(method="GET", json=SPOKE) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + await call(client, "get_spoke", "a/b") + assert httpx_mock.get_request().url.raw_path == b"/api/v1/sync-spokes/a%2Fb" + + +@pytest.mark.parametrize( + ("method", "args"), + [ + ("get_spoke", ("",)), + ("register_spoke", ("",)), + ("enable_spoke", ("",)), + ("import_history", ("",)), + ("revert_export", ("",)), + ("import_bundle", ("",)), + ("apply_ack", ("",)), + ("export_bundle", ("",)), + ], +) +async def test_required_arguments( + httpx_mock: HTTPXMock, async_mode: bool, method: str, args: tuple[Any, ...] +) -> None: + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcValidationError): + await call(client, method, *args) + assert httpx_mock.get_requests() == [] + + +@pytest.mark.parametrize( + "method", ["list_spokes", "import_bundle", "spoke_status", "run_sync", "spoke_ledger"] +) +async def test_non_admin_refused(httpx_mock: HTTPXMock, async_mode: bool, method: str) -> None: + httpx_mock.add_response(status_code=403, json={"error": "Admin permission required"}) + args = ("/mnt/drive",) if method == "import_bundle" else () + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcPermissionError): + await call(client, method, *args) + + +# Hub: bundle import ---------------------------------------------------------- + + +async def test_import_bundle(httpx_mock: HTTPXMock, async_mode: bool) -> None: + body = {"imported": True, "bundle_id": "b1", "spoke_id": "rocket-01", "conflicts": []} + httpx_mock.add_response(method="POST", url=f"{IMPORT}/", json=body) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert await call(client, "import_bundle", "/mnt/drive/b1") == body + assert body_of(httpx_mock) == {"path": "/mnt/drive/b1"} + + +@pytest.mark.parametrize( + ("status", "body"), + [ + (409, {"imported": False, "reason": "already imported", "detail": "on 2027-01-01"}), + (409, {"imported": False, "reason": "import in progress", "detail": "retry later"}), + (422, {"imported": False, "error": "bundle checksum mismatch"}), + ], +) +async def test_import_bundle_refusals_raise( + httpx_mock: HTTPXMock, async_mode: bool, status: int, body: dict[str, Any] +) -> None: + httpx_mock.add_response(method="POST", status_code=status, json=body) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcServerError) as info: + await call(client, "import_bundle", "/mnt/drive/b1") + assert info.value.status_code == status + assert info.value.response_body == body + + +async def test_import_history(httpx_mock: HTTPXMock, async_mode: bool) -> None: + body = {"spoke_id": "rocket-01", "bundles": [], "limit": 5} + httpx_mock.add_response(method="GET", url=f"{IMPORT}/history/rocket-01?limit=5", json=body) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert await call(client, "import_history", "rocket-01", limit=5) == body + + +@pytest.mark.parametrize("limit", [0, 1001, True]) +async def test_import_history_limit(httpx_mock: HTTPXMock, async_mode: bool, limit: Any) -> None: + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcValidationError): + await call(client, "import_history", "rocket-01", limit=limit) + + +# Spoke: sync and ledger ----------------------------------------------------- + + +async def test_spoke_status(httpx_mock: HTTPXMock, async_mode: bool) -> None: + body = {"hub_id": "hub", "pending": 3, "failed": 1, "seconds_since_last_sync": 60} + httpx_mock.add_response(method="GET", url=f"{SPOKE_SYNC}/status", json=body) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert await call(client, "spoke_status") == body + + +async def test_spoke_ledger(httpx_mock: HTTPXMock, async_mode: bool) -> None: + body = {"entries": [{"path": "db/cpu/a.parquet", "state": "skipped"}], "limit": 10} + httpx_mock.add_response( + method="GET", url=f"{SPOKE_SYNC}/ledger?limit=10&state=skipped", json=body + ) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert await call(client, "spoke_ledger", limit=10, state="skipped") == body + + +async def test_spoke_ledger_default(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(method="GET", url=f"{SPOKE_SYNC}/ledger", json={"entries": []}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + await call(client, "spoke_ledger") + + +@pytest.mark.parametrize("kwargs", [{"state": "done"}, {"limit": 0}, {"limit": 1001}]) +async def test_spoke_ledger_validation( + httpx_mock: HTTPXMock, async_mode: bool, kwargs: dict[str, Any] +) -> None: + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcValidationError): + await call(client, "spoke_ledger", **kwargs) + + +async def test_run_sync(httpx_mock: HTTPXMock, async_mode: bool) -> None: + body = {"discovered": 3, "sent": 3, "conflicts": []} + httpx_mock.add_response(method="POST", url=f"{SPOKE_SYNC}/run", json=body) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert await call(client, "run_sync") == body + + +@pytest.mark.parametrize("status", [409, 503]) +async def test_run_sync_refused(httpx_mock: HTTPXMock, async_mode: bool, status: int) -> None: + httpx_mock.add_response(method="POST", status_code=status, json={"error": "nope"}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcServerError) as info: + await call(client, "run_sync") + assert info.value.status_code == status + assert len(httpx_mock.get_requests()) == 1 + + +@pytest.mark.parametrize( + ("kwargs", "wire"), + [ + ({"path": "db/cpu/a.parquet"}, {"path": "db/cpu/a.parquet"}), + ({"all_failed": True}, {"all": True}), + ], +) +async def test_requeue_failed( + httpx_mock: HTTPXMock, async_mode: bool, kwargs: dict[str, Any], wire: dict[str, Any] +) -> None: + httpx_mock.add_response(method="POST", url=f"{SPOKE_SYNC}/ledger/requeue", json={"requeued": 2}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert await call(client, "requeue_failed", **kwargs) == {"requeued": 2} + assert body_of(httpx_mock) == wire + + +async def test_dismiss_failed(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response( + method="POST", url=f"{SPOKE_SYNC}/ledger/dismiss", json={"dismissed": 5} + ) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + result = await call(client, "dismiss_failed", all_failed=True, confirm=True) + assert result == {"dismissed": 5} + assert body_of(httpx_mock) == {"all": True} + + +@pytest.mark.parametrize("method", ["requeue_failed", "dismiss_failed"]) +@pytest.mark.parametrize("kwargs", [{}, {"path": "x", "all_failed": True}]) +async def test_selector_exactly_one( + httpx_mock: HTTPXMock, async_mode: bool, method: str, kwargs: dict[str, Any] +) -> None: + if method == "dismiss_failed": + kwargs = {**kwargs, "confirm": True} + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcValidationError): + await call(client, method, **kwargs) + assert httpx_mock.get_requests() == [] + + +async def test_requeue_path_miss(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(status_code=404, json={"error": "no failed entry at that path"}) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcNotFoundError): + await call(client, "requeue_failed", path="missing") + + +# Spoke: air-gap export ------------------------------------------------------ + + +async def test_export_bundle(httpx_mock: HTTPXMock, async_mode: bool) -> None: + body = {"exported": True, "bundle_id": "b1", "files": 2} + httpx_mock.add_response(method="POST", url=f"{SPOKE_SYNC}/export", json=body) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert await call(client, "export_bundle", "/mnt/usb", limit=50) == body + assert body_of(httpx_mock) == {"path": "/mnt/usb", "limit": 50} + + +async def test_export_nothing_is_not_an_error(httpx_mock: HTTPXMock, async_mode: bool) -> None: + body = {"exported": False, "reason": "nothing to export"} + httpx_mock.add_response(method="POST", url=f"{SPOKE_SYNC}/export", json=body) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert await call(client, "export_bundle", "/mnt/usb") == body + assert body_of(httpx_mock) == {"path": "/mnt/usb"} + + +@pytest.mark.parametrize("limit", [0, -1, False]) +async def test_export_limit_validation(httpx_mock: HTTPXMock, async_mode: bool, limit: Any) -> None: + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcValidationError): + await call(client, "export_bundle", "/mnt/usb", limit=limit) + + +async def test_revert_export(httpx_mock: HTTPXMock, async_mode: bool) -> None: + body = {"bundle_id": "b1", "reverted": 7} + httpx_mock.add_response(method="POST", url=f"{SPOKE_SYNC}/export/b1/revert", json=body) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert await call(client, "revert_export", "b1") == body + + +async def test_apply_ack(httpx_mock: HTTPXMock, async_mode: bool) -> None: + body = {"applied": True, "bundle_id": "b1", "synced": 7, "conflicts": []} + httpx_mock.add_response(method="POST", url=f"{SPOKE_SYNC}/ack", json=body) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + assert await call(client, "apply_ack", "/mnt/usb/b1") == body + assert body_of(httpx_mock) == {"path": "/mnt/usb/b1"} + + +async def test_export_disabled_503(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response( + method="POST", + status_code=503, + json={"error": "air-gap bundle export is not enabled (edge_sync.spoke.bundle.enabled)"}, + ) + async with AsyncExitStack() as stack: + client = await enter_client(stack, async_mode) + with pytest.raises(ArcServerError) as info: + await call(client, "apply_ack", "/mnt/usb/b1") + assert info.value.status_code == 503 From 9335fe6d530e621c0ed4f4db65806f5dc02bf270 Mon Sep 17 00:00:00 2001 From: Ignacio Van Droogenbroeck <64545348+xe-nvdk@users.noreply.github.com> Date: Mon, 2 Nov 2026 09:06:24 -0600 Subject: [PATCH 11/20] fix(http): retry query reads on 429/503 and type Arc ingest refusals Query POSTs only read, but _request retried status codes for GET/HEAD/OPTIONS only, so Arc's replication catch-up gate (503 + Retry-After) and the 27.01 cold-tier/vanished-partition 503s failed queries outright. Requests can now be marked idempotent=True (status retries like a GET; transport retries keep the mutation rule) or carry a retry_if predicate. A 503 carrying X-Arc-Ingest-Fenced or X-Arc-Ingest-Refused now raises ArcIngestRefusedError(reason, retry_safe); is_retryable_write_refusal accepts every reason except wal_not_durable, which may have applied part of the request. Adds ArcPartialWriteError and status_code on ArcQueryError and ArcIngestionError. --- src/arc_client/__init__.py | 4 ++ src/arc_client/exceptions.py | 75 +++++++++++++++++++++++++++++-- src/arc_client/http/async_http.py | 54 +++++++++++++++++----- src/arc_client/http/base.py | 62 +++++++++++++++++++++++-- src/arc_client/http/sync_http.py | 54 +++++++++++++++++----- tests/unit/http/_parity.py | 35 +++++++++++++++ 6 files changed, 255 insertions(+), 29 deletions(-) create mode 100644 tests/unit/http/_parity.py diff --git a/src/arc_client/__init__.py b/src/arc_client/__init__.py index 0118e42..0810037 100644 --- a/src/arc_client/__init__.py +++ b/src/arc_client/__init__.py @@ -15,9 +15,11 @@ ArcConnectionError, ArcError, ArcIngestionError, + ArcIngestRefusedError, ArcLicenseError, ArcNotFoundError, ArcNotSupportedError, + ArcPartialWriteError, ArcPermissionDataUnavailableError, ArcPermissionError, ArcQueryError, @@ -41,6 +43,8 @@ "ArcAuthenticationError", "ArcQueryError", "ArcIngestionError", + "ArcPartialWriteError", + "ArcIngestRefusedError", "ArcValidationError", "ArcNotFoundError", "ArcRateLimitError", diff --git a/src/arc_client/exceptions.py b/src/arc_client/exceptions.py index fad56fd..ad71fef 100644 --- a/src/arc_client/exceptions.py +++ b/src/arc_client/exceptions.py @@ -40,15 +40,82 @@ class ArcAuthenticationError(ArcError): class ArcQueryError(ArcError): - """Query execution failed.""" + """Query execution failed. - pass + Attributes: + status_code: The HTTP status Arc answered with when the failure was an + HTTP error without a more specific exception (e.g. 400 for invalid + SQL, 500, or a 503 that outlasted the retries). ``None`` otherwise. + """ + + def __init__(self, message: str, status_code: Optional[int] = None) -> None: + super().__init__(message) + self.status_code = status_code class ArcIngestionError(ArcError): - """Data ingestion failed.""" + """Data ingestion failed. - pass + Attributes: + status_code: The HTTP status Arc answered with, when there was one. + """ + + def __init__(self, message: str, status_code: Optional[int] = None) -> None: + super().__init__(message) + self.status_code = status_code + + +class ArcPartialWriteError(ArcIngestionError): + """Arc dropped some points of a write (HTTP 400 "partial write"). + + Arc 2027.01 drops individual points it cannot store (a value of the wrong + type for its column, a tag or field named ``time``, a name Arc could not + read back) and still writes the rest of the request, answering + ``400 "partial write: N points dropped: ..."``. + + The request has been applied as far as Arc could apply it, so it must + **not** be resent: resending would duplicate every point that was stored. + The SDK never retries it, and a buffered writer discards the batch and + reports this error once. + + Attributes: + dropped: Number of points Arc dropped. + others_written: ``True`` when Arc reported that the other points of the + request were written; ``False`` when every point was dropped and + nothing was stored. Arc does not report an accepted-point count. + """ + + def __init__(self, message: str, dropped: int, others_written: bool) -> None: + super().__init__(message, status_code=400) + self.dropped = dropped + self.others_written = others_written + + +class ArcIngestRefusedError(ArcIngestionError): + """Arc refused a write with HTTP 503 before (or while) applying it. + + Raised for a 503 that carries ``X-Arc-Ingest-Fenced`` (this node is not the + available primary writer, e.g. during a failover) or + ``X-Arc-Ingest-Refused: `` (a node-local, transient condition such + as ``wal_disk``, WAL volume disk pressure). Arc documents these as "nothing + was written, retry", and the SDK resends them automatically up to + ``max_retries``; this error means the retries ran out. + + The exception is reason ``wal_not_durable`` (``wal.ack_mode = "durable"`` + could not make the WAL entry durable): earlier parts of the same request + may already be stored, so a resend may duplicate rows. The SDK never + retries it, and ``retry_safe`` is ``False``. + + Attributes: + reason: Arc's refusal reason (``"fenced"`` for a fence, otherwise the + ``X-Arc-Ingest-Refused`` value such as ``"wal_disk"``). + retry_safe: Whether Arc guarantees nothing of the request was applied. + """ + + def __init__(self, message: str, reason: str, retry_safe: bool) -> None: + super().__init__(message, status_code=503) + self.reason = reason + self.retry_safe = retry_safe class ArcValidationError(ArcError): diff --git a/src/arc_client/http/async_http.py b/src/arc_client/http/async_http.py index 17d9fee..446639f 100644 --- a/src/arc_client/http/async_http.py +++ b/src/arc_client/http/async_http.py @@ -12,6 +12,7 @@ from arc_client.http.base import ( RETRYABLE_TRANSPORT_ERRORS, HTTPClientBase, + RetryPredicate, compute_backoff, handle_connection_error, handle_response_error, @@ -50,13 +51,26 @@ async def _request( path: str, *, headers: Optional[dict[str, str]] = None, + idempotent: bool = False, + retry_if: Optional[RetryPredicate] = None, **kwargs: Any, ) -> httpx.Response: """Send a request with retry/backoff for transient failures. - Reads retry transient failures up to ``config.max_retries`` times. - Mutations retry only connection establishment failures; data DELETE - never retries. Unknown write outcomes are returned to the caller. + Reads (GET/HEAD/OPTIONS) retry transient transport failures and + 429/503 responses up to ``config.max_retries`` times. Requests the + caller marks ``idempotent=True`` (Arc's query POSTs, which only read) + retry 429/503 like a read; for transport failures they keep the + mutation rule (only connection establishment failures), because a + timeout or a connection lost mid-response means Arc already ran the + query, and silently re-running it would multiply its cost and hide a + truncated stream. Data DELETE never retries. Unknown write outcomes + are returned to the caller. + + ``retry_if`` replaces the status-retry rule for this one request: a + response for which it returns True is retried (honoring Retry-After), + whatever the method. Writes use it to resend only the 503 refusals Arc + documents as "nothing was written". """ request_kwargs = self._prepare_request_kwargs(headers=headers, **kwargs) client = self._get_client() @@ -80,11 +94,13 @@ async def _request( await asyncio.sleep(compute_backoff(attempt, self.config.retry_backoff)) continue - if ( - read_only - and is_retryable_status(response.status_code) - and attempt < self.config.max_retries - ): + if retry_if is not None: + retry_status = retry_if(response) + else: + retry_status = (read_only or idempotent) and is_retryable_status( + response.status_code + ) + if retry_status and attempt < self.config.max_retries: retry_after = parse_retry_after(response) await asyncio.sleep( compute_backoff(attempt, self.config.retry_backoff, retry_after) @@ -104,9 +120,10 @@ async def get( path: str, params: Optional[dict[str, Any]] = None, headers: Optional[dict[str, str]] = None, + retry_if: Optional[RetryPredicate] = None, ) -> httpx.Response: """Make a GET request.""" - return await self._request("GET", path, headers=headers, params=params) + return await self._request("GET", path, headers=headers, params=params, retry_if=retry_if) async def post( self, @@ -117,8 +134,15 @@ async def post( headers: Optional[dict[str, str]] = None, files: Optional[Any] = None, data: Optional[Any] = None, + idempotent: bool = False, + retry_if: Optional[RetryPredicate] = None, ) -> httpx.Response: - """Make a POST request.""" + """Make a POST request. + + ``idempotent=True`` marks a POST that only reads (Arc's query + endpoints) so it retries like a GET. ``retry_if`` overrides the status + retry rule for this request (see ``_request``). + """ extra: dict[str, Any] = {} if json is not None: extra["json"] = json @@ -128,7 +152,15 @@ async def post( extra["files"] = files if data is not None: extra["data"] = data - return await self._request("POST", path, headers=headers, params=params, **extra) + return await self._request( + "POST", + path, + headers=headers, + params=params, + idempotent=idempotent, + retry_if=retry_if, + **extra, + ) async def put( self, diff --git a/src/arc_client/http/base.py b/src/arc_client/http/base.py index 1cb8a3f..1f3bd1c 100644 --- a/src/arc_client/http/base.py +++ b/src/arc_client/http/base.py @@ -4,7 +4,7 @@ import os import re -from typing import Any, Optional +from typing import Any, Callable, Optional import httpx @@ -14,6 +14,7 @@ ArcAuthenticationError, ArcConnectionError, ArcError, + ArcIngestRefusedError, ArcLicenseError, ArcNotFoundError, ArcNotSupportedError, @@ -131,6 +132,7 @@ def _raise_response_error(response: httpx.Response) -> None: ArcNotFoundError: HTTP 404. ArcRateLimitError: HTTP 429. ArcNotSupportedError: HTTP 501. + ArcIngestRefusedError: HTTP 503 carrying Arc's ingest-refusal headers. ArcServerError: HTTP 5xx and any other non-2xx status. """ if response.is_success: @@ -188,6 +190,13 @@ def _raise_response_error(response: httpx.Response) -> None: raise ArcRateLimitError(f"Rate limit exceeded: {message}") elif status_code == 501: raise ArcNotSupportedError(f"Operation not supported by server: {message}") + elif (refusal := ingest_refusal(response)) is not None: + reason, retry_safe = refusal + raise ArcIngestRefusedError( + f"Write refused by Arc ({reason}): {message}", + reason=reason, + retry_safe=retry_safe, + ) elif status_code >= 500: raise ArcServerError(f"Server error: {message}", status_code=status_code) else: @@ -209,16 +218,63 @@ def handle_connection_error(error: Exception, url: str) -> None: httpx.RemoteProtocolError, ) -# HTTP status codes worth retrying: 429 rate-limit, 503 schema-churn / -# replication catch-up (both explicitly retryable per the Arc server). +# HTTP status codes worth retrying on reads and idempotent query POSTs: +# 429 (rate limit / query governance) and 503. Arc answers 503 with +# Retry-After for a read it cannot serve right now: the replication catch-up +# gate (internal/api/query.go checkReplicationReady), a partition removed +# while the query ran, and a cold tier configured but unavailable. Write-side +# 503s are NOT covered by this set; see is_retryable_write_refusal. RETRYABLE_STATUS_CODES = frozenset({429, 503}) +# Arc marks a 503 that refused a write before applying it +# (internal/api/routing.go IngestFencedHeader / IngestRefusedHeader). +INGEST_FENCED_HEADER = "X-Arc-Ingest-Fenced" +INGEST_REFUSED_HEADER = "X-Arc-Ingest-Refused" +# The one refusal reason that may have applied part of the request +# (internal/ingest/refusal.go RefusalWALNotDurable): a retry can duplicate rows. +REFUSAL_WAL_NOT_DURABLE = "wal_not_durable" + +# A per-request status-retry predicate: True means "retry this response". +RetryPredicate = Callable[[httpx.Response], bool] + def is_retryable_status(status_code: int) -> bool: """Return True if a response status should be retried.""" return status_code in RETRYABLE_STATUS_CODES +def ingest_refusal(response: httpx.Response) -> Optional[tuple[str, bool]]: + """Describe an Arc ingest refusal as ``(reason, retry_safe)``, else None. + + Only a 503 carrying ``X-Arc-Ingest-Fenced`` (the node is not the available + primary writer) or ``X-Arc-Ingest-Refused: `` (a node-local, + transient condition such as ``wal_disk``) is a refusal. Arc documents both + as "nothing was written, retry", except reason ``wal_not_durable``: earlier + parts of the same request may already be stored, so a retry may duplicate + rows and is never automatic. + """ + if response.status_code != 503: + return None + refused = response.headers.get(INGEST_REFUSED_HEADER) + if refused is not None: + reason = refused.strip().lower() or "other" + return reason, reason != REFUSAL_WAL_NOT_DURABLE + if response.headers.get(INGEST_FENCED_HEADER) is not None: + return "fenced", True + return None + + +def is_retryable_write_refusal(response: httpx.Response) -> bool: + """True for an ingest refusal Arc documents as safe to resend.""" + refusal = ingest_refusal(response) + return refusal is not None and refusal[1] + + +def never_retry_status(response: httpx.Response) -> bool: + """Status-retry predicate that disables status retries for one request.""" + return False + + def parse_retry_after(response: httpx.Response) -> Optional[float]: """Parse a ``Retry-After`` header (seconds only) into a float, if present.""" value = response.headers.get("Retry-After") diff --git a/src/arc_client/http/sync_http.py b/src/arc_client/http/sync_http.py index 49ac570..b128d79 100644 --- a/src/arc_client/http/sync_http.py +++ b/src/arc_client/http/sync_http.py @@ -12,6 +12,7 @@ from arc_client.http.base import ( RETRYABLE_TRANSPORT_ERRORS, HTTPClientBase, + RetryPredicate, compute_backoff, handle_connection_error, handle_response_error, @@ -50,13 +51,26 @@ def _request( path: str, *, headers: Optional[dict[str, str]] = None, + idempotent: bool = False, + retry_if: Optional[RetryPredicate] = None, **kwargs: Any, ) -> httpx.Response: """Send a request with retry/backoff for transient failures. - Reads retry transient failures up to ``config.max_retries`` times. - Mutations retry only connection establishment failures; data DELETE - never retries. Unknown write outcomes are returned to the caller. + Reads (GET/HEAD/OPTIONS) retry transient transport failures and + 429/503 responses up to ``config.max_retries`` times. Requests the + caller marks ``idempotent=True`` (Arc's query POSTs, which only read) + retry 429/503 like a read; for transport failures they keep the + mutation rule (only connection establishment failures), because a + timeout or a connection lost mid-response means Arc already ran the + query, and silently re-running it would multiply its cost and hide a + truncated stream. Data DELETE never retries. Unknown write outcomes + are returned to the caller. + + ``retry_if`` replaces the status-retry rule for this one request: a + response for which it returns True is retried (honoring Retry-After), + whatever the method. Writes use it to resend only the 503 refusals Arc + documents as "nothing was written". """ request_kwargs = self._prepare_request_kwargs(headers=headers, **kwargs) client = self._get_client() @@ -80,11 +94,13 @@ def _request( time.sleep(compute_backoff(attempt, self.config.retry_backoff)) continue - if ( - read_only - and is_retryable_status(response.status_code) - and attempt < self.config.max_retries - ): + if retry_if is not None: + retry_status = retry_if(response) + else: + retry_status = (read_only or idempotent) and is_retryable_status( + response.status_code + ) + if retry_status and attempt < self.config.max_retries: retry_after = parse_retry_after(response) time.sleep(compute_backoff(attempt, self.config.retry_backoff, retry_after)) continue @@ -102,9 +118,10 @@ def get( path: str, params: Optional[dict[str, Any]] = None, headers: Optional[dict[str, str]] = None, + retry_if: Optional[RetryPredicate] = None, ) -> httpx.Response: """Make a GET request.""" - return self._request("GET", path, headers=headers, params=params) + return self._request("GET", path, headers=headers, params=params, retry_if=retry_if) def post( self, @@ -115,8 +132,15 @@ def post( headers: Optional[dict[str, str]] = None, files: Optional[Any] = None, data: Optional[Any] = None, + idempotent: bool = False, + retry_if: Optional[RetryPredicate] = None, ) -> httpx.Response: - """Make a POST request.""" + """Make a POST request. + + ``idempotent=True`` marks a POST that only reads (Arc's query + endpoints) so it retries like a GET. ``retry_if`` overrides the status + retry rule for this request (see ``_request``). + """ extra: dict[str, Any] = {} if json is not None: extra["json"] = json @@ -126,7 +150,15 @@ def post( extra["files"] = files if data is not None: extra["data"] = data - return self._request("POST", path, headers=headers, params=params, **extra) + return self._request( + "POST", + path, + headers=headers, + params=params, + idempotent=idempotent, + retry_if=retry_if, + **extra, + ) def put( self, diff --git a/tests/unit/http/_parity.py b/tests/unit/http/_parity.py new file mode 100644 index 0000000..84a2164 --- /dev/null +++ b/tests/unit/http/_parity.py @@ -0,0 +1,35 @@ +"""Shared helpers for the sync/async parametrized HTTP contract tests.""" + +from __future__ import annotations + +import inspect +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Any + +import pytest + +from arc_client import ArcClient, AsyncArcClient + +MODES = pytest.mark.parametrize("async_mode", [False, True], ids=["sync", "async"]) + + +@asynccontextmanager +async def make_client(async_mode: bool, **kwargs: Any) -> AsyncIterator[Any]: + """Yield an ArcClient or AsyncArcClient with instant retry backoff.""" + kwargs.setdefault("host", "h") + kwargs.setdefault("token", "t") + kwargs.setdefault("retry_backoff", 0) + if async_mode: + async with AsyncArcClient(**kwargs) as client: + yield client + else: + with ArcClient(**kwargs) as client: + yield client + + +async def call(result: Any) -> Any: + """Await the result of an async client call; pass sync results through.""" + if inspect.isawaitable(result): + return await result + return result From 47f3b26eb0a7261dbd81e44d8a18f2d2120f3ce0 Mon Sep 17 00:00:00 2001 From: Ignacio Van Droogenbroeck <64545348+xe-nvdk@users.noreply.github.com> Date: Mon, 2 Nov 2026 09:06:25 -0600 Subject: [PATCH 12/20] feat(query): profiling, Arrow encodings, query IDs and measurement reads - Query executors no longer rewrap typed errors: authentication, permission, scoped-access, license, rate-limit and not-supported errors surface as themselves. Generic HTTP and transport failures stay ArcQueryError (with status_code and __cause__) so except ArcQueryError keeps working. - All four query POSTs are sent idempotent=True. - profile=True on query/query_msgpack sends x-arc-profile and returns QueryResponse.profile (QueryProfile). - query_arrow/query_pandas/query_polars accept dictionary= and compression=('zstd'|'lz4'). - X-Arc-Query-ID is exposed as QueryResponse.query_id and arc.query_id in Arrow schema metadata. - query_measurement() wraps GET /api/v1/query/:measurement. - msgpack: documented as stable; an incomplete body (Arc's only signal for a stream that failed after the 200) raises ArcQueryError. --- src/arc_client/models/query.py | 39 ++- src/arc_client/query/arrow_response.py | 8 +- src/arc_client/query/async_executor.py | 225 ++++++++++--- src/arc_client/query/executor.py | 221 ++++++++++--- src/arc_client/query/msgpack_response.py | 43 ++- src/arc_client/query/options.py | 92 ++++++ tests/unit/http/test_query_parity.py | 397 +++++++++++++++++++++++ 7 files changed, 914 insertions(+), 111 deletions(-) create mode 100644 src/arc_client/query/options.py create mode 100644 tests/unit/http/test_query_parity.py diff --git a/src/arc_client/models/query.py b/src/arc_client/models/query.py index 770e4cf..804acc1 100644 --- a/src/arc_client/models/query.py +++ b/src/arc_client/models/query.py @@ -4,7 +4,30 @@ from typing import Any, Optional -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field + + +class QueryProfile(BaseModel): + """Server-side timing for one query, returned when ``profile=True``. + + Mirrors Arc's ``database.QueryProfile``. Extra keys a newer server adds + are kept rather than dropped. + + Attributes: + total_ms: Total server-side time in milliseconds. + planner_ms: Planning time in milliseconds. + execution_ms: Execution time in milliseconds. + rows_scanned: Rows DuckDB scanned. + latency_ms: Latency DuckDB itself reported, in milliseconds. + """ + + model_config = ConfigDict(extra="allow") + + total_ms: float = 0.0 + planner_ms: float = 0.0 + execution_ms: float = 0.0 + rows_scanned: int = 0 + latency_ms: float = 0.0 class QueryResponse(BaseModel): @@ -15,13 +38,23 @@ class QueryResponse(BaseModel): columns: List of column names in the result. data: List of rows, where each row is a list of values. row_count: Number of rows returned. - truncated: Whether the result was cut short by a query failure. + truncated: Whether the result was cut short by a query failure. Arc's + JSON writer sets it when a stream fails after its headers were + sent. Arc's MessagePack writer never sends it: a msgpack stream + that fails part way arrives as an incomplete body, which + ``query_msgpack`` raises as :class:`ArcQueryError` instead of + returning partial rows. truncation_reason: Reason the result was truncated, if available. rows_capped: Whether the result reached a configured row cap. row_cap: Configured row cap, if available. execution_time_ms: Query execution time in milliseconds. timestamp: Server timestamp when the query was executed. error: Error message if the query failed. + profile: Server-side timing, present when the query asked for it with + ``profile=True``. + query_id: Arc's ID for this query (``X-Arc-Query-ID``), usable with + ``client.queries`` to look it up or cancel it. ``None`` when the + server does not run the query registry. """ success: bool @@ -37,6 +70,8 @@ class QueryResponse(BaseModel): error: Optional[str] = None # Parallel to ``columns``: Arrow type name per column, when supplied. types: list[str] = Field(default_factory=list) + profile: Optional[QueryProfile] = None + query_id: Optional[str] = None class EstimateResponse(BaseModel): diff --git a/src/arc_client/query/arrow_response.py b/src/arc_client/query/arrow_response.py index de8f50e..4d2d1fb 100644 --- a/src/arc_client/query/arrow_response.py +++ b/src/arc_client/query/arrow_response.py @@ -16,7 +16,10 @@ _TRUNCATED = b"arc-stream-truncated" _TIME = b"arc-execution-time-ms" _FIELDS = {_CAP, _TRUNCATED, _TIME} -_RESERVED = {b"arc.rows_capped", b"arc.row_cap", b"arc.execution_time_ms"} +_RESERVED = {b"arc.rows_capped", b"arc.row_cap", b"arc.execution_time_ms", b"arc.query_id"} +# Sent as a regular header before streaming starts (query_arrow.go), so it is +# never a trailer. It identifies the query for client.queries.get()/kill(). +QUERY_ID_HEADER = "X-Arc-Query-ID" def decode_arrow_response(response: httpx.Response) -> pa.Table: @@ -79,6 +82,9 @@ def decode_arrow_response(response: httpx.Response) -> pa.Table: if not math.isfinite(elapsed) or elapsed < 0: raise ArcQueryError("Invalid Arrow execution-time metadata") metadata[b"arc.execution_time_ms"] = values[_TIME] + query_id = response.headers.get(QUERY_ID_HEADER, "").strip() + if query_id: + metadata[b"arc.query_id"] = query_id.encode("utf-8") with ipc.open_stream(response.content) as reader: table = reader.read_all() diff --git a/src/arc_client/query/async_executor.py b/src/arc_client/query/async_executor.py index ff06fcb..a906174 100644 --- a/src/arc_client/query/async_executor.py +++ b/src/arc_client/query/async_executor.py @@ -2,21 +2,31 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional +from typing import Any, Optional import pyarrow as pa from arc_client.config import ClientConfig -from arc_client.exceptions import ArcNotSupportedError, ArcQueryError, ArcValidationError +from arc_client.exceptions import ( + ArcConnectionError, + ArcError, + ArcQueryError, + ArcServerError, + ArcValidationError, +) from arc_client.http.async_http import AsyncHTTPClient from arc_client.models.query import EstimateResponse, MeasurementInfo, QueryResponse -from arc_client.query.arrow_response import decode_arrow_response +from arc_client.query.arrow_response import QUERY_ID_HEADER, decode_arrow_response from arc_client.query.msgpack_response import decode_msgpack_query_response +from arc_client.query.options import ( + arrow_headers, + measurement_params, + query_headers, + query_response_from_json, +) +from arc_client.query.options import status_code as _status from arc_client.validation import validate_name -if TYPE_CHECKING: - pass - class AsyncQueryClient: """Asynchronous client for querying data from Arc. @@ -45,49 +55,63 @@ def __init__(self, http: AsyncHTTPClient, config: ClientConfig) -> None: self._http = http self._config = config - async def query(self, sql: str, database: Optional[str] = None) -> QueryResponse: + async def query( + self, sql: str, database: Optional[str] = None, *, profile: bool = False + ) -> QueryResponse: """Execute a SQL query and return results as JSON. This method returns structured JSON data. For large result sets, consider using query_arrow() or query_polars() for better performance. + Queries only read, so a 429 or 503 (Arc's replication catch-up gate, a + partition removed mid-query, an unavailable cold tier) is retried with + backoff, honoring ``Retry-After``, up to ``max_retries``. + Args: sql: SQL query to execute. database: Target database. Uses client default if not specified. + profile: Ask Arc for server-side timing (``x-arc-profile``), + returned in ``QueryResponse.profile``. Returns: - QueryResponse with columns, data, row_count, and metadata. + QueryResponse with columns, data, row_count, metadata and + ``query_id``. Raises: - ArcQueryError: If the query fails. + ArcQueryError: If the query fails. ``status_code`` carries the HTTP + status when Arc answered with a generic error status (e.g. 400 + for invalid SQL). ArcValidationError: If the SQL is invalid. + ArcError: Typed failures (ArcAuthenticationError, + ArcPermissionError, ArcScopedAccessError, ArcRateLimitError, + ArcNotSupportedError, ...) are raised as themselves. Transport + failures (ArcConnectionError) are wrapped in ArcQueryError + with the original as ``__cause__``. """ if not sql or not sql.strip(): raise ArcValidationError("SQL query cannot be empty") - headers = {} - if database: - headers["x-arc-database"] = database - try: response = await self._http.post( "/api/v1/query", json={"sql": sql}, - headers=headers if headers else None, + headers=query_headers(database, profile) or None, + idempotent=True, ) - - data = response.json() - return QueryResponse(**data) - - except ArcQueryError: + return query_response_from_json(response) + except (ArcServerError, ArcConnectionError) as e: + raise ArcQueryError(f"Query failed: {e}", status_code=_status(e)) from e + except ArcError: raise except Exception as e: raise ArcQueryError(f"Query failed: {e}") from e - async def query_msgpack(self, sql: str, database: Optional[str] = None) -> QueryResponse: + async def query_msgpack( + self, sql: str, database: Optional[str] = None, *, profile: bool = False + ) -> QueryResponse: """Execute a SQL query and decode the MessagePack response. - **Experimental.** Uses Arc's ``/api/v1/query/msgpack`` endpoint, which + Uses Arc's ``/api/v1/query/msgpack`` endpoint, which returns a compact columnar MessagePack envelope with native timestamp encoding. The result is transposed to row-major and returned as a :class:`QueryResponse` (same shape as :meth:`query`), with per-column @@ -96,38 +120,57 @@ async def query_msgpack(self, sql: str, database: Optional[str] = None) -> Query Requires the Arc server to be built with the ``duckdb_arrow`` build tag; otherwise the server responds with HTTP 501. + A stream Arc fails after sending the 200 (client disconnect, deadline) + arrives as an incomplete MessagePack body and raises ArcQueryError; + partial rows are never returned (Arc's msgpack writer has no + ``truncated`` marker). + Args: sql: SQL query to execute. database: Target database. Uses client default if not specified. + profile: Ask Arc for server-side timing (``x-arc-profile``), + returned in ``QueryResponse.profile``. Returns: - QueryResponse with row-major ``data`` and populated ``types``. + QueryResponse with row-major ``data``, populated ``types`` and + ``query_id``. Raises: ArcValidationError: If the SQL is empty. ArcNotSupportedError: If the server lacks msgpack query support (501). - ArcQueryError: If the query fails. + ArcQueryError: If the query fails or the response is incomplete. """ if not sql or not sql.strip(): raise ArcValidationError("SQL query cannot be empty") - headers = {"Accept": "application/msgpack"} - if database: - headers["x-arc-database"] = database + headers = query_headers(database, profile) + headers["Accept"] = "application/msgpack" try: response = await self._http.post( "/api/v1/query/msgpack", json={"sql": sql}, headers=headers, + idempotent=True, ) - return decode_msgpack_query_response(response.content) - except (ArcQueryError, ArcNotSupportedError, ArcValidationError): + return decode_msgpack_query_response( + response.content, query_id=response.headers.get(QUERY_ID_HEADER) + ) + except (ArcServerError, ArcConnectionError) as e: + raise ArcQueryError(f"Query failed: {e}", status_code=_status(e)) from e + except ArcError: raise except Exception as e: raise ArcQueryError(f"Query failed: {e}") from e - async def query_pandas(self, sql: str, database: Optional[str] = None) -> Any: + async def query_pandas( + self, + sql: str, + database: Optional[str] = None, + *, + dictionary: bool = False, + compression: Optional[str] = None, + ) -> Any: """Execute a SQL query and return results as a pandas DataFrame. Uses Arrow IPC streaming for efficient data transfer. Requires @@ -136,6 +179,9 @@ async def query_pandas(self, sql: str, database: Optional[str] = None) -> Any: Args: sql: SQL query to execute. database: Target database. Uses client default if not specified. + dictionary: See :meth:`query_arrow`. Dictionary-encoded string + columns become pandas ``Categorical`` columns. + compression: See :meth:`query_arrow`. Returns: pandas.DataFrame with query results. @@ -152,10 +198,19 @@ async def query_pandas(self, sql: str, database: Optional[str] = None) -> Any: "Install it with: pip install arc-client[pandas]" ) from e - table = await self.query_arrow(sql, database) + table = await self.query_arrow( + sql, database, dictionary=dictionary, compression=compression + ) return table.to_pandas() - async def query_polars(self, sql: str, database: Optional[str] = None) -> Any: + async def query_polars( + self, + sql: str, + database: Optional[str] = None, + *, + dictionary: bool = False, + compression: Optional[str] = None, + ) -> Any: """Execute a SQL query and return results as a Polars DataFrame. Uses Arrow IPC streaming for efficient zero-copy data transfer. @@ -165,6 +220,9 @@ async def query_polars(self, sql: str, database: Optional[str] = None) -> Any: Args: sql: SQL query to execute. database: Target database. Uses client default if not specified. + dictionary: See :meth:`query_arrow`. Dictionary-encoded string + columns become polars ``Categorical`` columns. + compression: See :meth:`query_arrow`. Returns: polars.DataFrame with query results. @@ -181,10 +239,19 @@ async def query_polars(self, sql: str, database: Optional[str] = None) -> Any: "Install it with: pip install arc-client[polars]" ) from e - table = await self.query_arrow(sql, database) + table = await self.query_arrow( + sql, database, dictionary=dictionary, compression=compression + ) return pl.from_arrow(table) - async def query_arrow(self, sql: str, database: Optional[str] = None) -> pa.Table: + async def query_arrow( + self, + sql: str, + database: Optional[str] = None, + *, + dictionary: bool = False, + compression: Optional[str] = None, + ) -> pa.Table: """Execute a SQL query and return results as a PyArrow Table. Uses Arrow IPC streaming for efficient zero-copy data transfer. @@ -194,35 +261,44 @@ async def query_arrow(self, sql: str, database: Optional[str] = None) -> pa.Tabl Args: sql: SQL query to execute. database: Target database. Uses client default if not specified. + dictionary: Ask Arc to dictionary-encode low-cardinality string + columns (``x-arc-arrow-dictionary: true``); those columns + arrive as ``dictionary``. Smaller on the wire for + repetitive tags. + compression: Arrow IPC buffer compression, ``"zstd"`` or ``"lz4"`` + (``x-arc-arrow-compression``), decoded transparently by + pyarrow. ``None`` (default) sends uncompressed buffers. Returns: pyarrow.Table with query results and completion metadata in - its schema: arc.rows_capped, arc.row_cap, arc.execution_time_ms. - Missing cap metadata means unknown. Truncation raises ArcQueryError. + its schema: arc.rows_capped, arc.row_cap, arc.execution_time_ms, + and arc.query_id (Arc's ``X-Arc-Query-ID``, for + ``client.queries``). Missing cap metadata means unknown. + Truncation raises ArcQueryError. Raises: ArcQueryError: If the query fails. - ArcValidationError: If the SQL is invalid. + ArcValidationError: If the SQL is empty, or ``compression`` is not + ``"zstd"`` or ``"lz4"``. """ if not sql or not sql.strip(): raise ArcValidationError("SQL query cannot be empty") - headers = { - "Accept": "application/vnd.apache.arrow.stream", - } - if database: - headers["x-arc-database"] = database + headers = arrow_headers(database, dictionary, compression) try: response = await self._http.post( "/api/v1/query/arrow", json={"sql": sql}, headers=headers, + idempotent=True, ) return decode_arrow_response(response) - except ArcQueryError: + except (ArcServerError, ArcConnectionError) as e: + raise ArcQueryError(f"Query failed: {e}", status_code=_status(e)) from e + except ArcError: raise except pa.ArrowInvalid as e: raise ArcQueryError(f"Failed to parse Arrow response: {e}") from e @@ -249,21 +325,20 @@ async def estimate(self, sql: str, database: Optional[str] = None) -> EstimateRe if not sql or not sql.strip(): raise ArcValidationError("SQL query cannot be empty") - headers = {} - if database: - headers["x-arc-database"] = database - try: response = await self._http.post( "/api/v1/query/estimate", json={"sql": sql}, - headers=headers if headers else None, + headers=query_headers(database, False) or None, + idempotent=True, ) data = response.json() return EstimateResponse(**data) - except ArcQueryError: + except (ArcServerError, ArcConnectionError) as e: + raise ArcQueryError(f"Estimation failed: {e}", status_code=_status(e)) from e + except ArcError: raise except Exception as e: raise ArcQueryError(f"Estimation failed: {e}") from e @@ -294,7 +369,9 @@ async def list_measurements(self, database: Optional[str] = None) -> list[Measur return [MeasurementInfo(**m) for m in data.get("measurements", [])] - except ArcQueryError: + except (ArcServerError, ArcConnectionError) as e: + raise ArcQueryError(f"Failed to list measurements: {e}", status_code=_status(e)) from e + except ArcError: raise except Exception as e: raise ArcQueryError(f"Failed to list measurements: {e}") from e @@ -318,11 +395,59 @@ async def list_databases(self) -> list[str]: # Extract database names from result return [row[0] for row in result.data if row] - except ArcQueryError: + except ArcError: raise except Exception as e: raise ArcQueryError(f"Failed to list databases: {e}") from e + async def query_measurement( + self, + measurement: str, + database: Optional[str] = None, + *, + limit: Optional[int] = None, + offset: Optional[int] = None, + order_by: Optional[str] = None, + where: Optional[str] = None, + ) -> QueryResponse: + """Read rows of one measurement with ``GET /api/v1/query/:measurement``. + + Arc builds ``SELECT * FROM . [WHERE ] + ORDER BY LIMIT OFFSET `` itself and + validates every piece. Omitted arguments take Arc's defaults: + ``limit`` 100, ``offset`` 0, ``order_by`` ``"time DESC"``. + + Args: + measurement: Measurement (table) name. + database: Database name. This endpoint takes the database as a + query parameter, not the ``x-arc-database`` header; the client + default is sent when not specified, else Arc uses + ``"default"``. + limit: Maximum rows, 0..1,000,000. + offset: Rows to skip, >= 0. + order_by: ORDER BY clause, e.g. ``"time ASC"``. + where: WHERE clause without the keyword, e.g. ``"host = 'a'"``. + + Returns: + QueryResponse, as from :meth:`query`. + + Raises: + ArcValidationError: If a name or a numeric argument is invalid. + ArcQueryError: If Arc rejects the request (``status_code`` set). + """ + params = measurement_params( + measurement, database or self._config.database, limit, offset, order_by, where + ) + try: + response = await self._http.get(f"/api/v1/query/{measurement}", params=params) + return query_response_from_json(response) + except (ArcServerError, ArcConnectionError) as e: + raise ArcQueryError(f"Query failed: {e}", status_code=_status(e)) from e + except ArcError: + raise + except Exception as e: + raise ArcQueryError(f"Query failed: {e}") from e + async def show_tables(self, database: Optional[str] = None) -> QueryResponse: """Show tables/measurements in a database. diff --git a/src/arc_client/query/executor.py b/src/arc_client/query/executor.py index c9c64c1..c106c90 100644 --- a/src/arc_client/query/executor.py +++ b/src/arc_client/query/executor.py @@ -2,21 +2,31 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional +from typing import Any, Optional import pyarrow as pa from arc_client.config import ClientConfig -from arc_client.exceptions import ArcNotSupportedError, ArcQueryError, ArcValidationError +from arc_client.exceptions import ( + ArcConnectionError, + ArcError, + ArcQueryError, + ArcServerError, + ArcValidationError, +) from arc_client.http.sync_http import SyncHTTPClient from arc_client.models.query import EstimateResponse, MeasurementInfo, QueryResponse -from arc_client.query.arrow_response import decode_arrow_response +from arc_client.query.arrow_response import QUERY_ID_HEADER, decode_arrow_response from arc_client.query.msgpack_response import decode_msgpack_query_response +from arc_client.query.options import ( + arrow_headers, + measurement_params, + query_headers, + query_response_from_json, +) +from arc_client.query.options import status_code as _status from arc_client.validation import validate_name -if TYPE_CHECKING: - pass - class QueryClient: """Synchronous client for querying data from Arc. @@ -45,22 +55,38 @@ def __init__(self, http: SyncHTTPClient, config: ClientConfig) -> None: self._http = http self._config = config - def query(self, sql: str, database: Optional[str] = None) -> QueryResponse: + def query( + self, sql: str, database: Optional[str] = None, *, profile: bool = False + ) -> QueryResponse: """Execute a SQL query and return results as JSON. This method returns structured JSON data. For large result sets, consider using query_arrow() or query_polars() for better performance. + Queries only read, so a 429 or 503 (Arc's replication catch-up gate, a + partition removed mid-query, an unavailable cold tier) is retried with + backoff, honoring ``Retry-After``, up to ``max_retries``. + Args: sql: SQL query to execute. database: Target database. Uses client default if not specified. + profile: Ask Arc for server-side timing (``x-arc-profile``), + returned in ``QueryResponse.profile``. Returns: - QueryResponse with columns, data, row_count, and metadata. + QueryResponse with columns, data, row_count, metadata and + ``query_id``. Raises: - ArcQueryError: If the query fails. + ArcQueryError: If the query fails. ``status_code`` carries the HTTP + status when Arc answered with a generic error status (e.g. 400 + for invalid SQL). ArcValidationError: If the SQL is invalid. + ArcError: Typed failures (ArcAuthenticationError, + ArcPermissionError, ArcScopedAccessError, ArcRateLimitError, + ArcNotSupportedError, ...) are raised as themselves. Transport + failures (ArcConnectionError) are wrapped in ArcQueryError + with the original as ``__cause__``. Example: >>> result = client.query.query("SELECT * FROM cpu WHERE time > now() - '1h'") @@ -70,29 +96,27 @@ def query(self, sql: str, database: Optional[str] = None) -> QueryResponse: if not sql or not sql.strip(): raise ArcValidationError("SQL query cannot be empty") - headers = {} - if database: - headers["x-arc-database"] = database - try: response = self._http.post( "/api/v1/query", json={"sql": sql}, - headers=headers if headers else None, + headers=query_headers(database, profile) or None, + idempotent=True, ) - - data = response.json() - return QueryResponse(**data) - - except ArcQueryError: + return query_response_from_json(response) + except (ArcServerError, ArcConnectionError) as e: + raise ArcQueryError(f"Query failed: {e}", status_code=_status(e)) from e + except ArcError: raise except Exception as e: raise ArcQueryError(f"Query failed: {e}") from e - def query_msgpack(self, sql: str, database: Optional[str] = None) -> QueryResponse: + def query_msgpack( + self, sql: str, database: Optional[str] = None, *, profile: bool = False + ) -> QueryResponse: """Execute a SQL query and decode the MessagePack response. - **Experimental.** This uses Arc's ``/api/v1/query/msgpack`` endpoint, + This uses Arc's ``/api/v1/query/msgpack`` endpoint, which returns a compact columnar MessagePack envelope with native timestamp encoding without silently discarding nanoseconds. Exactly representable microseconds become UTC datetime; other values remain msgpack.Timestamp. @@ -103,38 +127,57 @@ def query_msgpack(self, sql: str, database: Optional[str] = None) -> QueryRespon The endpoint requires the Arc server to be built with the ``duckdb_arrow`` build tag; otherwise the server responds with HTTP 501. + A stream Arc fails after sending the 200 (client disconnect, deadline) + arrives as an incomplete MessagePack body and raises ArcQueryError; + partial rows are never returned (Arc's msgpack writer has no + ``truncated`` marker). + Args: sql: SQL query to execute. database: Target database. Uses client default if not specified. + profile: Ask Arc for server-side timing (``x-arc-profile``), + returned in ``QueryResponse.profile``. Returns: - QueryResponse with row-major ``data`` and populated ``types``. + QueryResponse with row-major ``data``, populated ``types`` and + ``query_id``. Raises: ArcValidationError: If the SQL is empty. ArcNotSupportedError: If the server lacks msgpack query support (501). - ArcQueryError: If the query fails. + ArcQueryError: If the query fails or the response is incomplete. """ if not sql or not sql.strip(): raise ArcValidationError("SQL query cannot be empty") - headers = {"Accept": "application/msgpack"} - if database: - headers["x-arc-database"] = database + headers = query_headers(database, profile) + headers["Accept"] = "application/msgpack" try: response = self._http.post( "/api/v1/query/msgpack", json={"sql": sql}, headers=headers, + idempotent=True, ) - return decode_msgpack_query_response(response.content) - except (ArcQueryError, ArcNotSupportedError, ArcValidationError): + return decode_msgpack_query_response( + response.content, query_id=response.headers.get(QUERY_ID_HEADER) + ) + except (ArcServerError, ArcConnectionError) as e: + raise ArcQueryError(f"Query failed: {e}", status_code=_status(e)) from e + except ArcError: raise except Exception as e: raise ArcQueryError(f"Query failed: {e}") from e - def query_pandas(self, sql: str, database: Optional[str] = None) -> Any: + def query_pandas( + self, + sql: str, + database: Optional[str] = None, + *, + dictionary: bool = False, + compression: Optional[str] = None, + ) -> Any: """Execute a SQL query and return results as a pandas DataFrame. Uses Arrow IPC streaming for efficient data transfer. Requires @@ -143,6 +186,9 @@ def query_pandas(self, sql: str, database: Optional[str] = None) -> Any: Args: sql: SQL query to execute. database: Target database. Uses client default if not specified. + dictionary: See :meth:`query_arrow`. Dictionary-encoded string + columns become pandas ``Categorical`` columns. + compression: See :meth:`query_arrow`. Returns: pandas.DataFrame with query results. @@ -163,10 +209,17 @@ def query_pandas(self, sql: str, database: Optional[str] = None) -> Any: "Install it with: pip install arc-client[pandas]" ) from e - table = self.query_arrow(sql, database) + table = self.query_arrow(sql, database, dictionary=dictionary, compression=compression) return table.to_pandas() - def query_polars(self, sql: str, database: Optional[str] = None) -> Any: + def query_polars( + self, + sql: str, + database: Optional[str] = None, + *, + dictionary: bool = False, + compression: Optional[str] = None, + ) -> Any: """Execute a SQL query and return results as a Polars DataFrame. Uses Arrow IPC streaming for efficient zero-copy data transfer. @@ -176,6 +229,9 @@ def query_polars(self, sql: str, database: Optional[str] = None) -> Any: Args: sql: SQL query to execute. database: Target database. Uses client default if not specified. + dictionary: See :meth:`query_arrow`. Dictionary-encoded string + columns become polars ``Categorical`` columns. + compression: See :meth:`query_arrow`. Returns: polars.DataFrame with query results. @@ -196,10 +252,17 @@ def query_polars(self, sql: str, database: Optional[str] = None) -> Any: "Install it with: pip install arc-client[polars]" ) from e - table = self.query_arrow(sql, database) + table = self.query_arrow(sql, database, dictionary=dictionary, compression=compression) return pl.from_arrow(table) - def query_arrow(self, sql: str, database: Optional[str] = None) -> pa.Table: + def query_arrow( + self, + sql: str, + database: Optional[str] = None, + *, + dictionary: bool = False, + compression: Optional[str] = None, + ) -> pa.Table: """Execute a SQL query and return results as a PyArrow Table. Uses Arrow IPC streaming for efficient zero-copy data transfer. @@ -209,15 +272,25 @@ def query_arrow(self, sql: str, database: Optional[str] = None) -> pa.Table: Args: sql: SQL query to execute. database: Target database. Uses client default if not specified. + dictionary: Ask Arc to dictionary-encode low-cardinality string + columns (``x-arc-arrow-dictionary: true``); those columns + arrive as ``dictionary``. Smaller on the wire for + repetitive tags. + compression: Arrow IPC buffer compression, ``"zstd"`` or ``"lz4"`` + (``x-arc-arrow-compression``), decoded transparently by + pyarrow. ``None`` (default) sends uncompressed buffers. Returns: pyarrow.Table with query results and completion metadata in - its schema: arc.rows_capped, arc.row_cap, arc.execution_time_ms. - Missing cap metadata means unknown. Truncation raises ArcQueryError. + its schema: arc.rows_capped, arc.row_cap, arc.execution_time_ms, + and arc.query_id (Arc's ``X-Arc-Query-ID``, for + ``client.queries``). Missing cap metadata means unknown. + Truncation raises ArcQueryError. Raises: ArcQueryError: If the query fails. - ArcValidationError: If the SQL is invalid. + ArcValidationError: If the SQL is empty, or ``compression`` is not + ``"zstd"`` or ``"lz4"``. Example: >>> table = client.query.query_arrow("SELECT * FROM cpu LIMIT 1000") @@ -227,22 +300,21 @@ def query_arrow(self, sql: str, database: Optional[str] = None) -> pa.Table: if not sql or not sql.strip(): raise ArcValidationError("SQL query cannot be empty") - headers = { - "Accept": "application/vnd.apache.arrow.stream", - } - if database: - headers["x-arc-database"] = database + headers = arrow_headers(database, dictionary, compression) try: response = self._http.post( "/api/v1/query/arrow", json={"sql": sql}, headers=headers, + idempotent=True, ) return decode_arrow_response(response) - except ArcQueryError: + except (ArcServerError, ArcConnectionError) as e: + raise ArcQueryError(f"Query failed: {e}", status_code=_status(e)) from e + except ArcError: raise except pa.ArrowInvalid as e: raise ArcQueryError(f"Failed to parse Arrow response: {e}") from e @@ -275,21 +347,20 @@ def estimate(self, sql: str, database: Optional[str] = None) -> EstimateResponse if not sql or not sql.strip(): raise ArcValidationError("SQL query cannot be empty") - headers = {} - if database: - headers["x-arc-database"] = database - try: response = self._http.post( "/api/v1/query/estimate", json={"sql": sql}, - headers=headers if headers else None, + headers=query_headers(database, False) or None, + idempotent=True, ) data = response.json() return EstimateResponse(**data) - except ArcQueryError: + except (ArcServerError, ArcConnectionError) as e: + raise ArcQueryError(f"Estimation failed: {e}", status_code=_status(e)) from e + except ArcError: raise except Exception as e: raise ArcQueryError(f"Estimation failed: {e}") from e @@ -325,7 +396,9 @@ def list_measurements(self, database: Optional[str] = None) -> list[MeasurementI return [MeasurementInfo(**m) for m in data.get("measurements", [])] - except ArcQueryError: + except (ArcServerError, ArcConnectionError) as e: + raise ArcQueryError(f"Failed to list measurements: {e}", status_code=_status(e)) from e + except ArcError: raise except Exception as e: raise ArcQueryError(f"Failed to list measurements: {e}") from e @@ -354,11 +427,59 @@ def list_databases(self) -> list[str]: # Extract database names from result return [row[0] for row in result.data if row] - except ArcQueryError: + except ArcError: raise except Exception as e: raise ArcQueryError(f"Failed to list databases: {e}") from e + def query_measurement( + self, + measurement: str, + database: Optional[str] = None, + *, + limit: Optional[int] = None, + offset: Optional[int] = None, + order_by: Optional[str] = None, + where: Optional[str] = None, + ) -> QueryResponse: + """Read rows of one measurement with ``GET /api/v1/query/:measurement``. + + Arc builds ``SELECT * FROM . [WHERE ] + ORDER BY LIMIT OFFSET `` itself and + validates every piece. Omitted arguments take Arc's defaults: + ``limit`` 100, ``offset`` 0, ``order_by`` ``"time DESC"``. + + Args: + measurement: Measurement (table) name. + database: Database name. This endpoint takes the database as a + query parameter, not the ``x-arc-database`` header; the client + default is sent when not specified, else Arc uses + ``"default"``. + limit: Maximum rows, 0..1,000,000. + offset: Rows to skip, >= 0. + order_by: ORDER BY clause, e.g. ``"time ASC"``. + where: WHERE clause without the keyword, e.g. ``"host = 'a'"``. + + Returns: + QueryResponse, as from :meth:`query`. + + Raises: + ArcValidationError: If a name or a numeric argument is invalid. + ArcQueryError: If Arc rejects the request (``status_code`` set). + """ + params = measurement_params( + measurement, database or self._config.database, limit, offset, order_by, where + ) + try: + response = self._http.get(f"/api/v1/query/{measurement}", params=params) + return query_response_from_json(response) + except (ArcServerError, ArcConnectionError) as e: + raise ArcQueryError(f"Query failed: {e}", status_code=_status(e)) from e + except ArcError: + raise + except Exception as e: + raise ArcQueryError(f"Query failed: {e}") from e + def show_tables(self, database: Optional[str] = None) -> QueryResponse: """Show tables/measurements in a database. diff --git a/src/arc_client/query/msgpack_response.py b/src/arc_client/query/msgpack_response.py index 6385f8e..0735e9b 100644 --- a/src/arc_client/query/msgpack_response.py +++ b/src/arc_client/query/msgpack_response.py @@ -1,4 +1,4 @@ -"""Decoder for the experimental MessagePack query response format. +"""Decoder for the MessagePack query response format. Arc's ``POST /api/v1/query/msgpack`` returns a **columnar** MessagePack envelope (``application/msgpack``): a map with keys ``success, columns, types, data, @@ -7,19 +7,28 @@ native MessagePack timestamp extensions. Values exactly representable by Python ``datetime`` are decoded to UTC datetimes; submicrosecond or out-of-range values remain ``msgpack.Timestamp`` objects, preserving their seconds and nanoseconds. -Per-cell nulls are possible. +Per-cell nulls are possible. Optional keys: ``rows_capped``/``row_cap`` (a +governance row cap was reached) and ``profile`` (``x-arc-profile: true``). + +Arc's msgpack writer declares the map's length before it streams the columns +and never emits ``truncated``/``truncation_reason`` (internal/api/ +query_msgpack.go streamMsgPackFromBatches). A stream that fails after the 200 +was sent (client disconnect, deadline, encoder panic) therefore arrives as an +incomplete MessagePack document. This decoder raises that as ``ArcQueryError`` +rather than returning a partial result. ``truncated`` is still read if a +server ever sends it, for parity with the JSON envelope. The error envelope is a 4-key map ``{success: false, error, execution_time_ms, timestamp}``. -This endpoint is experimental and requires the server's ``duckdb_arrow`` build -tag; without it the server responds 501, surfaced as ``ArcNotSupportedError`` by -the HTTP error handler before this decoder runs. +The endpoint requires the server's ``duckdb_arrow`` build tag; without it the +server responds 501, surfaced as ``ArcNotSupportedError`` by the HTTP error +handler before this decoder runs. """ from __future__ import annotations -from typing import Any +from typing import Any, Optional import msgpack @@ -27,7 +36,7 @@ from arc_client.models.query import QueryResponse -def decode_msgpack_query_response(content: bytes) -> QueryResponse: +def decode_msgpack_query_response(content: bytes, query_id: Optional[str] = None) -> QueryResponse: """Decode a MessagePack query response envelope into a :class:`QueryResponse`. The columnar ``data`` is transposed to row-major so the result matches the @@ -38,18 +47,34 @@ def decode_msgpack_query_response(content: bytes) -> QueryResponse: Args: content: Raw MessagePack response body. + query_id: The ``X-Arc-Query-ID`` response header, if any. Returns: A :class:`QueryResponse`. For a success envelope, ``data`` is row-major and ``types`` carries the per-column Arrow type names. Raises: - ArcQueryError: If the body cannot be decoded or reports ``success=false``. + ArcQueryError: If the body is incomplete (the server's stream ended + early), cannot be decoded, or reports ``success=false``. """ try: # timestamp=3 silently drops submicrosecond precision. Decode the native # values first, then convert only timestamps datetime can represent exactly. envelope = _decode_timestamps(msgpack.unpackb(content, timestamp=0, raw=False)) + except ValueError as e: + # msgpack reports a body that ends inside the declared map as + # "incomplete input", or, when a container header promises more items + # than the remaining bytes could hold, "exceeds max_*_len" (both + # ValueError). Arc sends such a body when its stream fails after the + # 200 was committed; the rows received are partial. + text = str(e).lower() + if "incomplete" in text or "exceeds max_" in text: + raise ArcQueryError( + "MessagePack query response is incomplete: the server's stream " + "ended before the result was complete (partial results are not " + "returned)" + ) from e + raise ArcQueryError(f"Failed to decode msgpack query response: {e}") from e except Exception as e: # pragma: no cover - malformed body is rare raise ArcQueryError(f"Failed to decode msgpack query response: {e}") from e @@ -82,6 +107,8 @@ def decode_msgpack_query_response(content: bytes) -> QueryResponse: row_cap=envelope.get("row_cap"), execution_time_ms=float(execution_time_ms), timestamp=str(timestamp) if timestamp is not None else None, + profile=envelope.get("profile"), + query_id=query_id, ) diff --git a/src/arc_client/query/options.py b/src/arc_client/query/options.py new file mode 100644 index 0000000..e26cddf --- /dev/null +++ b/src/arc_client/query/options.py @@ -0,0 +1,92 @@ +"""Request options and response helpers shared by the sync and async query clients.""" + +from __future__ import annotations + +from typing import Any, Optional + +import httpx + +from arc_client.exceptions import ArcValidationError +from arc_client.models.query import QueryResponse +from arc_client.query.arrow_response import QUERY_ID_HEADER +from arc_client.validation import validate_name + +# Arc accepts exactly these (case-insensitive) for x-arc-arrow-compression and +# answers 400 for anything else (internal/api/query_arrow.go). +ARROW_COMPRESSIONS = frozenset({"zstd", "lz4"}) + +# GET /api/v1/query/:measurement rejects a limit above this (query.go). +MAX_MEASUREMENT_LIMIT = 1_000_000 + + +def status_code(error: Exception) -> Optional[int]: + """The HTTP status an error carries, if any (None for transport failures).""" + value = getattr(error, "status_code", None) + return value if isinstance(value, int) else None + + +def query_headers(database: Optional[str], profile: bool) -> dict[str, str]: + """Headers for the JSON/msgpack/estimate query endpoints.""" + headers: dict[str, str] = {} + if database: + headers["x-arc-database"] = database + if profile: + # Arc compares the value with == "true" (query.go executeQuery). + headers["x-arc-profile"] = "true" + return headers + + +def arrow_headers( + database: Optional[str], dictionary: bool, compression: Optional[str] +) -> dict[str, str]: + """Headers for the Arrow IPC endpoint, validating the opt-in encodings.""" + headers = {"Accept": "application/vnd.apache.arrow.stream"} + if database: + headers["x-arc-database"] = database + if dictionary: + headers["x-arc-arrow-dictionary"] = "true" + if compression is not None: + codec = compression.strip().lower() + if codec not in ARROW_COMPRESSIONS: + raise ArcValidationError( + f"Invalid Arrow compression {compression!r}: must be 'zstd' or 'lz4'" + ) + headers["x-arc-arrow-compression"] = codec + return headers + + +def measurement_params( + measurement: str, + database: Optional[str], + limit: Optional[int], + offset: Optional[int], + order_by: Optional[str], + where: Optional[str], +) -> dict[str, Any]: + """Validate and build the query string for GET /api/v1/query/:measurement.""" + validate_name(measurement, "measurement") + params: dict[str, Any] = {} + if database: + params["database"] = validate_name(database, "database") + if limit is not None: + if isinstance(limit, bool) or not 0 <= limit <= MAX_MEASUREMENT_LIMIT: + raise ArcValidationError(f"limit must be between 0 and {MAX_MEASUREMENT_LIMIT}") + params["limit"] = limit + if offset is not None: + if isinstance(offset, bool) or offset < 0: + raise ArcValidationError("offset must be a non-negative integer") + params["offset"] = offset + if order_by is not None: + params["order_by"] = order_by + if where is not None: + params["where"] = where + return params + + +def query_response_from_json(response: httpx.Response) -> QueryResponse: + """Build a QueryResponse from a JSON body plus the X-Arc-Query-ID header.""" + data = response.json() + query_id = response.headers.get(QUERY_ID_HEADER) + if query_id and isinstance(data, dict) and not data.get("query_id"): + data = {**data, "query_id": query_id} + return QueryResponse.model_validate(data) diff --git a/tests/unit/http/test_query_parity.py b/tests/unit/http/test_query_parity.py new file mode 100644 index 0000000..441f873 --- /dev/null +++ b/tests/unit/http/test_query_parity.py @@ -0,0 +1,397 @@ +"""Query endpoint contracts against Arc main (2027.01): retries, errors, options.""" + +from __future__ import annotations + +import gzip +import io +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, ClassVar + +import httpx +import msgpack +import pyarrow as pa +import pyarrow.ipc as ipc +import pytest +import zstandard +from pytest_httpx import HTTPXMock + +from arc_client.exceptions import ( + ArcAuthenticationError, + ArcLicenseError, + ArcPermissionError, + ArcQueryError, + ArcRateLimitError, + ArcScopedAccessError, + ArcValidationError, +) + +from ._parity import MODES, call, make_client + +JSON_OK = { + "success": True, + "columns": ["host", "v"], + "data": [["a", 1]], + "row_count": 1, + "execution_time_ms": 2, + "timestamp": "2027-01-01T00:00:00Z", +} + + +def arrow_body() -> bytes: + table = pa.table({"host": ["a", "a"], "v": [1, 2]}) + sink = io.BytesIO() + with ipc.new_stream(sink, table.schema) as writer: + writer.write_table(table) + return sink.getvalue() + + +def msgpack_body(**extra: Any) -> bytes: + body = {**JSON_OK, "data": [["a"], [1]], "types": ["utf8", "int64"], **extra} + return msgpack.packb(body, use_bin_type=True) + + +ENDPOINTS = { + "query": ("/api/v1/query", lambda c: c.query.query("SELECT 1")), + "msgpack": ("/api/v1/query/msgpack", lambda c: c.query.query_msgpack("SELECT 1")), + "arrow": ("/api/v1/query/arrow", lambda c: c.query.query_arrow("SELECT 1")), + "estimate": ("/api/v1/query/estimate", lambda c: c.query.estimate("SELECT 1")), +} + + +def ok_response(httpx_mock: HTTPXMock, endpoint: str) -> None: + path = ENDPOINTS[endpoint][0] + url = f"http://h:8000{path}" + if endpoint == "msgpack": + httpx_mock.add_response( + url=url, content=msgpack_body(), headers={"Content-Type": "application/msgpack"} + ) + elif endpoint == "arrow": + httpx_mock.add_response( + url=url, + content=arrow_body(), + headers={"Content-Type": "application/vnd.apache.arrow.stream"}, + ) + elif endpoint == "estimate": + httpx_mock.add_response(url=url, json={"success": True, "estimated_rows": 3}) + else: + httpx_mock.add_response(url=url, json=JSON_OK) + + +# --- Bug 1: query POSTs are reads and retry 429/503 ------------------------ + + +@MODES +@pytest.mark.parametrize("endpoint", list(ENDPOINTS)) +@pytest.mark.parametrize("status", [429, 503]) +async def test_query_post_retries_retryable_status( + httpx_mock: HTTPXMock, async_mode: bool, endpoint: str, status: int +) -> None: + path, run = ENDPOINTS[endpoint] + # Arc's replication catch-up gate (query.go checkReplicationReady). + httpx_mock.add_response( + url=f"http://h:8000{path}", + status_code=status, + headers={"Retry-After": "0"}, + json={"success": False, "error": "replication_catch_up_in_progress"}, + ) + ok_response(httpx_mock, endpoint) + async with make_client(async_mode, max_retries=2) as client: + await call(run(client)) + assert len(httpx_mock.get_requests()) == 2 + + +@MODES +async def test_query_503_exhausted_raises_query_error_with_status( + httpx_mock: HTTPXMock, async_mode: bool +) -> None: + httpx_mock.add_response(status_code=503, headers={"Retry-After": "0"}, is_reusable=True) + async with make_client(async_mode, max_retries=2) as client: + with pytest.raises(ArcQueryError) as info: + await call(client.query.query("SELECT 1")) + assert info.value.status_code == 503 + assert len(httpx_mock.get_requests()) == 3 + + +@MODES +async def test_query_read_timeout_is_not_replayed(httpx_mock: HTTPXMock, async_mode: bool) -> None: + # Arc already ran (or is running) the query; replaying would multiply cost. + httpx_mock.add_exception(httpx.ReadTimeout("slow")) + async with make_client(async_mode, max_retries=3) as client: + with pytest.raises(ArcQueryError): + await call(client.query.query("SELECT 1")) + assert len(httpx_mock.get_requests()) == 1 + + +@MODES +async def test_query_bad_sql_is_still_query_error(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(status_code=400, json={"success": False, "error": "bad sql"}) + async with make_client(async_mode, max_retries=3) as client: + with pytest.raises(ArcQueryError, match="bad sql") as info: + await call(client.query.query("SELECT nope")) + assert info.value.status_code == 400 + assert len(httpx_mock.get_requests()) == 1 + + +# --- Bug 2: typed errors are not rewrapped --------------------------------- + +TYPED = [ + pytest.param(401, "invalid token", ArcAuthenticationError, id="401"), + pytest.param(403, "token does not have 'read' permission", ArcPermissionError, id="403"), + pytest.param( + 403, "access denied: no read permission for database 'x'", ArcScopedAccessError, id="scoped" + ), + pytest.param(403, "requires an enterprise license", ArcLicenseError, id="license"), + pytest.param(429, "slow down", ArcRateLimitError, id="429"), +] + + +@MODES +@pytest.mark.parametrize("endpoint", list(ENDPOINTS)) +@pytest.mark.parametrize(("status", "message", "expected"), TYPED) +async def test_query_typed_errors_surface_unchanged( + httpx_mock: HTTPXMock, + async_mode: bool, + endpoint: str, + status: int, + message: str, + expected: type[Exception], +) -> None: + _, run = ENDPOINTS[endpoint] + httpx_mock.add_response(status_code=status, json={"error": message}) + async with make_client(async_mode, max_retries=0) as client: + with pytest.raises(expected) as info: + await call(run(client)) + assert not isinstance(info.value, ArcQueryError) + + +@MODES +async def test_list_measurements_typed_error_unchanged( + httpx_mock: HTTPXMock, async_mode: bool +) -> None: + httpx_mock.add_response(status_code=401, json={"error": "expired"}) + async with make_client(async_mode, max_retries=0) as client: + with pytest.raises(ArcAuthenticationError): + await call(client.query.list_measurements()) + + +# --- Item 9/10: profile and query ID --------------------------------------- + +PROFILE = { + "total_ms": 3.5, + "planner_ms": 0.5, + "execution_ms": 3.0, + "rows_scanned": 10, + "latency_ms": 2.9, +} + + +@MODES +async def test_json_query_profile_and_query_id(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(json={**JSON_OK, "profile": PROFILE}, headers={"X-Arc-Query-ID": "q-1"}) + async with make_client(async_mode) as client: + result = await call(client.query.query("SELECT 1", profile=True)) + request = httpx_mock.get_request() + assert request.headers["x-arc-profile"] == "true" + assert result.profile is not None + assert result.profile.rows_scanned == 10 + assert result.profile.total_ms == 3.5 + assert result.query_id == "q-1" + + +@MODES +async def test_query_without_profile_sends_no_header( + httpx_mock: HTTPXMock, async_mode: bool +) -> None: + httpx_mock.add_response(json=JSON_OK) + async with make_client(async_mode) as client: + result = await call(client.query.query("SELECT 1")) + assert "x-arc-profile" not in httpx_mock.get_request().headers + assert result.profile is None + assert result.query_id is None + + +@MODES +async def test_msgpack_query_profile_and_query_id(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response( + content=msgpack_body(profile=PROFILE), + headers={"Content-Type": "application/msgpack", "X-Arc-Query-ID": "q-2"}, + ) + async with make_client(async_mode) as client: + result = await call(client.query.query_msgpack("SELECT 1", profile=True)) + assert httpx_mock.get_request().headers["x-arc-profile"] == "true" + assert result.profile is not None + assert result.profile.planner_ms == 0.5 + assert result.query_id == "q-2" + assert result.data == [["a", 1]] + + +@MODES +async def test_arrow_options_and_query_id(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response( + content=arrow_body(), + headers={ + "Content-Type": "application/vnd.apache.arrow.stream", + "X-Arc-Query-ID": "q-3", + }, + ) + async with make_client(async_mode) as client: + table = await call( + client.query.query_arrow("SELECT 1", dictionary=True, compression="ZSTD") + ) + request = httpx_mock.get_request() + assert request.headers["x-arc-arrow-dictionary"] == "true" + assert request.headers["x-arc-arrow-compression"] == "zstd" + assert table.schema.metadata[b"arc.query_id"] == b"q-3" + assert table.num_rows == 2 + + +@MODES +async def test_arrow_defaults_send_no_encoding_headers( + httpx_mock: HTTPXMock, async_mode: bool +) -> None: + httpx_mock.add_response( + content=arrow_body(), headers={"Content-Type": "application/vnd.apache.arrow.stream"} + ) + async with make_client(async_mode) as client: + table = await call(client.query.query_arrow("SELECT 1")) + headers = httpx_mock.get_request().headers + assert "x-arc-arrow-dictionary" not in headers + assert "x-arc-arrow-compression" not in headers + assert b"arc.query_id" not in (table.schema.metadata or {}) + + +@MODES +async def test_arrow_rejects_unknown_compression(async_mode: bool) -> None: + async with make_client(async_mode) as client: + with pytest.raises(ArcValidationError, match="zstd"): + await call(client.query.query_arrow("SELECT 1", compression="gzip")) + + +@MODES +async def test_arrow_options_reach_polars(httpx_mock: HTTPXMock, async_mode: bool) -> None: + pytest.importorskip("polars") + httpx_mock.add_response( + content=arrow_body(), headers={"Content-Type": "application/vnd.apache.arrow.stream"} + ) + async with make_client(async_mode) as client: + frame = await call(client.query.query_polars("SELECT 1", compression="lz4")) + assert httpx_mock.get_request().headers["x-arc-arrow-compression"] == "lz4" + assert frame.height == 2 + + +# --- Item 11: msgpack truncation is a decode failure, never partial rows --- + + +@MODES +async def test_msgpack_incomplete_body_raises(httpx_mock: HTTPXMock, async_mode: bool) -> None: + body = msgpack_body() + httpx_mock.add_response(content=body[:-6], headers={"Content-Type": "application/msgpack"}) + async with make_client(async_mode) as client: + with pytest.raises(ArcQueryError, match="incomplete"): + await call(client.query.query_msgpack("SELECT 1")) + + +# --- Item 12: GET /api/v1/query/:measurement -------------------------------- + + +@MODES +async def test_query_measurement_params(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(json=JSON_OK, headers={"X-Arc-Query-ID": "q-4"}) + async with make_client(async_mode, database="metrics") as client: + result = await call( + client.query.query_measurement( + "cpu", limit=10, offset=5, order_by="time ASC", where="host = 'a'" + ) + ) + request = httpx_mock.get_request() + assert request.method == "GET" + assert request.url.path == "/api/v1/query/cpu" + assert dict(request.url.params) == { + "database": "metrics", + "limit": "10", + "offset": "5", + "order_by": "time ASC", + "where": "host = 'a'", + } + assert result.data == [["a", 1]] + assert result.query_id == "q-4" + + +@MODES +async def test_query_measurement_defaults_left_to_arc( + httpx_mock: HTTPXMock, async_mode: bool +) -> None: + httpx_mock.add_response(json=JSON_OK) + async with make_client(async_mode) as client: + await call(client.query.query_measurement("cpu", "db1")) + assert dict(httpx_mock.get_request().url.params) == {"database": "db1"} + + +@MODES +@pytest.mark.parametrize( + "kwargs", + [{"limit": -1}, {"limit": 1_000_001}, {"offset": -1}, {"measurement": "bad name"}], +) +async def test_query_measurement_validates(async_mode: bool, kwargs: dict[str, Any]) -> None: + kwargs = dict(kwargs) + measurement = kwargs.pop("measurement", "cpu") + async with make_client(async_mode) as client: + with pytest.raises(ArcValidationError): + await call(client.query.query_measurement(measurement, **kwargs)) + + +# --- Item 15: compressed JSON/msgpack responses ---------------------------- +# A real HTTP/1.1 server: pytest-httpx pre-reads mocked bodies, which runs the +# content decoder twice and is not what a socket does. + + +class _CompressingHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + seen: ClassVar[list[str]] = [] + + def do_POST(self) -> None: # noqa: N802 - http.server API + self.rfile.read(int(self.headers["Content-Length"])) + accept = self.headers.get("Accept-Encoding", "") + self.seen.append(accept) + if self.path.endswith("/msgpack"): + raw, content_type = msgpack_body(), "application/msgpack" + else: + raw, content_type = json.dumps(JSON_OK).encode(), "application/json" + if "zstd" in accept: + # Streamed blocks inside one frame, as Arc's pooled encoder writes. + compressor = zstandard.ZstdCompressor().compressobj() + data = compressor.compress(raw) + compressor.flush() + encoding = "zstd" + else: + data, encoding = gzip.compress(raw), "gzip" + self.send_response(200) + self.send_header("Content-Type", content_type) + self.send_header("Content-Encoding", encoding) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, *args: Any) -> None: + pass + + +@MODES +@pytest.mark.parametrize("wire", ["json", "msgpack"]) +async def test_compressed_query_responses_decode(async_mode: bool, wire: str) -> None: + # Arc negotiates zstd (preferred) or gzip from Accept-Encoding on the JSON + # and msgpack paths (internal/api/response_compression.go). httpx sends + # "gzip, deflate" and, from 0.27.1 with zstandard installed, "zstd". + server = ThreadingHTTPServer(("127.0.0.1", 0), _CompressingHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + async with make_client(async_mode, host="127.0.0.1", port=server.server_address[1]) as c: + run = c.query.query_msgpack if wire == "msgpack" else c.query.query + result = await call(run("SELECT 1")) + finally: + server.shutdown() + server.server_close() + assert result.data == [["a", 1]] + assert "gzip" in _CompressingHandler.seen[-1] From 387ac79828369b3711d18804a38f4d0ab80a4bf7 Mon Sep 17 00:00:00 2001 From: Ignacio Van Droogenbroeck <64545348+xe-nvdk@users.noreply.github.com> Date: Mon, 2 Nov 2026 09:06:25 -0600 Subject: [PATCH 13/20] feat(write): precision, tag_keys, partial writes and ingest refusals - write_line_protocol/write_point take precision= (ns|us|ms|s) sent as ?precision=; write_point sends the timestamp unconverted when it is set. - tag_columns is no longer a no-op: write_columnar/write_dataframe and the buffered writers send the payload's tag_keys (Arc 2027.01). - Writes resend the 503 refusals Arc documents as 'nothing was written'; wal_not_durable and every other failure are never resent. - Arc's 400 'partial write: N points dropped' raises ArcPartialWriteError (dropped, others_written). Buffered writers discard that batch, keep flushing other measurements, and report the error once. - Auth, permission and rate-limit errors on writes surface as themselves. --- src/arc_client/ingestion/async_buffered.py | 97 +++++- src/arc_client/ingestion/async_writer.py | 75 ++++- src/arc_client/ingestion/buffered.py | 99 ++++++- src/arc_client/ingestion/msgpack.py | 63 +++- src/arc_client/ingestion/write_support.py | 92 ++++++ src/arc_client/ingestion/writer.py | 96 ++++-- tests/unit/http/test_write_parity.py | 325 +++++++++++++++++++++ tests/unit/test_buffered_partial_write.py | 169 +++++++++++ 8 files changed, 953 insertions(+), 63 deletions(-) create mode 100644 src/arc_client/ingestion/write_support.py create mode 100644 tests/unit/http/test_write_parity.py create mode 100644 tests/unit/test_buffered_partial_write.py diff --git a/src/arc_client/ingestion/async_buffered.py b/src/arc_client/ingestion/async_buffered.py index 6f5c582..797c575 100644 --- a/src/arc_client/ingestion/async_buffered.py +++ b/src/arc_client/ingestion/async_buffered.py @@ -5,7 +5,10 @@ import asyncio import time from collections import defaultdict -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Optional + +from arc_client.exceptions import ArcPartialWriteError +from arc_client.ingestion.msgpack import validate_tag_keys if TYPE_CHECKING: from arc_client.ingestion.async_writer import AsyncWriteClient @@ -14,6 +17,23 @@ class AsyncBufferedWriter: """Async buffered writer that batches records for optimal throughput. + Failure semantics: + + * A write Arc answers with a partial write (``ArcPartialWriteError``, + HTTP 400 "partial write: N points dropped") has been applied as far as + Arc could apply it. The batch is **discarded**, never resent (a resend + would duplicate every stored point), and the error is raised once: by + the flush that sent it, or, for a background flush, by the next + ``write``/``write_columnar`` call (before it buffers anything) or the + next ``flush``/``close`` (after it has flushed the remaining data). + The writer stays usable. + * Any other failure keeps the batch and makes the writer terminal: later + calls raise that failure and nothing is resent, because the server-side + outcome may be unknown. + + Tags passed in ``write(record)["tags"]`` and ``write_columnar(..., + tag_columns=...)`` are sent as the batch's ``tag_keys``. + Example: >>> async with client.write.buffered(batch_size=10000) as buffer: ... for record in records: @@ -39,6 +59,11 @@ def __init__( self._closed = False self._closing = False self._flush_error: BaseException | None = None + # A partial write from a background flush, reported once by the next + # foreground call (see the class docstring). + self._partial_error: Optional[ArcPartialWriteError] = None + # Columns sent as tag_keys, per measurement, in first-seen order. + self._tag_keys: dict[str, dict[str, None]] = defaultdict(dict) # Background task that flushes on flush_interval even when idle. self._flush_task: asyncio.Task[None] | None = None @@ -72,6 +97,8 @@ async def write(self, record: dict[str, Any]) -> None: raise RuntimeError("Buffered writer is closed") self._buffers[measurement].append(columns) self._record_counts[measurement] += 1 + for key in tags: + self._tag_keys[measurement][key] = None if self._record_counts[measurement] >= self._batch_size: await self._flush_measurement_unlocked(measurement) @@ -83,10 +110,12 @@ async def write_columnar( self, measurement: str, columns: dict[str, list[Any]], + tag_columns: Optional[list[str]] = None, ) -> None: """Write columnar data to the buffer.""" if not columns: return + validate_tag_keys(tag_columns, columns) num_records = len(next(iter(columns.values()))) @@ -96,6 +125,8 @@ async def write_columnar( raise RuntimeError("Buffered writer is closed") self._buffers[measurement].append(columns) self._record_counts[measurement] += num_records + for key in tag_columns or (): + self._tag_keys[measurement][key] = None if self._record_counts[measurement] >= self._batch_size: await self._flush_measurement_unlocked(measurement) @@ -109,9 +140,14 @@ async def flush(self) -> None: A failed write leaves its data pending and makes the buffer terminal: subsequent flush/close calls raise that failure without retrying data whose server outcome may be unknown. Producers wait while I/O runs. + + A partial write (``ArcPartialWriteError``) is the exception: its batch + is discarded and the error raised once, after the other measurements + were flushed. """ async with self._lock: await self._flush_all_unlocked() + self._raise_pending_partial() async def _flush_loop(self) -> None: """Periodically flush measurements whose data has aged past the interval.""" @@ -129,15 +165,33 @@ async def _flush_loop(self) -> None: self._buffers and time.monotonic() - self._last_flush_time >= self._flush_interval ): - await self._flush_all_unlocked() + try: + await self._flush_all_unlocked() + except ArcPartialWriteError as error: + # Batch already discarded; report it once from the + # foreground and keep the timer running. + self._partial_error = error except Exception as error: # Foreground flush/close will surface this error. Do not retry an # uncertain outcome, or leave a task exception unobserved. self._flush_error = error def _raise_if_failed(self) -> None: + """Raise a terminal failure, then a pending partial write (once).""" if self._flush_error is not None: raise self._flush_error + self._raise_pending_partial() + + def _raise_pending_partial(self) -> None: + error, self._partial_error = self._partial_error, None + if error is not None: + raise error + + def _discard(self, measurement: str) -> None: + """Drop a measurement's buffered batch. Must hold lock.""" + self._buffers[measurement] = [] + self._record_counts[measurement] = 0 + self._tag_keys.pop(measurement, None) async def _flush_measurement_unlocked(self, measurement: str) -> None: """Flush a single measurement. Must hold lock.""" @@ -148,22 +202,41 @@ async def _flush_measurement_unlocked(self, measurement: str) -> None: # Retain data and the lock until acknowledgement. Concurrent timer, # manual, and close flushes must never send the same batch twice. try: - await self._client.write_columnar(measurement, merged) + tags = list(self._tag_keys.get(measurement, ())) + if tags: + await self._client.write_columnar(measurement, merged, tag_columns=tags) + else: + await self._client.write_columnar(measurement, merged) + except ArcPartialWriteError: + # Arc stored what it could; resending would duplicate it. + self._discard(measurement) + self._last_flush_time = time.monotonic() + raise except BaseException as error: self._flush_error = error raise - self._buffers[measurement] = [] - self._record_counts[measurement] = 0 + self._discard(measurement) self._last_flush_time = time.monotonic() async def _flush_all_unlocked(self) -> None: - """Flush all measurements. Must hold lock.""" - self._raise_if_failed() + """Flush all measurements. Must hold lock. + + A partial write does not stop the other measurements from flushing; the + first one is raised after they have been sent. + """ + if self._flush_error is not None: + raise self._flush_error + partial: Optional[ArcPartialWriteError] = None measurements = list(self._buffers.keys()) for measurement in measurements: if self._buffers[measurement]: - await self._flush_measurement_unlocked(measurement) + try: + await self._flush_measurement_unlocked(measurement) + except ArcPartialWriteError as error: + partial = partial or error + if partial is not None: + raise partial def _merge_columnar(self, batches: list[dict[str, list[Any]]]) -> dict[str, list[Any]]: """Merge multiple columnar batches into one. @@ -210,8 +283,14 @@ async def close(self) -> None: self._flush_task = None async with self._lock: - await self._flush_all_unlocked() + try: + await self._flush_all_unlocked() + except ArcPartialWriteError: + # Everything was flushed or discarded; the writer is done. + self._closed = True + raise self._closed = True + self._raise_pending_partial() async def __aenter__(self) -> AsyncBufferedWriter: if self._flush_task is None and self._flush_interval > 0: diff --git a/src/arc_client/ingestion/async_writer.py b/src/arc_client/ingestion/async_writer.py index 985a4bd..5d2a24d 100644 --- a/src/arc_client/ingestion/async_writer.py +++ b/src/arc_client/ingestion/async_writer.py @@ -6,8 +6,9 @@ from arc_client.compression import Compression from arc_client.config import ClientConfig -from arc_client.exceptions import ArcIngestionError, ArcValidationError +from arc_client.exceptions import ArcError, ArcIngestionError, ArcValidationError from arc_client.http.async_http import AsyncHTTPClient +from arc_client.http.base import is_retryable_write_refusal from arc_client.ingestion.compression import compress as compress_data from arc_client.ingestion.line_protocol import format_line_protocol from arc_client.ingestion.msgpack import ( @@ -15,6 +16,11 @@ encode_columnar, encode_records, ) +from arc_client.ingestion.write_support import ( + as_write_error, + tle_stats_body, + validate_precision, +) # Compression argument accepted by write methods: a codec, its name, a bool # (back-compat), or None to use the client default. @@ -54,6 +60,7 @@ async def write_columnar( database: Optional[str] = None, compress: CompressArg = None, time_unit: str = "us", + tag_columns: Optional[list[str]] = None, ) -> None: """Write columnar data to Arc (highest performance method). @@ -63,9 +70,12 @@ async def write_columnar( database: Target database. Uses client default if not specified. compress: Whether to gzip compress. Uses client default if not specified. time_unit: Unit of timestamps - "s", "ms", or "us" (default). + tag_columns: Columns Arc should store as tags (indexed dimensions) + rather than fields. Sent as the payload's ``tag_keys``; see + the sync client's ``write_columnar`` for details. """ try: - data = encode_columnar(measurement, columns, time_unit) + data = encode_columnar(measurement, columns, time_unit, tag_keys=tag_columns) except ArcValidationError: raise except Exception as e: @@ -106,17 +116,20 @@ async def write_dataframe( except Exception as e: raise ArcValidationError(f"Failed to convert DataFrame: {e}") from e - await self.write_columnar(measurement, columns, database, compress) + await self.write_columnar(measurement, columns, database, compress, tag_columns=tag_columns) async def write_line_protocol( self, lines: str | list[str], database: Optional[str] = None, compress: CompressArg = None, + precision: Optional[str] = None, ) -> None: """Write data using InfluxDB Line Protocol format.""" data = "\n".join(lines) if isinstance(lines, list) else lines - await self._write_line_protocol(data.encode("utf-8"), database, compress) + await self._write_line_protocol( + data.encode("utf-8"), database, compress, validate_precision(precision) + ) async def write_point( self, @@ -126,10 +139,15 @@ async def write_point( timestamp: Optional[int] = None, database: Optional[str] = None, compress: CompressArg = None, + precision: Optional[str] = None, ) -> None: """Write a single data point using Line Protocol.""" - line = format_line_protocol(measurement, fields, tags, timestamp) - await self._write_line_protocol(line.encode("utf-8"), database, compress) + precision = validate_precision(precision) + # With an explicit precision the timestamp goes out as-is ("ns" means + # "no conversion") and Arc interprets it in that unit. + time_unit = "us" if precision is None else "ns" + line = format_line_protocol(measurement, fields, tags, timestamp, time_unit) + await self._write_line_protocol(line.encode("utf-8"), database, compress, precision) async def write_tle( self, @@ -158,9 +176,11 @@ async def tle_stats(self) -> dict[str, Any]: try: response = await self._http.get("/api/v1/write/tle/stats") body = response.json() + except ArcError: + raise except Exception as e: raise ArcIngestionError(f"Failed to get TLE stats: {e}") from e - return body.get("stats", body) if isinstance(body, dict) else {} + return tle_stats_body(body) def buffered( self, @@ -186,11 +206,19 @@ async def _post_write( database: Optional[str], compress: CompressArg, extra_headers: Optional[dict[str, str]] = None, + params: Optional[dict[str, Any]] = None, ) -> None: """Compress (if requested) and POST a write payload. Arc auto-detects gzip/zstd from the payload's magic bytes, so no ``Content-Encoding`` header is sent. + + A 503 Arc marks as an ingest refusal that applied nothing + (``X-Arc-Ingest-Fenced``, or ``X-Arc-Ingest-Refused`` with any reason + but ``wal_not_durable``) is resent up to ``max_retries`` times, + honoring ``Retry-After``. Nothing else is resent: a partial write, a + ``wal_not_durable`` refusal or any other failure may have stored + data. Errors are mapped by ``as_write_error``. """ codec = self._resolve_codec(compress) data = compress_data(data, codec) @@ -202,15 +230,24 @@ async def _post_write( headers["x-arc-database"] = database try: - response = await self._http.post(path, content=data, headers=headers) - if response.status_code not in (200, 204): - raise ArcIngestionError( - f"Write failed with status {response.status_code}: {response.text}" - ) - except ArcIngestionError: - raise + response = await self._http.post( + path, + content=data, + headers=headers, + params=params, + retry_if=is_retryable_write_refusal, + ) except Exception as e: - raise ArcIngestionError(f"Failed to write data: {e}") from e + mapped = as_write_error(e) + if mapped is e: + raise + raise mapped from e + # Arc returns 204 No Content on success + if response.status_code not in (200, 204): + raise ArcIngestionError( + f"Write failed with status {response.status_code}: {response.text}", + status_code=response.status_code, + ) async def _write_msgpack( self, @@ -228,8 +265,14 @@ async def _write_line_protocol( data: bytes, database: Optional[str], compress: CompressArg, + precision: Optional[str] = None, ) -> None: """Send Line Protocol data to Arc.""" await self._post_write( - "/api/v1/write/line-protocol", data, "text/plain", database, compress + "/api/v1/write/line-protocol", + data, + "text/plain", + database, + compress, + params={"precision": precision} if precision else None, ) diff --git a/src/arc_client/ingestion/buffered.py b/src/arc_client/ingestion/buffered.py index 9f23843..e699918 100644 --- a/src/arc_client/ingestion/buffered.py +++ b/src/arc_client/ingestion/buffered.py @@ -9,7 +9,10 @@ import threading import time from collections import defaultdict -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Optional + +from arc_client.exceptions import ArcPartialWriteError +from arc_client.ingestion.msgpack import validate_tag_keys if TYPE_CHECKING: from arc_client.ingestion.writer import WriteClient @@ -23,6 +26,23 @@ class BufferedWriter: - flush_interval seconds have passed since last flush - The context manager exits + Failure semantics: + + * A write Arc answers with a partial write (``ArcPartialWriteError``, + HTTP 400 "partial write: N points dropped") has been applied as far as + Arc could apply it. The batch is **discarded**, never resent (a resend + would duplicate every stored point), and the error is raised once: by + the flush that sent it, or, for a background flush, by the next + ``write``/``write_columnar`` call (before it buffers anything) or the + next ``flush``/``close`` (after it has flushed the remaining data). + The writer stays usable. + * Any other failure keeps the batch and makes the writer terminal: later + calls raise that failure and nothing is resent, because the server-side + outcome may be unknown. + + Tags passed in ``write(record)["tags"]`` and ``write_columnar(..., + tag_columns=...)`` are sent as the batch's ``tag_keys``. + Example: >>> with client.write.buffered(batch_size=10000) as buffer: ... for record in records: @@ -60,6 +80,11 @@ def __init__( self._closed = False self._closing = False self._flush_error: BaseException | None = None + # A partial write from a background flush, reported once by the next + # foreground call (see the class docstring). + self._partial_error: Optional[ArcPartialWriteError] = None + # Columns sent as tag_keys, per measurement, in first-seen order. + self._tag_keys: dict[str, dict[str, None]] = defaultdict(dict) # Background timer that flushes on flush_interval even when no writes # arrive. Started on __enter__ and torn down on close(). @@ -106,6 +131,8 @@ def write(self, record: dict[str, Any]) -> None: raise RuntimeError("Buffered writer is closed") self._buffers[measurement].append(columns) self._record_counts[measurement] += 1 + for key in tags: + self._tag_keys[measurement][key] = None # Check if we should flush this measurement if self._record_counts[measurement] >= self._batch_size: @@ -119,15 +146,19 @@ def write_columnar( self, measurement: str, columns: dict[str, list[Any]], + tag_columns: Optional[list[str]] = None, ) -> None: """Write columnar data to the buffer. Args: measurement: The measurement name. columns: Dictionary of column name to value list. + tag_columns: Columns of ``columns`` that are tags; sent as the + batch's ``tag_keys``. """ if not columns: return + validate_tag_keys(tag_columns, columns) num_records = len(next(iter(columns.values()))) @@ -137,6 +168,8 @@ def write_columnar( raise RuntimeError("Buffered writer is closed") self._buffers[measurement].append(columns) self._record_counts[measurement] += num_records + for key in tag_columns or (): + self._tag_keys[measurement][key] = None # Check if we should flush this measurement if self._record_counts[measurement] >= self._batch_size: @@ -152,9 +185,14 @@ def flush(self) -> None: A failed write leaves its data pending and makes the buffer terminal: subsequent flush/close calls raise that failure without retrying data whose server outcome may be unknown. Producers wait while I/O runs. + + A partial write (``ArcPartialWriteError``) is the exception: its batch + is discarded and the error raised once, after the other measurements + were flushed. """ with self._lock: self._flush_all() + self._raise_pending_partial() def _start_flush_thread(self) -> None: """Start the background flush timer (idempotent).""" @@ -177,7 +215,12 @@ def _flush_loop(self) -> None: self._buffers and time.monotonic() - self._last_flush_time >= self._flush_interval ): - self._flush_all() + try: + self._flush_all() + except ArcPartialWriteError as error: + # Batch already discarded; report it once from the + # foreground and keep the timer running. + self._partial_error = error except Exception as error: # Keep the error observable on foreground operations; never retry # an uncertain write automatically from the background thread. @@ -185,8 +228,21 @@ def _flush_loop(self) -> None: self._flush_error = error def _raise_if_failed(self) -> None: + """Raise a terminal failure, then a pending partial write (once).""" if self._flush_error is not None: raise self._flush_error + self._raise_pending_partial() + + def _raise_pending_partial(self) -> None: + error, self._partial_error = self._partial_error, None + if error is not None: + raise error + + def _discard(self, measurement: str) -> None: + """Drop a measurement's buffered batch. Must hold lock.""" + self._buffers[measurement] = [] + self._record_counts[measurement] = 0 + self._tag_keys.pop(measurement, None) def _flush_measurement(self, measurement: str) -> None: """Flush a single measurement's buffer. Must hold lock.""" @@ -199,22 +255,41 @@ def _flush_measurement(self, measurement: str) -> None: # Keep ownership and data until the write is acknowledged. Holding # the lock across I/O serializes timer, manual, and close flushes. try: - self._client.write_columnar(measurement, merged) + tags = list(self._tag_keys.get(measurement, ())) + if tags: + self._client.write_columnar(measurement, merged, tag_columns=tags) + else: + self._client.write_columnar(measurement, merged) + except ArcPartialWriteError: + # Arc stored what it could; resending would duplicate it. + self._discard(measurement) + self._last_flush_time = time.monotonic() + raise except BaseException as error: self._flush_error = error raise - self._buffers[measurement] = [] - self._record_counts[measurement] = 0 + self._discard(measurement) self._last_flush_time = time.monotonic() def _flush_all(self) -> None: - """Flush all measurements. Must hold lock.""" - self._raise_if_failed() + """Flush all measurements. Must hold lock. + + A partial write does not stop the other measurements from flushing; the + first one is raised after they have been sent. + """ + if self._flush_error is not None: + raise self._flush_error + partial: Optional[ArcPartialWriteError] = None measurements = list(self._buffers.keys()) for measurement in measurements: if self._buffers[measurement]: - self._flush_measurement(measurement) + try: + self._flush_measurement(measurement) + except ArcPartialWriteError as error: + partial = partial or error + if partial is not None: + raise partial def _merge_columnar(self, batches: list[dict[str, list[Any]]]) -> dict[str, list[Any]]: """Merge multiple columnar batches into one. @@ -261,8 +336,14 @@ def close(self) -> None: self._flush_thread = None with self._lock: - self._flush_all() + try: + self._flush_all() + except ArcPartialWriteError: + # Everything was flushed or discarded; the writer is done. + self._closed = True + raise self._closed = True + self._raise_pending_partial() def __enter__(self) -> BufferedWriter: """Enter context manager.""" diff --git a/src/arc_client/ingestion/msgpack.py b/src/arc_client/ingestion/msgpack.py index 900ae4c..bb678f3 100644 --- a/src/arc_client/ingestion/msgpack.py +++ b/src/arc_client/ingestion/msgpack.py @@ -11,7 +11,8 @@ "time": [1633024800000000, 1633024801000000], # microseconds "host": ["server01", "server01"], "usage": [45.2, 47.8], - } + }, + "tag_keys": ["host"], # optional: which columns are tags } Row Format (LEGACY): @@ -37,6 +38,7 @@ def encode_columnar( measurement: str, columns: dict[str, list[Any]], time_unit: str = "us", + tag_keys: Optional[list[str]] = None, ) -> bytes: """Encode columnar data to MessagePack format. @@ -51,12 +53,17 @@ def encode_columnar( time_unit: Unit of timestamps in the 'time' column. Options: "s" (seconds), "ms" (milliseconds), "us" (microseconds). Default is "us" (microseconds). + tag_keys: Columns Arc should store as tags rather than fields, sent as + the payload's ``tag_keys`` (Arc 2027.01, internal/ingest/ + msgpack_typed.go; older servers ignore the key). Each must name a + column other than ``time``. Omitted from the payload when empty. Returns: MessagePack encoded bytes ready for Arc ingestion. Raises: - ArcValidationError: If columns are invalid or missing required 'time'. + ArcValidationError: If columns are invalid or missing required 'time', + or a tag key does not name a non-time column. Example: >>> data = encode_columnar( @@ -91,15 +98,44 @@ def encode_columnar( # Normalize timestamps to microseconds if needed columns = _normalize_timestamps(columns, time_unit) - payload = { + payload: dict[str, Any] = { "m": measurement, "columns": columns, } + tags = validate_tag_keys(tag_keys, columns) + if tags: + payload["tag_keys"] = tags result: bytes = msgpack.packb(payload, use_bin_type=True) return result +def validate_tag_keys( + tag_keys: Optional[list[str]], columns: Any, time_column: str = "time" +) -> list[str]: + """Validate tag column names against a column collection. + + Returns the tag keys de-duplicated in their given order. Raises + ArcValidationError for a non-string name, the time column, or a name that + is not one of ``columns``. + """ + if not tag_keys: + return [] + if isinstance(tag_keys, str): + raise ArcValidationError("tag columns must be a list of column names, not a string") + seen: list[str] = [] + for key in tag_keys: + if not isinstance(key, str) or not key: + raise ArcValidationError(f"Invalid tag column {key!r}: must be a column name") + if key == time_column or key == "time": + raise ArcValidationError(f"The time column {key!r} cannot be a tag") + if key not in columns: + raise ArcValidationError(f"Tag column {key!r} is not one of the columns") + if key not in seen: + seen.append(key) + return seen + + def encode_records(records: list[dict[str, Any]]) -> bytes: """Encode multiple row-format records to MessagePack. @@ -249,20 +285,27 @@ def dataframe_to_columnar( df: DataFrame to convert (pandas, polars, or pyarrow). measurement: Measurement name (not used in output, just for validation). time_column: Name of the timestamp column. - tag_columns: Accepted for API stability but currently a no-op. The - columnar MessagePack write format has no tag/field distinction on the - wire (every non-time column is sent as a plain column); Arc derives - tag columns server-side from the data. Kept so callers can pass it - without breakage and so a future wire format can honor it. + tag_columns: Columns that are tags (dimensions) rather than fields. + Validated here (each must be a column of ``df`` other than the time + column); the caller sends them as the payload's ``tag_keys``, e.g. + ``encode_columnar(..., tag_keys=tag_columns)``, which is what + ``write_dataframe`` does. Without it Arc stores every non-time + column as a field. Returns: - Dictionary of column name to list of values. + Dictionary of column name to list of values. The tag/field split is not + part of this dict; it travels separately as ``tag_keys``. Raises: - ArcValidationError: If DataFrame is invalid or time column is missing. + ArcValidationError: If DataFrame is invalid, the time column is missing, + or a tag column is not a non-time column of the DataFrame. """ if tag_columns is None: tag_columns = [] + column_names = list(df.column_names) if hasattr(df, "column_names") else list(df.columns) + if tag_columns and time_column not in column_names: + raise ArcValidationError(f"Time column '{time_column}' not found in DataFrame") + validate_tag_keys(tag_columns, column_names, time_column) # Detect DataFrame type and convert df_type = type(df).__module__ diff --git a/src/arc_client/ingestion/write_support.py b/src/arc_client/ingestion/write_support.py new file mode 100644 index 0000000..578a772 --- /dev/null +++ b/src/arc_client/ingestion/write_support.py @@ -0,0 +1,92 @@ +"""Helpers shared by the sync and async write clients.""" + +from __future__ import annotations + +import re +from typing import Any, Optional + +from arc_client.exceptions import ( + ArcConnectionError, + ArcError, + ArcIngestionError, + ArcPartialWriteError, + ArcServerError, + ArcValidationError, +) + +# Timestamp precisions Arc's line-protocol handler accepts; anything else is a +# 400 (internal/api/lineprotocol.go handleWrite). Arc's default is "ns". +LP_PRECISIONS = ("ns", "us", "ms", "s") + +# Arc's answer when it dropped some points and (maybe) wrote the rest +# (internal/api/partial_write.go partialWriteMessage): +# "partial write: N points dropped: [ the other points were written]" +_PARTIAL_WRITE_RE = re.compile(r"^\s*partial write:\s*(\d+)\s+points? dropped", re.IGNORECASE) +_OTHERS_WRITTEN = "the other points were written" + + +def validate_precision(precision: Optional[str]) -> Optional[str]: + """Return ``precision`` if Arc accepts it, raising ArcValidationError otherwise.""" + if precision is None: + return None + if precision not in LP_PRECISIONS: + raise ArcValidationError( + f"Invalid precision {precision!r}: must be one of {', '.join(LP_PRECISIONS)}" + ) + return precision + + +def partial_write_error(error: ArcServerError) -> Optional[ArcPartialWriteError]: + """Recognize Arc's ``400 "partial write"`` answer, else return None.""" + if error.status_code != 400: + return None + body = error.response_body or {} + message = body.get("error") if isinstance(body.get("error"), str) else None + if message is None: + return None + match = _PARTIAL_WRITE_RE.match(message) + if match is None: + return None + others_written = _OTHERS_WRITTEN in message.lower() + return ArcPartialWriteError( + f"Arc dropped {match.group(1)} point(s) of this write" + + (" and wrote the others" if others_written else "; nothing was written") + + f": {message}", + dropped=int(match.group(1)), + others_written=others_written, + ) + + +def as_write_error(error: Exception) -> Exception: + """Map an exception from a write request to the one the caller should see. + + * ArcIngestionError (including ArcIngestRefusedError) passes unchanged. + * A ``400 "partial write"`` becomes ArcPartialWriteError. + * Other generic HTTP failures (ArcServerError) and transport failures + (ArcConnectionError) become ArcIngestionError, as before, with the + original as ``__cause__`` and the HTTP status on ``status_code``. + * Every other typed ArcError (authentication, permission, scoped access, + license, rate limit, not found, not supported) passes unchanged, so + callers can tell "bad token" or "slow down" from "the data was rejected". + * Anything else becomes ArcIngestionError. + """ + if isinstance(error, ArcIngestionError): + return error + if isinstance(error, ArcServerError): + partial = partial_write_error(error) + if partial is not None: + return partial + return ArcIngestionError(f"Failed to write data: {error}", status_code=error.status_code) + if isinstance(error, ArcConnectionError): + return ArcIngestionError(f"Failed to write data: {error}") + if isinstance(error, ArcError): + return error + return ArcIngestionError(f"Failed to write data: {error}") + + +def tle_stats_body(body: Any) -> dict[str, Any]: + """Unwrap Arc's TLE stats envelope.""" + if not isinstance(body, dict): + return {} + stats = body.get("stats", body) + return stats if isinstance(stats, dict) else {} diff --git a/src/arc_client/ingestion/writer.py b/src/arc_client/ingestion/writer.py index 8b99f5a..bc0e653 100644 --- a/src/arc_client/ingestion/writer.py +++ b/src/arc_client/ingestion/writer.py @@ -6,7 +6,8 @@ from arc_client.compression import Compression from arc_client.config import ClientConfig -from arc_client.exceptions import ArcIngestionError, ArcValidationError +from arc_client.exceptions import ArcError, ArcIngestionError, ArcValidationError +from arc_client.http.base import is_retryable_write_refusal from arc_client.http.sync_http import SyncHTTPClient from arc_client.ingestion.compression import compress as compress_data from arc_client.ingestion.line_protocol import format_line_protocol @@ -15,6 +16,11 @@ encode_columnar, encode_records, ) +from arc_client.ingestion.write_support import ( + as_write_error, + tle_stats_body, + validate_precision, +) # Compression argument accepted by write methods: a codec, its name, a bool # (back-compat), or None to use the client default. @@ -60,6 +66,7 @@ def write_columnar( database: Optional[str] = None, compress: CompressArg = None, time_unit: str = "us", + tag_columns: Optional[list[str]] = None, ) -> None: """Write columnar data to Arc (highest performance method). @@ -74,6 +81,11 @@ def write_columnar( database: Target database. Uses client default if not specified. compress: Whether to gzip compress. Uses client default if not specified. time_unit: Unit of timestamps - "s", "ms", or "us" (default). + tag_columns: Columns Arc should store as tags (indexed dimensions) + rather than fields. Sent as the payload's ``tag_keys``, which + Arc records in the Parquet ``arc:tags`` metadata (used for + compaction dedup and tag-aware queries). Each must be a column + other than ``time``. Arc releases before 2027.01 ignore it. Raises: ArcIngestionError: If the write fails. @@ -91,7 +103,7 @@ def write_columnar( """ # Encode to MessagePack try: - data = encode_columnar(measurement, columns, time_unit) + data = encode_columnar(measurement, columns, time_unit, tag_keys=tag_columns) except ArcValidationError: raise except Exception as e: @@ -151,7 +163,9 @@ def write_dataframe( measurement: The measurement (table) name. database: Target database. Uses client default if not specified. time_column: Name of the timestamp column. Default "time". - tag_columns: Columns to treat as tags (dimensions). + tag_columns: Columns to store as tags (dimensions) rather than + fields, sent as ``tag_keys``. Must name existing columns other + than the time column. compress: Whether to gzip compress. Uses client default if not specified. Raises: @@ -174,13 +188,14 @@ def write_dataframe( except Exception as e: raise ArcValidationError(f"Failed to convert DataFrame: {e}") from e - self.write_columnar(measurement, columns, database, compress) + self.write_columnar(measurement, columns, database, compress, tag_columns=tag_columns) def write_line_protocol( self, lines: str | list[str], database: Optional[str] = None, compress: CompressArg = None, + precision: Optional[str] = None, ) -> None: """Write data using InfluxDB Line Protocol format. @@ -191,9 +206,14 @@ def write_line_protocol( lines: Line Protocol string or list of lines. database: Target database. Uses client default if not specified. compress: Whether to gzip compress. Uses client default if not specified. + precision: Unit of the timestamps in ``lines``: ``"ns"``, ``"us"``, + ``"ms"`` or ``"s"``, sent as Arc's ``?precision=``. ``None`` + (default) sends nothing and Arc assumes nanoseconds. Raises: ArcIngestionError: If the write fails. + ArcPartialWriteError: If Arc dropped some points (never retried). + ArcValidationError: If ``precision`` is not one Arc accepts. Example: >>> client.write.write_line_protocol( @@ -201,7 +221,9 @@ def write_line_protocol( ... ) """ data = "\n".join(lines) if isinstance(lines, list) else lines - self._write_line_protocol(data.encode("utf-8"), database, compress) + self._write_line_protocol( + data.encode("utf-8"), database, compress, validate_precision(precision) + ) def write_point( self, @@ -211,6 +233,7 @@ def write_point( timestamp: Optional[int] = None, database: Optional[str] = None, compress: CompressArg = None, + precision: Optional[str] = None, ) -> None: """Write a single data point using Line Protocol. @@ -220,12 +243,21 @@ def write_point( measurement: The measurement name. fields: Dictionary of field values. tags: Optional dictionary of tag values. - timestamp: Optional timestamp in microseconds. + timestamp: Optional timestamp, in microseconds unless ``precision`` + is given. database: Target database. Uses client default if not specified. compress: Whether to gzip compress. Uses client default if not specified. + precision: Unit of ``timestamp`` (``"ns"``, ``"us"``, ``"ms"`` or + ``"s"``). When given, the timestamp is sent unconverted with + Arc's ``?precision=``. ``None`` (default) keeps the historical + behavior: microseconds, converted to nanoseconds on the wire. """ - line = format_line_protocol(measurement, fields, tags, timestamp) - self._write_line_protocol(line.encode("utf-8"), database, compress) + precision = validate_precision(precision) + # With an explicit precision the timestamp goes out as-is ("ns" means + # "no conversion") and Arc interprets it in that unit. + time_unit = "us" if precision is None else "ns" + line = format_line_protocol(measurement, fields, tags, timestamp, time_unit) + self._write_line_protocol(line.encode("utf-8"), database, compress, precision) def write_tle( self, @@ -265,9 +297,11 @@ def tle_stats(self) -> dict[str, Any]: try: response = self._http.get("/api/v1/write/tle/stats") body = response.json() + except ArcError: + raise except Exception as e: raise ArcIngestionError(f"Failed to get TLE stats: {e}") from e - return body.get("stats", body) if isinstance(body, dict) else {} + return tle_stats_body(body) def buffered( self, @@ -317,11 +351,19 @@ def _post_write( database: Optional[str], compress: CompressArg, extra_headers: Optional[dict[str, str]] = None, + params: Optional[dict[str, Any]] = None, ) -> None: """Compress (if requested) and POST a write payload. Arc auto-detects gzip/zstd from the payload's magic bytes, so no ``Content-Encoding`` header is sent. + + A 503 Arc marks as an ingest refusal that applied nothing + (``X-Arc-Ingest-Fenced``, or ``X-Arc-Ingest-Refused`` with any reason + but ``wal_not_durable``) is resent up to ``max_retries`` times, + honoring ``Retry-After``. Nothing else is resent: a partial write, a + ``wal_not_durable`` refusal or any other failure may have stored + data. Errors are mapped by ``as_write_error``. """ codec = self._resolve_codec(compress) data = compress_data(data, codec) @@ -333,16 +375,24 @@ def _post_write( headers["x-arc-database"] = database try: - response = self._http.post(path, content=data, headers=headers) - # Arc returns 204 No Content on success - if response.status_code not in (200, 204): - raise ArcIngestionError( - f"Write failed with status {response.status_code}: {response.text}" - ) - except ArcIngestionError: - raise + response = self._http.post( + path, + content=data, + headers=headers, + params=params, + retry_if=is_retryable_write_refusal, + ) except Exception as e: - raise ArcIngestionError(f"Failed to write data: {e}") from e + mapped = as_write_error(e) + if mapped is e: + raise + raise mapped from e + # Arc returns 204 No Content on success + if response.status_code not in (200, 204): + raise ArcIngestionError( + f"Write failed with status {response.status_code}: {response.text}", + status_code=response.status_code, + ) def _write_msgpack( self, @@ -358,6 +408,14 @@ def _write_line_protocol( data: bytes, database: Optional[str], compress: CompressArg, + precision: Optional[str] = None, ) -> None: """Send Line Protocol data to Arc.""" - self._post_write("/api/v1/write/line-protocol", data, "text/plain", database, compress) + self._post_write( + "/api/v1/write/line-protocol", + data, + "text/plain", + database, + compress, + params={"precision": precision} if precision else None, + ) diff --git a/tests/unit/http/test_write_parity.py b/tests/unit/http/test_write_parity.py new file mode 100644 index 0000000..bf68a2b --- /dev/null +++ b/tests/unit/http/test_write_parity.py @@ -0,0 +1,325 @@ +"""Write endpoint contracts against Arc main (2027.01). + +Refusals (internal/api/routing.go respondIngestRefused/respondIngestFenced), +partial writes (internal/api/partial_write.go), line-protocol precision +(internal/api/lineprotocol.go) and msgpack tag_keys (internal/ingest/ +msgpack_typed.go). +""" + +from __future__ import annotations + +from typing import Any + +import msgpack +import pyarrow as pa +import pytest +from pytest_httpx import HTTPXMock + +from arc_client import ArcIngestRefusedError, ArcPartialWriteError +from arc_client.compression import Compression +from arc_client.exceptions import ( + ArcAuthenticationError, + ArcIngestionError, + ArcPermissionError, + ArcRateLimitError, + ArcValidationError, +) + +from ._parity import MODES, call, make_client + +LP = "/api/v1/write/line-protocol" +MSGPACK = "/api/v1/write/msgpack" + +PARTIAL_REST = ( + 'partial write: 3 points dropped: 3 in "cpu" column "v": value type does not ' + "match the column; the other points were written" +) +PARTIAL_ALL = 'partial write: 2 points dropped: 2 in "cpu": field named time;' + + +def client_kwargs(**extra: Any) -> dict[str, Any]: + return {"compression": Compression.NONE, **extra} + + +def write_lp(client: Any) -> Any: + return client.write.write_line_protocol("cpu,host=a v=1i") + + +def write_columnar(client: Any) -> Any: + return client.write.write_columnar("cpu", {"time": [1], "v": [1.0]}) + + +WRITES = [pytest.param(write_lp, id="lp"), pytest.param(write_columnar, id="msgpack")] + + +# --- Bug 4: ingest refusals -------------------------------------------------- + + +@MODES +@pytest.mark.parametrize("write", WRITES) +@pytest.mark.parametrize( + "headers", + [ + pytest.param({"X-Arc-Ingest-Refused": "wal_disk"}, id="wal_disk"), + pytest.param({"X-Arc-Ingest-Fenced": "1"}, id="fenced"), + ], +) +async def test_safe_refusal_is_resent( + httpx_mock: HTTPXMock, async_mode: bool, write: Any, headers: dict[str, str] +) -> None: + httpx_mock.add_response( + status_code=503, + headers={**headers, "Retry-After": "0"}, + json={"error": "Write rejected (retry): nothing was written"}, + ) + httpx_mock.add_response(status_code=204) + async with make_client(async_mode, max_retries=2, **client_kwargs()) as client: + await call(write(client)) + requests = httpx_mock.get_requests() + assert len(requests) == 2 + assert requests[0].content == requests[1].content + + +@MODES +async def test_safe_refusal_exhausted_reports_reason( + httpx_mock: HTTPXMock, async_mode: bool +) -> None: + httpx_mock.add_response( + status_code=503, + headers={"X-Arc-Ingest-Refused": "wal_disk", "Retry-After": "0"}, + json={"error": "Write rejected (retry): WAL volume disk pressure"}, + is_reusable=True, + ) + async with make_client(async_mode, max_retries=1, **client_kwargs()) as client: + with pytest.raises(ArcIngestRefusedError) as info: + await call(write_lp(client)) + assert info.value.reason == "wal_disk" + assert info.value.retry_safe is True + assert info.value.status_code == 503 + assert isinstance(info.value, ArcIngestionError) + assert len(httpx_mock.get_requests()) == 2 + + +@MODES +@pytest.mark.parametrize("write", WRITES) +async def test_wal_not_durable_is_never_resent( + httpx_mock: HTTPXMock, async_mode: bool, write: Any +) -> None: + httpx_mock.add_response( + status_code=503, + headers={"X-Arc-Ingest-Refused": "wal_not_durable", "Retry-After": "5"}, + json={"error": "write refused: earlier parts of this request may already be stored"}, + ) + async with make_client(async_mode, max_retries=3, **client_kwargs()) as client: + with pytest.raises(ArcIngestRefusedError) as info: + await call(write(client)) + assert info.value.reason == "wal_not_durable" + assert info.value.retry_safe is False + assert len(httpx_mock.get_requests()) == 1 + + +@MODES +async def test_plain_write_503_is_not_resent(httpx_mock: HTTPXMock, async_mode: bool) -> None: + # Schema churn / shutdown 503s carry no refusal header and may follow a + # partially applied request: not resent. + httpx_mock.add_response(status_code=503, json={"error": "Write rejected (schema churn)"}) + async with make_client(async_mode, max_retries=3, **client_kwargs()) as client: + with pytest.raises(ArcIngestionError) as info: + await call(write_lp(client)) + assert not isinstance(info.value, ArcIngestRefusedError) + assert info.value.status_code == 503 + assert len(httpx_mock.get_requests()) == 1 + + +# --- Item 8: partial writes --------------------------------------------------- + + +@MODES +@pytest.mark.parametrize("write", WRITES) +@pytest.mark.parametrize( + ("message", "dropped", "others"), + [ + pytest.param(PARTIAL_REST, 3, True, id="others-written"), + pytest.param(PARTIAL_ALL, 2, False, id="all-dropped"), + ], +) +async def test_partial_write_error( + httpx_mock: HTTPXMock, + async_mode: bool, + write: Any, + message: str, + dropped: int, + others: bool, +) -> None: + httpx_mock.add_response(status_code=400, json={"error": message}) + async with make_client(async_mode, max_retries=3, **client_kwargs()) as client: + with pytest.raises(ArcPartialWriteError) as info: + await call(write(client)) + assert info.value.dropped == dropped + assert info.value.others_written is others + assert info.value.status_code == 400 + assert info.value.response_body == {"error": message} + assert len(httpx_mock.get_requests()) == 1 + + +@MODES +async def test_other_400_is_plain_ingestion_error(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(status_code=400, json={"error": "No valid records in request"}) + async with make_client(async_mode, **client_kwargs()) as client: + with pytest.raises(ArcIngestionError) as info: + await call(write_lp(client)) + assert not isinstance(info.value, ArcPartialWriteError) + assert info.value.status_code == 400 + + +# --- Bug 2 (writes): typed errors surface as themselves --------------------- + + +@MODES +@pytest.mark.parametrize( + ("status", "message", "expected"), + [ + (401, "invalid token", ArcAuthenticationError), + (403, "token does not have 'write' permission", ArcPermissionError), + (429, "slow down", ArcRateLimitError), + ], +) +async def test_write_typed_errors_surface_unchanged( + httpx_mock: HTTPXMock, + async_mode: bool, + status: int, + message: str, + expected: type[Exception], +) -> None: + httpx_mock.add_response(status_code=status, json={"error": message}) + async with make_client(async_mode, max_retries=3, **client_kwargs()) as client: + with pytest.raises(expected) as info: + await call(write_lp(client)) + assert not isinstance(info.value, ArcIngestionError) + assert len(httpx_mock.get_requests()) == 1 + + +# --- Item 6: line-protocol precision ---------------------------------------- + + +@MODES +async def test_line_protocol_precision_param(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(status_code=204) + async with make_client(async_mode, **client_kwargs()) as client: + await call(client.write.write_line_protocol("cpu v=1i 1700000000", precision="s")) + request = httpx_mock.get_request() + assert request.url.path == LP + assert dict(request.url.params) == {"precision": "s"} + assert request.content == b"cpu v=1i 1700000000" + + +@MODES +async def test_line_protocol_default_sends_no_precision( + httpx_mock: HTTPXMock, async_mode: bool +) -> None: + httpx_mock.add_response(status_code=204) + async with make_client(async_mode, **client_kwargs()) as client: + await call(client.write.write_line_protocol("cpu v=1i")) + assert "precision" not in httpx_mock.get_request().url.params + + +@MODES +async def test_write_point_precision_sends_timestamp_unconverted( + httpx_mock: HTTPXMock, async_mode: bool +) -> None: + httpx_mock.add_response(status_code=204) + async with make_client(async_mode, **client_kwargs()) as client: + await call(client.write.write_point("cpu", {"v": 1}, timestamp=1700000000, precision="s")) + request = httpx_mock.get_request() + assert dict(request.url.params) == {"precision": "s"} + assert request.content == b"cpu v=1i 1700000000" + + +@MODES +async def test_write_point_default_keeps_microseconds( + httpx_mock: HTTPXMock, async_mode: bool +) -> None: + httpx_mock.add_response(status_code=204) + async with make_client(async_mode, **client_kwargs()) as client: + await call(client.write.write_point("cpu", {"v": 1}, timestamp=1_000_000)) + request = httpx_mock.get_request() + assert "precision" not in request.url.params + assert request.content == b"cpu v=1i 1000000000" + + +@MODES +@pytest.mark.parametrize("precision", ["n", "u", "NS", "h", ""]) +async def test_invalid_precision_rejected(async_mode: bool, precision: str) -> None: + async with make_client(async_mode, **client_kwargs()) as client: + with pytest.raises(ArcValidationError): + await call(client.write.write_line_protocol("cpu v=1i", precision=precision)) + + +# --- Item 7: msgpack tag_keys ------------------------------------------------- + + +def sent_payload(httpx_mock: HTTPXMock) -> dict[str, Any]: + request = httpx_mock.get_request() + assert request.url.path == MSGPACK + payload: dict[str, Any] = msgpack.unpackb(request.content, raw=False) + return payload + + +@MODES +async def test_write_columnar_sends_tag_keys(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(status_code=204) + columns = {"time": [1], "host": ["a"], "region": ["eu"], "v": [1.0]} + async with make_client(async_mode, **client_kwargs()) as client: + await call(client.write.write_columnar("cpu", columns, tag_columns=["host", "region"])) + payload = sent_payload(httpx_mock) + assert payload["tag_keys"] == ["host", "region"] + assert payload["columns"]["host"] == ["a"] + + +@MODES +async def test_write_columnar_without_tags_omits_key( + httpx_mock: HTTPXMock, async_mode: bool +) -> None: + httpx_mock.add_response(status_code=204) + async with make_client(async_mode, **client_kwargs()) as client: + await call(write_columnar(client)) + assert "tag_keys" not in sent_payload(httpx_mock) + + +@MODES +async def test_write_dataframe_sends_tag_keys(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(status_code=204) + table = pa.table({"ts": [1, 2], "host": ["a", "b"], "v": [1.0, 2.0]}) + async with make_client(async_mode, **client_kwargs()) as client: + await call( + client.write.write_dataframe(table, "cpu", time_column="ts", tag_columns=["host"]) + ) + payload = sent_payload(httpx_mock) + assert payload["tag_keys"] == ["host"] + assert payload["columns"]["time"] == [1, 2] + + +@MODES +@pytest.mark.parametrize("tags", [["time"], ["missing"], "host"]) +async def test_invalid_tag_columns_rejected(async_mode: bool, tags: Any) -> None: + columns = {"time": [1], "host": ["a"], "v": [1.0]} + async with make_client(async_mode, **client_kwargs()) as client: + with pytest.raises(ArcValidationError): + await call(client.write.write_columnar("cpu", columns, tag_columns=tags)) + + +@MODES +async def test_buffered_record_tags_become_tag_keys( + httpx_mock: HTTPXMock, async_mode: bool +) -> None: + httpx_mock.add_response(status_code=204) + async with make_client(async_mode, **client_kwargs()) as client: + buffer = client.write.buffered(batch_size=100, flush_interval=3600) + await call( + buffer.write({"measurement": "cpu", "fields": {"v": 1.0}, "tags": {"host": "a"}}) + ) + await call(buffer.write({"measurement": "cpu", "fields": {"v": 2.0}})) + await call(buffer.close()) + payload = sent_payload(httpx_mock) + assert payload["tag_keys"] == ["host"] + assert payload["columns"]["host"] == ["a", None] diff --git a/tests/unit/test_buffered_partial_write.py b/tests/unit/test_buffered_partial_write.py new file mode 100644 index 0000000..dfd2f7e --- /dev/null +++ b/tests/unit/test_buffered_partial_write.py @@ -0,0 +1,169 @@ +"""A partial write is applied server-side: buffered writers discard it and report it once.""" + +from __future__ import annotations + +import asyncio +import threading +from typing import Any, Optional + +import pytest + +from arc_client import ArcPartialWriteError +from arc_client.exceptions import ArcServerError +from arc_client.ingestion.async_buffered import AsyncBufferedWriter +from arc_client.ingestion.buffered import BufferedWriter + + +def partial() -> ArcPartialWriteError: + return ArcPartialWriteError("partial write: 1 points dropped", dropped=1, others_written=True) + + +class FakeWriter: + """Records every write; fails the writes listed in ``fail`` (by call index).""" + + def __init__(self, fail: Optional[dict[int, BaseException]] = None) -> None: + self.calls: list[tuple[str, dict[str, list[Any]], Optional[list[str]]]] = [] + self.fail = fail or {} + self.sent = threading.Event() + + def _record(self, measurement: str, columns: Any, tag_columns: Any) -> None: + index = len(self.calls) + self.calls.append((measurement, columns, tag_columns)) + self.sent.set() + if index in self.fail: + raise self.fail[index] + + def write_columnar( + self, measurement: str, columns: Any, tag_columns: Optional[list[str]] = None + ) -> None: + self._record(measurement, columns, tag_columns) + + +class AsyncFakeWriter(FakeWriter): + async def write_columnar( # type: ignore[override] + self, measurement: str, columns: Any, tag_columns: Optional[list[str]] = None + ) -> None: + self._record(measurement, columns, tag_columns) + + +ROW = {"measurement": "cpu", "fields": {"v": 1.0}, "timestamp": 1} + + +def test_sync_partial_write_discards_batch_and_raises_once() -> None: + fake = FakeWriter(fail={0: partial()}) + buffer = BufferedWriter(fake, batch_size=100, flush_interval=3600) # type: ignore[arg-type] + buffer.write(ROW) + + with pytest.raises(ArcPartialWriteError): + buffer.flush() + assert buffer.pending_count == 0 + + # Not terminal and nothing resent: the next flush sends only new data. + buffer.flush() + buffer.write({**ROW, "timestamp": 2}) + buffer.close() + assert [c[1]["time"] for c in fake.calls] == [[1], [2]] + + +def test_sync_partial_write_does_not_block_other_measurements_on_close() -> None: + fake = FakeWriter(fail={0: partial()}) + buffer = BufferedWriter(fake, batch_size=100, flush_interval=3600) # type: ignore[arg-type] + buffer.write(ROW) + buffer.write({**ROW, "measurement": "mem"}) + + with pytest.raises(ArcPartialWriteError): + buffer.close() + assert sorted(c[0] for c in fake.calls) == ["cpu", "mem"] + assert buffer.pending_count == 0 + with pytest.raises(RuntimeError, match="closed"): + buffer.write(ROW) + + +def test_sync_other_failure_stays_terminal() -> None: + error = ArcServerError("boom", status_code=500) + fake = FakeWriter(fail={0: error}) + buffer = BufferedWriter(fake, batch_size=100, flush_interval=3600) # type: ignore[arg-type] + buffer.write(ROW) + for _ in range(2): + with pytest.raises(ArcServerError): + buffer.flush() + assert buffer.pending_count == 1 + assert len(fake.calls) == 1 + + +def test_sync_background_partial_is_reported_once_by_next_call() -> None: + fake = FakeWriter(fail={0: partial()}) + buffer = BufferedWriter(fake, batch_size=100, flush_interval=0.05) # type: ignore[arg-type] + with buffer: + buffer._last_flush_time -= 1 # make the first row old enough for the timer + with buffer._lock: + buffer._buffers["cpu"].append({"time": [1], "v": [1.0]}) + buffer._record_counts["cpu"] += 1 + assert fake.sent.wait(5) + # Wait until the timer has recorded the partial write. + for _ in range(100): + with buffer._lock: + if buffer._partial_error is not None: + break + threading.Event().wait(0.01) + with pytest.raises(ArcPartialWriteError): + buffer.write({**ROW, "timestamp": 2}) + buffer.write({**ROW, "timestamp": 3}) # reported once only + assert [c[1]["time"] for c in fake.calls] == [[1], [3]] + + +def test_sync_tags_are_sent_and_reset_per_batch() -> None: + fake = FakeWriter() + buffer = BufferedWriter(fake, batch_size=100, flush_interval=3600) # type: ignore[arg-type] + buffer.write({**ROW, "tags": {"host": "a"}}) + buffer.write_columnar("cpu", {"time": [2], "region": ["eu"], "v": [2.0]}, ["region"]) + buffer.flush() + buffer.write(ROW) + buffer.close() + assert fake.calls[0][2] == ["host", "region"] + assert fake.calls[0][1]["host"] == ["a", None] + assert fake.calls[1][2] is None + + +async def test_async_partial_write_discards_batch_and_raises_once() -> None: + fake = AsyncFakeWriter(fail={0: partial()}) + buffer = AsyncBufferedWriter(fake, batch_size=100, flush_interval=3600) # type: ignore[arg-type] + await buffer.write(ROW) + + with pytest.raises(ArcPartialWriteError): + await buffer.flush() + assert buffer.pending_count == 0 + + await buffer.flush() + await buffer.write({**ROW, "timestamp": 2}) + await buffer.close() + assert [c[1]["time"] for c in fake.calls] == [[1], [2]] + + +async def test_async_partial_write_does_not_block_other_measurements_on_close() -> None: + fake = AsyncFakeWriter(fail={0: partial()}) + buffer = AsyncBufferedWriter(fake, batch_size=100, flush_interval=3600) # type: ignore[arg-type] + await buffer.write(ROW) + await buffer.write({**ROW, "measurement": "mem"}) + + with pytest.raises(ArcPartialWriteError): + await buffer.close() + assert sorted(c[0] for c in fake.calls) == ["cpu", "mem"] + assert buffer.pending_count == 0 + + +async def test_async_background_partial_is_reported_once_by_next_call() -> None: + fake = AsyncFakeWriter(fail={0: partial()}) + buffer = AsyncBufferedWriter(fake, batch_size=100, flush_interval=0.05) # type: ignore[arg-type] + async with buffer: + buffer._last_flush_time -= 1 + buffer._buffers["cpu"].append({"time": [1], "v": [1.0]}) + buffer._record_counts["cpu"] += 1 + for _ in range(200): + if buffer._partial_error is not None: + break + await asyncio.sleep(0.01) + with pytest.raises(ArcPartialWriteError): + await buffer.write({**ROW, "timestamp": 2}) + await buffer.write({**ROW, "timestamp": 3}) + assert [c[1]["time"] for c in fake.calls] == [[1], [3]] From eb0e5abdacdd76549e68e9a9998e5c45dd45b131 Mon Sep 17 00:00:00 2001 From: Ignacio Van Droogenbroeck <64545348+xe-nvdk@users.noreply.github.com> Date: Mon, 2 Nov 2026 09:06:25 -0600 Subject: [PATCH 14/20] fix(auth): send empty permission lists and keep typed errors create_token(permissions=[]) dropped the empty list, so Arc created a read,write token instead of an RBAC-only one. Optional token fields are now tested with 'is not None' (sync and async). The auth client also stops rewrapping rate-limit, server and connection errors as ArcAuthenticationError, which made them look like a bad token. --- src/arc_client/auth/async_manager.py | 27 ++++++++++++++++----------- src/arc_client/auth/manager.py | 27 ++++++++++++++++----------- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/arc_client/auth/async_manager.py b/src/arc_client/auth/async_manager.py index 87958f8..45e27aa 100644 --- a/src/arc_client/auth/async_manager.py +++ b/src/arc_client/auth/async_manager.py @@ -5,7 +5,7 @@ from typing import Any, List, Optional from arc_client.config import ClientConfig -from arc_client.exceptions import ArcAuthenticationError, ArcNotFoundError +from arc_client.exceptions import ArcAuthenticationError, ArcError, ArcNotFoundError from arc_client.http.async_http import AsyncHTTPClient from arc_client.models.auth import ( CreateTokenResponse, @@ -34,6 +34,8 @@ async def verify(self) -> VerifyResponse: return VerifyResponse(**data) except ArcAuthenticationError: return VerifyResponse(valid=False, error="Invalid or expired token") + except ArcError: + raise except Exception as e: raise ArcAuthenticationError(f"Token verification failed: {e}") from e @@ -46,18 +48,21 @@ async def create_token( ) -> CreateTokenResponse: """Create a new API token. Requires admin permissions.""" payload: dict[str, Any] = {"name": name} - if description: + if description is not None: payload["description"] = description - if permissions: + # An empty list is meaningful: Arc creates an RBAC-only token with no + # OSS permissions. Omitting the key makes Arc default to read,write + # (internal/api/auth_routes.go CreateTokenRequest). + if permissions is not None: payload["permissions"] = permissions - if expires_in: + if expires_in is not None: payload["expires_in"] = expires_in try: response = await self._http.post("/api/v1/auth/tokens", json=payload) data = response.json() return CreateTokenResponse(**data) - except ArcAuthenticationError: + except ArcError: raise except Exception as e: raise ArcAuthenticationError(f"Failed to create token: {e}") from e @@ -71,7 +76,7 @@ async def list_tokens(self) -> List[TokenInfo]: if not result.success: raise ArcAuthenticationError(result.error or "Failed to list tokens") return result.tokens - except ArcAuthenticationError: + except ArcError: raise except Exception as e: raise ArcAuthenticationError(f"Failed to list tokens: {e}") from e @@ -87,7 +92,7 @@ async def get_token(self, token_id: int) -> TokenInfo: raise ArcNotFoundError(f"Token {token_id} not found") raise ArcAuthenticationError(error) return TokenInfo(**data["token"]) - except (ArcNotFoundError, ArcAuthenticationError): + except ArcError: raise except Exception as e: raise ArcAuthenticationError(f"Failed to get token: {e}") from e @@ -119,7 +124,7 @@ async def update_token( if "not found" in error.lower(): raise ArcNotFoundError(f"Token {token_id} not found") raise ArcAuthenticationError(error) - except (ArcNotFoundError, ArcAuthenticationError): + except ArcError: raise except Exception as e: raise ArcAuthenticationError(f"Failed to update token: {e}") from e @@ -134,7 +139,7 @@ async def delete_token(self, token_id: int) -> None: if "not found" in error.lower(): raise ArcNotFoundError(f"Token {token_id} not found") raise ArcAuthenticationError(error) - except (ArcNotFoundError, ArcAuthenticationError): + except ArcError: raise except Exception as e: raise ArcAuthenticationError(f"Failed to delete token: {e}") from e @@ -153,7 +158,7 @@ async def rotate_token(self, token_id: int) -> str: if not result.new_token: raise ArcAuthenticationError("No new token returned") return result.new_token - except (ArcNotFoundError, ArcAuthenticationError): + except ArcError: raise except Exception as e: raise ArcAuthenticationError(f"Failed to rotate token: {e}") from e @@ -168,7 +173,7 @@ async def revoke_token(self, token_id: int) -> None: if "not found" in error.lower(): raise ArcNotFoundError(f"Token {token_id} not found") raise ArcAuthenticationError(error) - except (ArcNotFoundError, ArcAuthenticationError): + except ArcError: raise except Exception as e: raise ArcAuthenticationError(f"Failed to revoke token: {e}") from e diff --git a/src/arc_client/auth/manager.py b/src/arc_client/auth/manager.py index 2c5861b..1edded4 100644 --- a/src/arc_client/auth/manager.py +++ b/src/arc_client/auth/manager.py @@ -5,7 +5,7 @@ from typing import Any, List, Optional from arc_client.config import ClientConfig -from arc_client.exceptions import ArcAuthenticationError, ArcNotFoundError +from arc_client.exceptions import ArcAuthenticationError, ArcError, ArcNotFoundError from arc_client.http.sync_http import SyncHTTPClient from arc_client.models.auth import ( CreateTokenResponse, @@ -46,6 +46,8 @@ def verify(self) -> VerifyResponse: return VerifyResponse(**data) except ArcAuthenticationError: return VerifyResponse(valid=False, error="Invalid or expired token") + except ArcError: + raise except Exception as e: raise ArcAuthenticationError(f"Token verification failed: {e}") from e @@ -58,18 +60,21 @@ def create_token( ) -> CreateTokenResponse: """Create a new API token. Requires admin permissions.""" payload: dict[str, Any] = {"name": name} - if description: + if description is not None: payload["description"] = description - if permissions: + # An empty list is meaningful: Arc creates an RBAC-only token with no + # OSS permissions. Omitting the key makes Arc default to read,write + # (internal/api/auth_routes.go CreateTokenRequest). + if permissions is not None: payload["permissions"] = permissions - if expires_in: + if expires_in is not None: payload["expires_in"] = expires_in try: response = self._http.post("/api/v1/auth/tokens", json=payload) data = response.json() return CreateTokenResponse(**data) - except ArcAuthenticationError: + except ArcError: raise except Exception as e: raise ArcAuthenticationError(f"Failed to create token: {e}") from e @@ -83,7 +88,7 @@ def list_tokens(self) -> List[TokenInfo]: if not result.success: raise ArcAuthenticationError(result.error or "Failed to list tokens") return result.tokens - except ArcAuthenticationError: + except ArcError: raise except Exception as e: raise ArcAuthenticationError(f"Failed to list tokens: {e}") from e @@ -99,7 +104,7 @@ def get_token(self, token_id: int) -> TokenInfo: raise ArcNotFoundError(f"Token {token_id} not found") raise ArcAuthenticationError(error) return TokenInfo(**data["token"]) - except (ArcNotFoundError, ArcAuthenticationError): + except ArcError: raise except Exception as e: raise ArcAuthenticationError(f"Failed to get token: {e}") from e @@ -131,7 +136,7 @@ def update_token( if "not found" in error.lower(): raise ArcNotFoundError(f"Token {token_id} not found") raise ArcAuthenticationError(error) - except (ArcNotFoundError, ArcAuthenticationError): + except ArcError: raise except Exception as e: raise ArcAuthenticationError(f"Failed to update token: {e}") from e @@ -146,7 +151,7 @@ def delete_token(self, token_id: int) -> None: if "not found" in error.lower(): raise ArcNotFoundError(f"Token {token_id} not found") raise ArcAuthenticationError(error) - except (ArcNotFoundError, ArcAuthenticationError): + except ArcError: raise except Exception as e: raise ArcAuthenticationError(f"Failed to delete token: {e}") from e @@ -165,7 +170,7 @@ def rotate_token(self, token_id: int) -> str: if not result.new_token: raise ArcAuthenticationError("No new token returned") return result.new_token - except (ArcNotFoundError, ArcAuthenticationError): + except ArcError: raise except Exception as e: raise ArcAuthenticationError(f"Failed to rotate token: {e}") from e @@ -180,7 +185,7 @@ def revoke_token(self, token_id: int) -> None: if "not found" in error.lower(): raise ArcNotFoundError(f"Token {token_id} not found") raise ArcAuthenticationError(error) - except (ArcNotFoundError, ArcAuthenticationError): + except ArcError: raise except Exception as e: raise ArcAuthenticationError(f"Failed to revoke token: {e}") from e From 2d9ee15c2ea4ae368cc3938d53200d22ff999df8 Mon Sep 17 00:00:00 2001 From: Ignacio Van Droogenbroeck <64545348+xe-nvdk@users.noreply.github.com> Date: Mon, 2 Nov 2026 09:06:25 -0600 Subject: [PATCH 15/20] feat(client): model /health, readiness reasons and measurement schema - HealthResponse models what Arc sends (time, uptime, uptime_sec, storage, license); version stays as an optional, always-None field. - readiness()/write_readiness() return ReadyResponse with status, reason and tier for /ready and /ready/write; ready() stays a bool and no longer retries a 503 'not ready' with backoff. - databases.get_measurement_schema() and rebuild_measurement_schema() cover the stable-schema registry routes (#16). --- src/arc_client/async_client.py | 59 ++++- src/arc_client/client.py | 59 ++++- src/arc_client/management/async_databases.py | 62 ++++- src/arc_client/management/databases.py | 60 ++++- src/arc_client/models/__init__.py | 23 +- src/arc_client/models/common.py | 80 +++++- src/arc_client/models/database.py | 34 +++ src/arc_client/validation.py | 20 ++ tests/unit/http/test_core_parity.py | 246 +++++++++++++++++++ 9 files changed, 627 insertions(+), 16 deletions(-) create mode 100644 tests/unit/http/test_core_parity.py diff --git a/src/arc_client/async_client.py b/src/arc_client/async_client.py index 969a421..4ee3676 100644 --- a/src/arc_client/async_client.py +++ b/src/arc_client/async_client.py @@ -6,8 +6,10 @@ from arc_client.compression import Compression from arc_client.config import ClientConfig +from arc_client.exceptions import ArcServerError from arc_client.http.async_http import AsyncHTTPClient -from arc_client.models.common import HealthResponse +from arc_client.http.base import never_retry_status +from arc_client.models.common import HealthResponse, ReadyResponse class AsyncArcClient: @@ -273,15 +275,57 @@ async def health(self) -> HealthResponse: async def ready(self) -> bool: """Check if server is ready to accept requests. + A single probe: a 503 "not ready" answer is not retried. Use + :meth:`readiness` for the reason. + Returns: - True if server is ready, False otherwise. + True if server is ready, False otherwise (including when it cannot + be reached). """ try: - response = await self._get_http().get("/ready") - return response.status_code == 200 + return (await self.readiness()).ready except Exception: return False + async def readiness(self) -> ReadyResponse: + """Ask ``/ready`` whether this node should receive traffic, and why not. + + Arc answers 503 while starting up, replaying its WAL or shutting down, + and (with ``server.storage_credentials_fail_ready``) while a storage + tier's credentials are expired or unavailable; ``reason`` and ``tier`` + say which. The 503 is returned as ``ready=False``, not raised, and not + retried. + + Raises: + ArcConnectionError: If the server cannot be reached. + """ + return await self._readiness("/ready") + + async def write_readiness(self) -> ReadyResponse: + """Ask ``/ready/write`` whether writes should be routed to this node. + + Strictly narrower than :meth:`readiness`: a reader, or a writer that is + not the primary in local-storage cluster mode, answers + ``status="not_write_target"``. + + Raises: + ArcConnectionError: If the server cannot be reached. + """ + return await self._readiness("/ready/write") + + async def _readiness(self, path: str) -> ReadyResponse: + try: + response = await self._get_http().get(path, retry_if=never_retry_status) + except ArcServerError as error: + if error.status_code != 503: + raise + return _ready_response(False, error.response_body) + try: + body = response.json() + except ValueError: + body = None + return _ready_response(True, body) + async def close(self) -> None: """Close the client and release resources.""" if self._http is not None: @@ -298,3 +342,10 @@ async def __aexit__(self, *args: Any) -> None: def __repr__(self) -> str: return f"AsyncArcClient(host={self._config.host!r}, port={self._config.port})" + + +def _ready_response(ready: bool, body: object) -> ReadyResponse: + data = dict(body) if isinstance(body, dict) else {} + data["ready"] = ready + data.setdefault("message", data.get("reason")) + return ReadyResponse.model_validate(data) diff --git a/src/arc_client/client.py b/src/arc_client/client.py index 9bc623f..d446f7b 100644 --- a/src/arc_client/client.py +++ b/src/arc_client/client.py @@ -6,8 +6,10 @@ from arc_client.compression import Compression from arc_client.config import ClientConfig +from arc_client.exceptions import ArcServerError +from arc_client.http.base import never_retry_status from arc_client.http.sync_http import SyncHTTPClient -from arc_client.models.common import HealthResponse +from arc_client.models.common import HealthResponse, ReadyResponse class ArcClient: @@ -271,15 +273,57 @@ def health(self) -> HealthResponse: def ready(self) -> bool: """Check if server is ready to accept requests. + A single probe: a 503 "not ready" answer is not retried. Use + :meth:`readiness` for the reason. + Returns: - True if server is ready, False otherwise. + True if server is ready, False otherwise (including when it cannot + be reached). """ try: - response = self._get_http().get("/ready") - return response.status_code == 200 + return (self.readiness()).ready except Exception: return False + def readiness(self) -> ReadyResponse: + """Ask ``/ready`` whether this node should receive traffic, and why not. + + Arc answers 503 while starting up, replaying its WAL or shutting down, + and (with ``server.storage_credentials_fail_ready``) while a storage + tier's credentials are expired or unavailable; ``reason`` and ``tier`` + say which. The 503 is returned as ``ready=False``, not raised, and not + retried. + + Raises: + ArcConnectionError: If the server cannot be reached. + """ + return self._readiness("/ready") + + def write_readiness(self) -> ReadyResponse: + """Ask ``/ready/write`` whether writes should be routed to this node. + + Strictly narrower than :meth:`readiness`: a reader, or a writer that is + not the primary in local-storage cluster mode, answers + ``status="not_write_target"``. + + Raises: + ArcConnectionError: If the server cannot be reached. + """ + return self._readiness("/ready/write") + + def _readiness(self, path: str) -> ReadyResponse: + try: + response = self._get_http().get(path, retry_if=never_retry_status) + except ArcServerError as error: + if error.status_code != 503: + raise + return _ready_response(False, error.response_body) + try: + body = response.json() + except ValueError: + body = None + return _ready_response(True, body) + def close(self) -> None: """Close the client and release resources.""" if self._http is not None: @@ -296,3 +340,10 @@ def __exit__(self, *args: Any) -> None: def __repr__(self) -> str: return f"ArcClient(host={self._config.host!r}, port={self._config.port})" + + +def _ready_response(ready: bool, body: object) -> ReadyResponse: + data = dict(body) if isinstance(body, dict) else {} + data["ready"] = ready + data.setdefault("message", data.get("reason")) + return ReadyResponse.model_validate(data) diff --git a/src/arc_client/management/async_databases.py b/src/arc_client/management/async_databases.py index c3c6bb0..173860f 100644 --- a/src/arc_client/management/async_databases.py +++ b/src/arc_client/management/async_databases.py @@ -11,8 +11,10 @@ DatabaseInfo, DatabaseListResponse, MeasurementListResponse, + MeasurementSchema, + SchemaRebuildResponse, ) -from arc_client.validation import validate_name +from arc_client.validation import path_segment, validate_name class AsyncDatabaseClient: @@ -88,6 +90,64 @@ async def list_measurements(self, name: str) -> MeasurementListResponse: except Exception as e: raise ArcError(f"Failed to list measurements for {name!r}: {e}") from e + async def get_measurement_schema(self, database: str, measurement: str) -> MeasurementSchema: + """Get a measurement's registered field schema. + + ``GET /api/v1/databases/:database/measurements/:measurement/schema`` + returns the columns and types every query binds for the measurement, + regardless of the time range it reads (Arc's ``query.stable_schema`` + registry, built by ingest or by :meth:`rebuild_measurement_schema`). + Needs read permission on the measurement. + + Raises: + ArcNotFoundError: The registry is disabled on the server, or no + schema is registered for this measurement yet. + ArcPermissionError: The token may not read the measurement. + ArcServerError: HTTP 503 when the registry could not be read. + ArcValidationError: A name cannot be a URL path segment. + """ + path = ( + f"/api/v1/databases/{path_segment(database, 'database')}" + f"/measurements/{path_segment(measurement, 'measurement')}/schema" + ) + try: + response = await self._http.get(path) + return MeasurementSchema.model_validate(response.json()) + except ArcError: + raise + except Exception as e: + raise ArcError(f"Failed to get schema for {database}.{measurement}: {e}") from e + + async def rebuild_measurement_schema( + self, database: str, measurement: str + ) -> SchemaRebuildResponse: + """Queue a rebuild of a measurement's registered field schema (admin). + + ``POST .../measurements/:measurement/schema/rebuild`` samples a bounded + set of the measurement's files and merges what it finds into the + registered schema (fields are added, never dropped). Arc answers 202 + once the rebuild is queued; it runs in the background. + + Raises: + ArcNotFoundError: The registry is disabled on the server. + ArcServerError: HTTP 409 when a rebuild for this measurement is + already queued or running; HTTP 503 when the rebuild queue is + full or rebuilds are unavailable on this node. + ArcPermissionError: The token is not an admin token. + ArcValidationError: A name cannot be a URL path segment. + """ + path = ( + f"/api/v1/databases/{path_segment(database, 'database')}" + f"/measurements/{path_segment(measurement, 'measurement')}/schema/rebuild" + ) + try: + response = await self._http.post(path) + return SchemaRebuildResponse.model_validate(response.json()) + except ArcError: + raise + except Exception as e: + raise ArcError(f"Failed to rebuild schema for {database}.{measurement}: {e}") from e + async def create(self, name: str) -> DatabaseInfo: """Create a database (admin).""" validate_name(name, "database") diff --git a/src/arc_client/management/databases.py b/src/arc_client/management/databases.py index 262b138..9ba6cab 100644 --- a/src/arc_client/management/databases.py +++ b/src/arc_client/management/databases.py @@ -11,8 +11,10 @@ DatabaseInfo, DatabaseListResponse, MeasurementListResponse, + MeasurementSchema, + SchemaRebuildResponse, ) -from arc_client.validation import validate_name +from arc_client.validation import path_segment, validate_name class DatabaseClient: @@ -106,6 +108,62 @@ def list_measurements(self, name: str) -> MeasurementListResponse: except Exception as e: raise ArcError(f"Failed to list measurements for {name!r}: {e}") from e + def get_measurement_schema(self, database: str, measurement: str) -> MeasurementSchema: + """Get a measurement's registered field schema. + + ``GET /api/v1/databases/:database/measurements/:measurement/schema`` + returns the columns and types every query binds for the measurement, + regardless of the time range it reads (Arc's ``query.stable_schema`` + registry, built by ingest or by :meth:`rebuild_measurement_schema`). + Needs read permission on the measurement. + + Raises: + ArcNotFoundError: The registry is disabled on the server, or no + schema is registered for this measurement yet. + ArcPermissionError: The token may not read the measurement. + ArcServerError: HTTP 503 when the registry could not be read. + ArcValidationError: A name cannot be a URL path segment. + """ + path = ( + f"/api/v1/databases/{path_segment(database, 'database')}" + f"/measurements/{path_segment(measurement, 'measurement')}/schema" + ) + try: + response = self._http.get(path) + return MeasurementSchema.model_validate(response.json()) + except ArcError: + raise + except Exception as e: + raise ArcError(f"Failed to get schema for {database}.{measurement}: {e}") from e + + def rebuild_measurement_schema(self, database: str, measurement: str) -> SchemaRebuildResponse: + """Queue a rebuild of a measurement's registered field schema (admin). + + ``POST .../measurements/:measurement/schema/rebuild`` samples a bounded + set of the measurement's files and merges what it finds into the + registered schema (fields are added, never dropped). Arc answers 202 + once the rebuild is queued; it runs in the background. + + Raises: + ArcNotFoundError: The registry is disabled on the server. + ArcServerError: HTTP 409 when a rebuild for this measurement is + already queued or running; HTTP 503 when the rebuild queue is + full or rebuilds are unavailable on this node. + ArcPermissionError: The token is not an admin token. + ArcValidationError: A name cannot be a URL path segment. + """ + path = ( + f"/api/v1/databases/{path_segment(database, 'database')}" + f"/measurements/{path_segment(measurement, 'measurement')}/schema/rebuild" + ) + try: + response = self._http.post(path) + return SchemaRebuildResponse.model_validate(response.json()) + except ArcError: + raise + except Exception as e: + raise ArcError(f"Failed to rebuild schema for {database}.{measurement}: {e}") from e + def create(self, name: str) -> DatabaseInfo: """Create a database (admin). diff --git a/src/arc_client/models/__init__.py b/src/arc_client/models/__init__.py index bd10416..01d3d30 100644 --- a/src/arc_client/models/__init__.py +++ b/src/arc_client/models/__init__.py @@ -7,7 +7,12 @@ TokenListResponse, VerifyResponse, ) -from arc_client.models.common import HealthResponse, ReadyResponse +from arc_client.models.common import ( + HealthResponse, + LicenseHealth, + ReadyResponse, + StorageTierHealth, +) from arc_client.models.continuous_query import ( ContinuousQuery, CQExecution, @@ -18,6 +23,9 @@ DatabaseListResponse, DatabaseMeasurement, MeasurementListResponse, + MeasurementSchema, + SchemaField, + SchemaRebuildResponse, ) from arc_client.models.delete import DeleteConfigResponse, DeleteResponse from arc_client.models.import_ import ( @@ -26,7 +34,12 @@ LPImportResult, TLEImportResult, ) -from arc_client.models.query import EstimateResponse, MeasurementInfo, QueryResponse +from arc_client.models.query import ( + EstimateResponse, + MeasurementInfo, + QueryProfile, + QueryResponse, +) from arc_client.models.retention import ( ExecuteRetentionResponse, RetentionExecution, @@ -41,6 +54,9 @@ "DatabaseListResponse", "DatabaseMeasurement", "MeasurementListResponse", + "MeasurementSchema", + "SchemaField", + "SchemaRebuildResponse", "ImportResult", "ImportStats", "LPImportResult", @@ -51,7 +67,10 @@ "ExecuteCQResponse", "ExecuteRetentionResponse", "HealthResponse", + "LicenseHealth", + "StorageTierHealth", "MeasurementInfo", + "QueryProfile", "QueryResponse", "ReadyResponse", "RetentionExecution", diff --git a/src/arc_client/models/common.py b/src/arc_client/models/common.py index 7ad2685..381299e 100644 --- a/src/arc_client/models/common.py +++ b/src/arc_client/models/common.py @@ -4,19 +4,91 @@ from typing import Optional -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict + + +class StorageTierHealth(BaseModel): + """Credential health of one storage tier, from ``/health`` ``storage``. + + Mirrors Arc's ``database.StorageTierStatus``. ``state`` is e.g. ``"ok"``, + ``"fallback"``, ``"expired"`` or ``"unavailable"``. + """ + + model_config = ConfigDict(extra="allow") + + backend: Optional[str] = None + credentials: Optional[str] = None + state: Optional[str] = None + expires_at: Optional[str] = None + source: Optional[str] = None + last_refresh: Optional[str] = None + consecutive_errors: int = 0 + + +class LicenseHealth(BaseModel): + """License status, from ``/health`` ``license`` (Arc's ``license.LicenseHealth``).""" + + model_config = ConfigDict(extra="allow") + + tier: Optional[str] = None + status: Optional[str] = None + source: Optional[str] = None + expires_at: Optional[str] = None + days_remaining: int = 0 + site_license: bool = False class HealthResponse(BaseModel): - """Response from the health check endpoint.""" + """Response from the ``/health`` liveness endpoint. + + ``status`` is ``"ok"`` whenever the process is alive; Arc keeps it ``"ok"`` + even with expired storage credentials (that is ``/ready``'s job), so check + ``storage`` for credential state. + + Attributes: + status: Always ``"ok"`` from a live server. + time: Server time (RFC 3339). + uptime: Uptime as a Go duration string, e.g. ``"1h2m3.5s"``. + uptime_sec: Uptime in seconds. + storage: Per-tier credential health (``"hot"``, ``"cold"``), when + the server reports any. + license: License status, when the server has a license source wired. + version: Not sent by Arc's ``/health``; kept for backward + compatibility and always ``None`` against current servers. + """ + + model_config = ConfigDict(extra="allow") status: str - version: Optional[str] = None + time: Optional[str] = None uptime: Optional[str] = None + uptime_sec: Optional[float] = None + storage: Optional[dict[str, StorageTierHealth]] = None + license: Optional[LicenseHealth] = None + version: Optional[str] = None class ReadyResponse(BaseModel): - """Response from the readiness check endpoint.""" + """Response from ``/ready`` or ``/ready/write``. + + Attributes: + ready: True when the endpoint answered 200. + status: ``"ready"``, ``"not_ready"`` or (``/ready/write`` only) + ``"not_write_target"``. + reason: Why the node is not ready, e.g. + ``"storage_credentials_expired"`` or the startup/shutdown message. + tier: The storage tier a credential problem is on (``"hot"``/``"cold"``). + time: Server time (RFC 3339). + uptime_sec: Uptime in seconds (``/ready`` when ready). + message: Alias of ``reason``, kept for backward compatibility. + """ + + model_config = ConfigDict(extra="allow") ready: bool + status: Optional[str] = None + reason: Optional[str] = None + tier: Optional[str] = None + time: Optional[str] = None + uptime_sec: Optional[float] = None message: Optional[str] = None diff --git a/src/arc_client/models/database.py b/src/arc_client/models/database.py index a6466a7..01557d6 100644 --- a/src/arc_client/models/database.py +++ b/src/arc_client/models/database.py @@ -49,3 +49,37 @@ class MeasurementListResponse(BaseModel): database: str measurements: list[DatabaseMeasurement] = Field(default_factory=list) count: int = 0 + + +class SchemaField(BaseModel): + """One registered field of a measurement: its name and DuckDB type.""" + + model_config = ConfigDict(extra="allow") + + name: str + type: str + + +class MeasurementSchema(BaseModel): + """Response from ``GET /api/v1/databases/:db/measurements/:m/schema``. + + The measurement's registered field schema (Arc's stable-schema registry, + ``query.stable_schema``): the columns and types every query binds for it, + regardless of the time range read. + """ + + model_config = ConfigDict(extra="allow") + + database: str + measurement: str + fields: list[SchemaField] = Field(default_factory=list) + + +class SchemaRebuildResponse(BaseModel): + """Response from ``POST .../schema/rebuild`` (HTTP 202): the rebuild was queued.""" + + model_config = ConfigDict(extra="allow") + + status: str + database: str + measurement: str diff --git a/src/arc_client/validation.py b/src/arc_client/validation.py index 78b3bf1..cc521c5 100644 --- a/src/arc_client/validation.py +++ b/src/arc_client/validation.py @@ -7,6 +7,7 @@ from __future__ import annotations import re +from urllib.parse import quote from arc_client.exceptions import ArcValidationError @@ -39,3 +40,22 @@ def validate_name(name: str, kind: str = "name") -> str: "letters, digits, underscores, or hyphens (max 64 characters)." ) return name + + +def path_segment(name: str, kind: str = "name") -> str: + """Validate a name used as one URL path segment and percent-encode it. + + Looser than :func:`validate_name`, matching Arc's per-name storage-segment + rule (internal/api/databases.go isSafeStoragePathSegment): routes such as + the measurement schema must still resolve names an older release or a + spoke pseudo-database uses. Rejects what can never be one segment: an empty + name, a ``/`` or a leading ``.``. + + Raises: + ArcValidationError: If ``name`` cannot be a single path segment. + """ + if not isinstance(name, str) or not name or "/" in name or name.startswith("."): + raise ArcValidationError( + f"Invalid {kind} {name!r}: must be non-empty, contain no '/', and not start with '.'" + ) + return quote(name, safe="") diff --git a/tests/unit/http/test_core_parity.py b/tests/unit/http/test_core_parity.py new file mode 100644 index 0000000..5bd61d7 --- /dev/null +++ b/tests/unit/http/test_core_parity.py @@ -0,0 +1,246 @@ +"""Auth, health/readiness and measurement-schema contracts against Arc main.""" + +from __future__ import annotations + +import json + +import pytest +from pytest_httpx import HTTPXMock + +from arc_client.exceptions import ( + ArcConnectionError, + ArcNotFoundError, + ArcRateLimitError, + ArcServerError, + ArcValidationError, +) + +from ._parity import MODES, call, make_client + +# --- Bug 3: create_token(permissions=[]) ----------------------------------- + + +@MODES +async def test_create_token_sends_empty_permissions( + httpx_mock: HTTPXMock, async_mode: bool +) -> None: + # Arc: absent -> read,write; [] -> RBAC-only token (auth_routes.go). + httpx_mock.add_response(json={"success": True, "token": "tok"}) + async with make_client(async_mode) as client: + await call(client.auth.create_token("rbac-only", permissions=[])) + assert json.loads(httpx_mock.get_request().content) == { + "name": "rbac-only", + "permissions": [], + } + + +@MODES +async def test_create_token_omits_unset_permissions( + httpx_mock: HTTPXMock, async_mode: bool +) -> None: + httpx_mock.add_response(json={"success": True, "token": "tok"}) + async with make_client(async_mode) as client: + await call(client.auth.create_token("default")) + assert json.loads(httpx_mock.get_request().content) == {"name": "default"} + + +@MODES +async def test_update_token_sends_empty_permissions( + httpx_mock: HTTPXMock, async_mode: bool +) -> None: + httpx_mock.add_response(json={"success": True}) + async with make_client(async_mode) as client: + await call(client.auth.update_token(7, permissions=[])) + assert json.loads(httpx_mock.get_request().content) == {"permissions": []} + + +@MODES +async def test_auth_rate_limit_not_reported_as_auth_failure( + httpx_mock: HTTPXMock, async_mode: bool +) -> None: + httpx_mock.add_response(status_code=429, json={"error": "slow down"}, is_reusable=True) + async with make_client(async_mode, max_retries=0) as client: + with pytest.raises(ArcRateLimitError): + await call(client.auth.list_tokens()) + with pytest.raises(ArcRateLimitError): + await call(client.auth.create_token("x")) + + +# --- Item 14: /health, /ready, /ready/write ---------------------------------- + + +@MODES +async def test_health_models_arc_fields(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response( + json={ + "status": "ok", + "time": "2027-01-01T00:00:00Z", + "uptime": "1h0m0s", + "uptime_sec": 3600.5, + "storage": { + "cold": { + "backend": "s3", + "credentials": "refresher", + "state": "expired", + "consecutive_errors": 3, + } + }, + "license": {"tier": "enterprise", "status": "active", "source": "file"}, + } + ) + async with make_client(async_mode) as client: + health = await call(client.health()) + assert health.status == "ok" + assert health.uptime_sec == 3600.5 + assert health.time == "2027-01-01T00:00:00Z" + assert health.storage is not None + assert health.storage["cold"].state == "expired" + assert health.license is not None + assert health.license.tier == "enterprise" + assert health.version is None + + +@MODES +async def test_health_minimal_oss(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(json={"status": "ok", "uptime": "1s", "uptime_sec": 1}) + async with make_client(async_mode) as client: + health = await call(client.health()) + assert health.storage is None + assert health.license is None + + +@MODES +@pytest.mark.parametrize( + ("path", "method"), [("/ready", "readiness"), ("/ready/write", "write_readiness")] +) +async def test_not_ready_reason_and_single_probe( + httpx_mock: HTTPXMock, async_mode: bool, path: str, method: str +) -> None: + httpx_mock.add_response( + url=f"http://h:8000{path}", + status_code=503, + json={ + "status": "not_ready", + "time": "2027-01-01T00:00:00Z", + "reason": "storage_credentials_expired", + "tier": "cold", + }, + ) + async with make_client(async_mode, max_retries=3) as client: + result = await call(getattr(client, method)()) + assert result.ready is False + assert result.status == "not_ready" + assert result.reason == "storage_credentials_expired" + assert result.message == "storage_credentials_expired" + assert result.tier == "cold" + assert len(httpx_mock.get_requests()) == 1 + + +@MODES +async def test_ready_write_not_write_target(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response( + status_code=503, + json={"status": "not_write_target", "reason": "route writes to a writer"}, + ) + async with make_client(async_mode) as client: + result = await call(client.write_readiness()) + assert result.ready is False + assert result.status == "not_write_target" + + +@MODES +async def test_ready_ok(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(json={"status": "ready", "time": "t", "uptime_sec": 5}) + async with make_client(async_mode) as client: + detail = await call(client.readiness()) + assert detail.ready is True + assert detail.uptime_sec == 5 + assert httpx_mock.get_request().url.path == "/ready" + + +@MODES +async def test_ready_bool_is_single_probe(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(status_code=503, json={"status": "not_ready", "reason": "starting"}) + async with make_client(async_mode, max_retries=3) as client: + assert await call(client.ready()) is False + assert len(httpx_mock.get_requests()) == 1 + + +@MODES +async def test_readiness_other_errors_raise(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(status_code=500, json={"error": "boom"}) + async with make_client(async_mode, max_retries=0) as client: + with pytest.raises(ArcServerError): + await call(client.readiness()) + + +@MODES +async def test_readiness_unreachable_raises(httpx_mock: HTTPXMock, async_mode: bool) -> None: + import httpx + + httpx_mock.add_exception(httpx.ConnectError("refused")) + async with make_client(async_mode, max_retries=0) as client: + with pytest.raises(ArcConnectionError): + await call(client.readiness()) + + +# --- Item 13: measurement schema --------------------------------------------- + + +@MODES +async def test_get_measurement_schema(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response( + json={ + "database": "db", + "measurement": "cpu", + "fields": [{"name": "time", "type": "TIMESTAMP"}, {"name": "v", "type": "DOUBLE"}], + } + ) + async with make_client(async_mode) as client: + schema = await call(client.databases.get_measurement_schema("db", "cpu")) + request = httpx_mock.get_request() + assert request.method == "GET" + assert request.url.path == "/api/v1/databases/db/measurements/cpu/schema" + assert [(f.name, f.type) for f in schema.fields] == [("time", "TIMESTAMP"), ("v", "DOUBLE")] + + +@MODES +async def test_get_measurement_schema_registry_disabled( + httpx_mock: HTTPXMock, async_mode: bool +) -> None: + httpx_mock.add_response( + status_code=404, json={"error": "Field schema registry is disabled (query.stable_schema)"} + ) + async with make_client(async_mode) as client: + with pytest.raises(ArcNotFoundError): + await call(client.databases.get_measurement_schema("db", "cpu")) + + +@MODES +async def test_rebuild_measurement_schema(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response( + status_code=202, json={"status": "queued", "database": "db", "measurement": "cpu"} + ) + async with make_client(async_mode) as client: + result = await call(client.databases.rebuild_measurement_schema("db", "cpu")) + request = httpx_mock.get_request() + assert request.method == "POST" + assert request.url.path == "/api/v1/databases/db/measurements/cpu/schema/rebuild" + assert result.status == "queued" + + +@MODES +async def test_rebuild_conflict_is_not_retried(httpx_mock: HTTPXMock, async_mode: bool) -> None: + httpx_mock.add_response(status_code=409, json={"error": "already queued"}) + async with make_client(async_mode, max_retries=3) as client: + with pytest.raises(ArcServerError) as info: + await call(client.databases.rebuild_measurement_schema("db", "cpu")) + assert info.value.status_code == 409 + + +@MODES +@pytest.mark.parametrize("name", ["", "a/b", ".hidden"]) +async def test_schema_rejects_unsafe_segments(async_mode: bool, name: str) -> None: + async with make_client(async_mode) as client: + with pytest.raises(ArcValidationError): + await call(client.databases.get_measurement_schema("db", name)) From 64ec0371d1a5ea32536534c7a46006bebb4c305e Mon Sep 17 00:00:00 2001 From: Ignacio Van Droogenbroeck <64545348+xe-nvdk@users.noreply.github.com> Date: Mon, 2 Nov 2026 09:06:25 -0600 Subject: [PATCH 16/20] build(deps): bound httpcore to >=1.0.9,<1.1 instead of pinning The Arrow trailer adapter walks httpcore private attributes that are identical in 1.0.0, 1.0.5 and 1.0.9; 1.0.9 is the floor because it is the first release that accepts h11 0.16. An exact pin in a library blocks any future 1.0.x security fix for every downstream app. If a release moves the attributes, capture is skipped and trailer-announcing Arrow responses fail closed with ArcQueryError. --- pyproject.toml | 2 +- src/arc_client/http/arrow_trailers.py | 12 +++++++++++- uv.lock | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0b5f2bb..a08ba6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ classifiers = [ dependencies = [ "httpx>=0.27.0,<0.29", - "httpcore==1.0.9", + "httpcore>=1.0.9,<1.1", "h11>=0.16.0,<0.17", "msgpack>=1.0.7", "pyarrow>=15.0.0", diff --git a/src/arc_client/http/arrow_trailers.py b/src/arc_client/http/arrow_trailers.py index 916a694..dc7ddc7 100644 --- a/src/arc_client/http/arrow_trailers.py +++ b/src/arc_client/http/arrow_trailers.py @@ -5,6 +5,15 @@ the parser's receive buffer. Only the owning connection's parser is observed; TLS, proxies, timeouts, framing and pool ownership remain with HTTPX/httpcore. An unsupported stream shape is detected by the Arrow decoder, never guessed. + +Dependency bounds (pyproject.toml): the private attributes walked below +(``_stream``/``_httpcore_stream``/``_connection`` and ``HTTP11Connection. +_h11_state``) are identical in httpcore 1.0.0, 1.0.5 and 1.0.9, so httpcore +is bounded ``>=1.0.9,<1.1`` rather than pinned: 1.0.9 is the first release +that accepts h11 0.16, and a 1.0.x patch keeps the shape. If a release ever +moves them, capture is skipped and any response that announces Arc trailers +fails closed with ArcQueryError ("could not be captured"); it never yields a +table with guessed completion metadata. """ from __future__ import annotations @@ -60,7 +69,8 @@ def capture_arrow_trailers(response: httpx.Response) -> None: response.extensions[TRAILERS_EXTENSION] = sink # Bound{Sync,Async}Stream -> HTTPX response stream -> PoolByteStream -> # HTTP11ConnectionByteStream -> owning HTTP11 connection. Deliberately - # bounded to the dependency versions declared in pyproject.toml. + # bounded to the dependency versions declared in pyproject.toml (see the + # module docstring). owner: Any = response.stream for attribute in ("_stream", "_httpcore_stream", "_stream", "_connection"): owner = getattr(owner, attribute, None) diff --git a/uv.lock b/uv.lock index 77268af..ef1b3cf 100644 --- a/uv.lock +++ b/uv.lock @@ -109,7 +109,7 @@ polars = [ [package.metadata] requires-dist = [ { name = "h11", specifier = ">=0.16.0,<0.17" }, - { name = "httpcore", specifier = "==1.0.9" }, + { name = "httpcore", specifier = ">=1.0.9,<1.1" }, { name = "httpx", specifier = ">=0.27.0,<0.29" }, { name = "msgpack", specifier = ">=1.0.7" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.8.0" }, From 333c1a62061c71acefb4bc5dd25f4a472afc1679 Mon Sep 17 00:00:00 2001 From: Ignacio Van Droogenbroeck <64545348+xe-nvdk@users.noreply.github.com> Date: Mon, 2 Nov 2026 09:13:43 -0600 Subject: [PATCH 17/20] fix(write): resend refusals only for single-measurement bodies; one readiness API Arc checks its ingest fence once per request in AdmitRequest and again per measurement in checkIngestGate (internal/ingest/arrow_writer.go), so a fenced or refused 503 to a multi-measurement line-protocol or row-format body can arrive after earlier measurements were buffered. Resending it duplicates rows. Such refusals are now raised once with retry_safe=False. Also drop DiagnosticsClient.readiness/ReadinessStatus, which duplicated client.readiness()/write_readiness(); the readiness tests now exercise the client methods, which probe once instead of retrying a 503. --- src/arc_client/exceptions.py | 5 ++ src/arc_client/ingestion/async_writer.py | 32 ++++++++-- src/arc_client/ingestion/write_support.py | 47 ++++++++++++++ src/arc_client/ingestion/writer.py | 34 +++++++++-- .../management/async_diagnostics.py | 37 ++--------- src/arc_client/management/diagnostics.py | 47 ++------------ src/arc_client/models/__init__.py | 2 - src/arc_client/models/diagnostics.py | 31 ---------- tests/unit/http/test_diagnostics_contracts.py | 24 +++----- tests/unit/http/test_write_parity.py | 61 +++++++++++++++++++ 10 files changed, 191 insertions(+), 129 deletions(-) delete mode 100644 src/arc_client/models/diagnostics.py diff --git a/src/arc_client/exceptions.py b/src/arc_client/exceptions.py index ad71fef..7553d54 100644 --- a/src/arc_client/exceptions.py +++ b/src/arc_client/exceptions.py @@ -106,6 +106,11 @@ class ArcIngestRefusedError(ArcIngestionError): may already be stored, so a resend may duplicate rows. The SDK never retries it, and ``retry_safe`` is ``False``. + A refusal of a body spanning several measurements (multi-measurement line + protocol, row-format records) is also never resent and has + ``retry_safe=False``: Arc re-checks its fence per measurement, so the + refusal may follow measurements it already stored. + Attributes: reason: Arc's refusal reason (``"fenced"`` for a fence, otherwise the ``X-Arc-Ingest-Refused`` value such as ``"wal_disk"``). diff --git a/src/arc_client/ingestion/async_writer.py b/src/arc_client/ingestion/async_writer.py index 5d2a24d..b8b0e14 100644 --- a/src/arc_client/ingestion/async_writer.py +++ b/src/arc_client/ingestion/async_writer.py @@ -6,9 +6,14 @@ from arc_client.compression import Compression from arc_client.config import ClientConfig -from arc_client.exceptions import ArcError, ArcIngestionError, ArcValidationError +from arc_client.exceptions import ( + ArcError, + ArcIngestionError, + ArcIngestRefusedError, + ArcValidationError, +) from arc_client.http.async_http import AsyncHTTPClient -from arc_client.http.base import is_retryable_write_refusal +from arc_client.http.base import is_retryable_write_refusal, never_retry_status from arc_client.ingestion.compression import compress as compress_data from arc_client.ingestion.line_protocol import format_line_protocol from arc_client.ingestion.msgpack import ( @@ -18,6 +23,9 @@ ) from arc_client.ingestion.write_support import ( as_write_error, + refusal_without_resend, + single_lp_measurement, + single_record_measurement, tle_stats_body, validate_precision, ) @@ -97,7 +105,7 @@ async def write_records( except Exception as e: raise ArcValidationError(f"Failed to encode records: {e}") from e - await self._write_msgpack(data, database, compress) + await self._write_msgpack(data, database, compress, single_record_measurement(records)) async def write_dataframe( self, @@ -207,6 +215,7 @@ async def _post_write( compress: CompressArg, extra_headers: Optional[dict[str, str]] = None, params: Optional[dict[str, Any]] = None, + resend_refusals: bool = True, ) -> None: """Compress (if requested) and POST a write payload. @@ -219,6 +228,10 @@ async def _post_write( honoring ``Retry-After``. Nothing else is resent: a partial write, a ``wal_not_durable`` refusal or any other failure may have stored data. Errors are mapped by ``as_write_error``. + + ``resend_refusals=False`` is for bodies spanning several measurements: + Arc re-checks its fence per measurement, so a refusal may follow + stored measurements. Such a refusal is raised with ``retry_safe=False``. """ codec = self._resolve_codec(compress) data = compress_data(data, codec) @@ -235,10 +248,12 @@ async def _post_write( content=data, headers=headers, params=params, - retry_if=is_retryable_write_refusal, + retry_if=is_retryable_write_refusal if resend_refusals else never_retry_status, ) except Exception as e: mapped = as_write_error(e) + if not resend_refusals and isinstance(mapped, ArcIngestRefusedError): + mapped = refusal_without_resend(mapped) if mapped is e: raise raise mapped from e @@ -254,10 +269,16 @@ async def _write_msgpack( data: bytes, database: Optional[str], compress: CompressArg, + resend_refusals: bool = True, ) -> None: """Send MessagePack data to Arc.""" await self._post_write( - "/api/v1/write/msgpack", data, "application/msgpack", database, compress + "/api/v1/write/msgpack", + data, + "application/msgpack", + database, + compress, + resend_refusals=resend_refusals, ) async def _write_line_protocol( @@ -275,4 +296,5 @@ async def _write_line_protocol( database, compress, params={"precision": precision} if precision else None, + resend_refusals=single_lp_measurement(data), ) diff --git a/src/arc_client/ingestion/write_support.py b/src/arc_client/ingestion/write_support.py index 578a772..1b36ccc 100644 --- a/src/arc_client/ingestion/write_support.py +++ b/src/arc_client/ingestion/write_support.py @@ -9,6 +9,7 @@ ArcConnectionError, ArcError, ArcIngestionError, + ArcIngestRefusedError, ArcPartialWriteError, ArcServerError, ArcValidationError, @@ -25,6 +26,52 @@ _OTHERS_WRITTEN = "the other points were written" +# A line-protocol measurement ends at the first unescaped comma or space. +_LP_MEASUREMENT_RE = re.compile(rb"^((?:[^\\, ]|\\.)+)") + + +def single_lp_measurement(data: bytes) -> bool: + """True when every line-protocol line in ``data`` names the same measurement. + + Arc checks its ingest fence once per measurement while writing a request + (internal/api/lineprotocol.go handleWrite, ingest.checkIngestGate), so a + fenced/refused answer to a multi-measurement body can arrive after earlier + measurements were buffered. Only a single-measurement body is resent. + """ + seen: Optional[bytes] = None + for raw in data.splitlines(): + line = raw.strip() + if not line or line.startswith(b"#"): + continue + match = _LP_MEASUREMENT_RE.match(line) + if match is None: + return False + if seen is None: + seen = match.group(1) + elif match.group(1) != seen: + return False + return True + + +def single_record_measurement(records: list[dict[str, Any]]) -> bool: + """True when every row-format record targets the same measurement (see above).""" + return len({r.get("measurement") for r in records}) <= 1 + + +def refusal_without_resend(error: ArcIngestRefusedError) -> ArcIngestRefusedError: + """Re-issue a refusal for a multi-measurement body as not safe to resend.""" + if not error.retry_safe: + return error + refused = ArcIngestRefusedError( + f"{error} (not resent: the request spans several measurements, and Arc " + "may have stored the ones before the refusal)", + reason=error.reason, + retry_safe=False, + ) + refused.__cause__ = error + return refused + + def validate_precision(precision: Optional[str]) -> Optional[str]: """Return ``precision`` if Arc accepts it, raising ArcValidationError otherwise.""" if precision is None: diff --git a/src/arc_client/ingestion/writer.py b/src/arc_client/ingestion/writer.py index bc0e653..9ef2a20 100644 --- a/src/arc_client/ingestion/writer.py +++ b/src/arc_client/ingestion/writer.py @@ -6,8 +6,13 @@ from arc_client.compression import Compression from arc_client.config import ClientConfig -from arc_client.exceptions import ArcError, ArcIngestionError, ArcValidationError -from arc_client.http.base import is_retryable_write_refusal +from arc_client.exceptions import ( + ArcError, + ArcIngestionError, + ArcIngestRefusedError, + ArcValidationError, +) +from arc_client.http.base import is_retryable_write_refusal, never_retry_status from arc_client.http.sync_http import SyncHTTPClient from arc_client.ingestion.compression import compress as compress_data from arc_client.ingestion.line_protocol import format_line_protocol @@ -18,6 +23,9 @@ ) from arc_client.ingestion.write_support import ( as_write_error, + refusal_without_resend, + single_lp_measurement, + single_record_measurement, tle_stats_body, validate_precision, ) @@ -142,7 +150,7 @@ def write_records( except Exception as e: raise ArcValidationError(f"Failed to encode records: {e}") from e - self._write_msgpack(data, database, compress) + self._write_msgpack(data, database, compress, single_record_measurement(records)) def write_dataframe( self, @@ -352,6 +360,7 @@ def _post_write( compress: CompressArg, extra_headers: Optional[dict[str, str]] = None, params: Optional[dict[str, Any]] = None, + resend_refusals: bool = True, ) -> None: """Compress (if requested) and POST a write payload. @@ -364,6 +373,10 @@ def _post_write( honoring ``Retry-After``. Nothing else is resent: a partial write, a ``wal_not_durable`` refusal or any other failure may have stored data. Errors are mapped by ``as_write_error``. + + ``resend_refusals=False`` is for bodies spanning several measurements: + Arc re-checks its fence per measurement, so a refusal may follow + stored measurements. Such a refusal is raised with ``retry_safe=False``. """ codec = self._resolve_codec(compress) data = compress_data(data, codec) @@ -380,10 +393,12 @@ def _post_write( content=data, headers=headers, params=params, - retry_if=is_retryable_write_refusal, + retry_if=is_retryable_write_refusal if resend_refusals else never_retry_status, ) except Exception as e: mapped = as_write_error(e) + if not resend_refusals and isinstance(mapped, ArcIngestRefusedError): + mapped = refusal_without_resend(mapped) if mapped is e: raise raise mapped from e @@ -399,9 +414,17 @@ def _write_msgpack( data: bytes, database: Optional[str], compress: CompressArg, + resend_refusals: bool = True, ) -> None: """Send MessagePack data to Arc.""" - self._post_write("/api/v1/write/msgpack", data, "application/msgpack", database, compress) + self._post_write( + "/api/v1/write/msgpack", + data, + "application/msgpack", + database, + compress, + resend_refusals=resend_refusals, + ) def _write_line_protocol( self, @@ -418,4 +441,5 @@ def _write_line_protocol( database, compress, params={"precision": precision} if precision else None, + resend_refusals=single_lp_measurement(data), ) diff --git a/src/arc_client/management/async_diagnostics.py b/src/arc_client/management/async_diagnostics.py index 1ab2b10..6d60f53 100644 --- a/src/arc_client/management/async_diagnostics.py +++ b/src/arc_client/management/async_diagnostics.py @@ -1,9 +1,10 @@ """Asynchronous operational-diagnostics client for Arc. -Wraps Arc's readiness probes, JSON metrics, Prometheus metrics, buffered -application logs, and the admin memory diagnostics under ``/api/v1/debug``. +Wraps Arc's JSON metrics, Prometheus metrics, buffered application logs, and +the admin memory diagnostics under ``/api/v1/debug``. Readiness probes live on +the client itself (``readiness()`` / ``write_readiness()``). -Everything except :meth:`DiagnosticsClient.readiness` returns raw dicts: these +Everything returns raw dicts: these are operator diagnostics whose fields follow Arc's internal counters and Go runtime, not a stable contract. @@ -21,20 +22,17 @@ from typing import Any, Optional from arc_client.config import ClientConfig -from arc_client.exceptions import ArcError, ArcServerError +from arc_client.exceptions import ArcError from arc_client.http.async_http import AsyncHTTPClient from arc_client.management.diagnostics import ( _free_os_memory_guard, _logs_params, - _readiness_from_error, - _readiness_path, _timeseries_request, ) -from arc_client.models.diagnostics import ReadinessStatus class AsyncDiagnosticsClient: - """Readiness, metrics, logs and memory diagnostics.""" + """Metrics, logs and memory diagnostics.""" def __init__(self, http: AsyncHTTPClient, config: ClientConfig) -> None: self._http = http @@ -52,29 +50,6 @@ async def _json(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]: raise ArcError(f"Unexpected response from {path}: {data!r}") return data - # Readiness ----------------------------------------------------------- - - async def readiness(self, write: bool = False) -> ReadinessStatus: - """Ask whether this node should receive traffic (public). - - ``write=False`` probes ``/ready``: is the node ready at all. - ``write=True`` probes ``/ready/write``: should a load balancer send - *writes* here. It is strictly narrower — a reader, or a non-primary - node in local-storage mode, is ready but answers - ``status="not_write_target"``. - - Unlike :meth:`AsyncArcClient.ready`, the reason is kept: a 503 from the - probe is returned as a :class:`ReadinessStatus` with ``ready == - False``, not raised. Note that the SDK retries a GET that returns 503 - up to ``max_retries`` times before answering; construct the client - with ``max_retries=0`` for a probe that must answer immediately. - """ - try: - data = await self._json("get", _readiness_path(write)) - except ArcServerError as e: - return _readiness_from_error(e) - return ReadinessStatus.model_validate(data) - # Metrics ------------------------------------------------------------- async def metrics(self) -> dict[str, Any]: diff --git a/src/arc_client/management/diagnostics.py b/src/arc_client/management/diagnostics.py index 84d92e8..0c3f7de 100644 --- a/src/arc_client/management/diagnostics.py +++ b/src/arc_client/management/diagnostics.py @@ -1,9 +1,10 @@ """Synchronous operational-diagnostics client for Arc. -Wraps Arc's readiness probes, JSON metrics, Prometheus metrics, buffered -application logs, and the admin memory diagnostics under ``/api/v1/debug``. +Wraps Arc's JSON metrics, Prometheus metrics, buffered application logs, and +the admin memory diagnostics under ``/api/v1/debug``. Readiness probes live on +the client itself (``readiness()`` / ``write_readiness()``). -Everything except :meth:`DiagnosticsClient.readiness` returns raw dicts: these +Everything returns raw dicts: these are operator diagnostics whose fields follow Arc's internal counters and Go runtime, not a stable contract. @@ -21,9 +22,8 @@ from typing import Any, Optional from arc_client.config import ClientConfig -from arc_client.exceptions import ArcError, ArcServerError, ArcValidationError +from arc_client.exceptions import ArcError, ArcValidationError from arc_client.http.sync_http import SyncHTTPClient -from arc_client.models.diagnostics import ReadinessStatus TIMESERIES_TYPES = ("system", "application", "api") @@ -60,18 +60,6 @@ def _timeseries_request( return f"/api/v1/metrics/timeseries/{metric_type}", params -def _readiness_path(write: bool) -> str: - return "/ready/write" if write else "/ready" - - -def _readiness_from_error(error: ArcServerError) -> ReadinessStatus: - """A 503 from a readiness probe is an answer, not a failure.""" - body = error.response_body - if error.status_code == 503 and isinstance(body, dict) and isinstance(body.get("status"), str): - return ReadinessStatus.model_validate(body) - raise error - - def _free_os_memory_guard(confirm: bool) -> None: if not confirm: raise ArcValidationError( @@ -81,7 +69,7 @@ def _free_os_memory_guard(confirm: bool) -> None: class DiagnosticsClient: - """Readiness, metrics, logs and memory diagnostics.""" + """Metrics, logs and memory diagnostics.""" def __init__(self, http: SyncHTTPClient, config: ClientConfig) -> None: self._http = http @@ -99,29 +87,6 @@ def _json(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]: raise ArcError(f"Unexpected response from {path}: {data!r}") return data - # Readiness ----------------------------------------------------------- - - def readiness(self, write: bool = False) -> ReadinessStatus: - """Ask whether this node should receive traffic (public). - - ``write=False`` probes ``/ready``: is the node ready at all. - ``write=True`` probes ``/ready/write``: should a load balancer send - *writes* here. It is strictly narrower — a reader, or a non-primary - node in local-storage mode, is ready but answers - ``status="not_write_target"``. - - Unlike :meth:`ArcClient.ready`, the reason is kept: a 503 from the - probe is returned as a :class:`ReadinessStatus` with ``ready == - False``, not raised. Note that the SDK retries a GET that returns 503 - up to ``max_retries`` times before answering; construct the client - with ``max_retries=0`` for a probe that must answer immediately. - """ - try: - data = self._json("get", _readiness_path(write)) - except ArcServerError as e: - return _readiness_from_error(e) - return ReadinessStatus.model_validate(data) - # Metrics ------------------------------------------------------------- def metrics(self) -> dict[str, Any]: diff --git a/src/arc_client/models/__init__.py b/src/arc_client/models/__init__.py index 39bef62..8e5be9c 100644 --- a/src/arc_client/models/__init__.py +++ b/src/arc_client/models/__init__.py @@ -28,7 +28,6 @@ SchemaRebuildResponse, ) from arc_client.models.delete import DeleteConfigResponse, DeleteResponse -from arc_client.models.diagnostics import ReadinessStatus from arc_client.models.edge_sync import Spoke, SpokeSecret from arc_client.models.import_ import ( ImportResult, @@ -75,7 +74,6 @@ "MeasurementInfo", "QueryProfile", "QueryResponse", - "ReadinessStatus", "ReconciliationRun", "ReadyResponse", "RetentionExecution", diff --git a/src/arc_client/models/diagnostics.py b/src/arc_client/models/diagnostics.py deleted file mode 100644 index c726574..0000000 --- a/src/arc_client/models/diagnostics.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Models for Arc readiness probes.""" - -from __future__ import annotations - -from typing import Optional - -from pydantic import BaseModel, ConfigDict - - -class ReadinessStatus(BaseModel): - """The answer of ``/ready`` or ``/ready/write``. - - ``status`` is ``"ready"``, ``"not_ready"`` (the node is not ready for any - traffic) or, from ``/ready/write`` only, ``"not_write_target"`` (the node - is healthy but its cluster role does not take writes; route writes to a - writer). ``reason`` and, for a storage-credential problem, ``tier`` - explain a not-ready answer. ``uptime_sec`` is sent by ``/ready`` only. - """ - - model_config = ConfigDict(extra="allow") - - status: str - time: Optional[str] = None - reason: Optional[str] = None - tier: Optional[str] = None - uptime_sec: Optional[float] = None - - @property - def ready(self) -> bool: - """True when the node should receive the traffic the probe asked about.""" - return self.status == "ready" diff --git a/tests/unit/http/test_diagnostics_contracts.py b/tests/unit/http/test_diagnostics_contracts.py index b57ff72..9c6c1c5 100644 --- a/tests/unit/http/test_diagnostics_contracts.py +++ b/tests/unit/http/test_diagnostics_contracts.py @@ -18,10 +18,9 @@ ArcError, ArcPermissionError, ArcRateLimitError, - ArcServerError, ArcValidationError, ) -from arc_client.models.diagnostics import ReadinessStatus +from arc_client.models import ReadyResponse ROOT = "http://localhost:8000" @@ -42,7 +41,12 @@ async def enter_client(stack: AsyncExitStack, async_mode: bool) -> Any: return stack.enter_context(ArcClient(token="t", max_retries=0)) -# Readiness ----------------------------------------------------------------- +# Readiness lives on the client (client.readiness / write_readiness) -------- + + +async def probe(client: Any, write: bool) -> Any: + result = client.write_readiness() if write else client.readiness() + return await result if inspect.isawaitable(result) else result @pytest.mark.parametrize(("write", "path"), [(False, "/ready"), (True, "/ready/write")]) @@ -53,8 +57,8 @@ async def test_readiness_ready( httpx_mock.add_response(method="GET", url=f"{ROOT}{path}", json=body) async with AsyncExitStack() as stack: client = await enter_client(stack, async_mode) - result = await call(client, "readiness", write=write) - assert isinstance(result, ReadinessStatus) + result = await probe(client, write) + assert isinstance(result, ReadyResponse) assert result.ready is True assert result.uptime_sec == 12.5 @@ -73,21 +77,13 @@ async def test_readiness_503_is_an_answer( httpx_mock.add_response(status_code=503, json={**body, "time": "2027-01-05T10:00:00Z"}) async with AsyncExitStack() as stack: client = await enter_client(stack, async_mode) - result = await call(client, "readiness", write=write) + result = await probe(client, write) assert result.ready is False assert result.status == body["status"] assert result.reason == body["reason"] assert result.tier == body.get("tier") -async def test_readiness_other_503_raises(httpx_mock: HTTPXMock, async_mode: bool) -> None: - httpx_mock.add_response(status_code=503, text="upstream unavailable") - async with AsyncExitStack() as stack: - client = await enter_client(stack, async_mode) - with pytest.raises(ArcServerError): - await call(client, "readiness") - - # Metrics ------------------------------------------------------------------- diff --git a/tests/unit/http/test_write_parity.py b/tests/unit/http/test_write_parity.py index bf68a2b..620222c 100644 --- a/tests/unit/http/test_write_parity.py +++ b/tests/unit/http/test_write_parity.py @@ -80,6 +80,67 @@ async def test_safe_refusal_is_resent( assert requests[0].content == requests[1].content +def write_lp_multi(client: Any) -> Any: + return client.write.write_line_protocol(["cpu,host=a v=1i", "mem,host=a used=2i"]) + + +def write_records_multi(client: Any) -> Any: + return client.write.write_records( + [ + {"measurement": "cpu", "fields": {"v": 1}}, + {"measurement": "mem", "fields": {"used": 2}}, + ] + ) + + +@MODES +@pytest.mark.parametrize( + "write", + [pytest.param(write_lp_multi, id="lp"), pytest.param(write_records_multi, id="records")], +) +@pytest.mark.parametrize( + "headers", + [ + pytest.param({"X-Arc-Ingest-Refused": "wal_disk"}, id="wal_disk"), + pytest.param({"X-Arc-Ingest-Fenced": "1"}, id="fenced"), + ], +) +async def test_multi_measurement_refusal_is_not_resent( + httpx_mock: HTTPXMock, async_mode: bool, write: Any, headers: dict[str, str] +) -> None: + # Arc re-checks the fence per measurement (ingest.checkIngestGate), so the + # refusal may follow measurements it already buffered: a resend duplicates. + httpx_mock.add_response( + status_code=503, + headers={**headers, "Retry-After": "0"}, + json={"error": "Write rejected (retry)"}, + ) + async with make_client(async_mode, max_retries=3, **client_kwargs()) as client: + with pytest.raises(ArcIngestRefusedError) as info: + await call(write(client)) + assert info.value.retry_safe is False + assert "several measurements" in str(info.value) + assert len(httpx_mock.get_requests()) == 1 + + +@pytest.mark.parametrize( + ("body", "single"), + [ + (b"cpu,host=a v=1i 1\ncpu v=2i 2\n", True), + (b"# comment\n\ncpu v=1i\n", True), + (b"cpu,host=a v=1i\nmem v=1i\n", False), + (b"my\\ cpu,host=a v=1i\nmy\\ cpu v=2i\n", True), + (b"my\\ cpu v=1i\nmy v=2i\n", False), + (b"a\\,b v=1i\na v=1i\n", False), + (b"", True), + ], +) +def test_single_lp_measurement(body: bytes, single: bool) -> None: + from arc_client.ingestion.write_support import single_lp_measurement + + assert single_lp_measurement(body) is single + + @MODES async def test_safe_refusal_exhausted_reports_reason( httpx_mock: HTTPXMock, async_mode: bool From 79700950b89d789473c8f88e2290c7db252140ff Mon Sep 17 00:00:00 2001 From: Ignacio Van Droogenbroeck <64545348+xe-nvdk@users.noreply.github.com> Date: Mon, 2 Nov 2026 09:15:01 -0600 Subject: [PATCH 18/20] docs: consolidate 0.2.0 parity changelog and README --- CHANGELOG.md | 79 +++++++++++++++++++++++++++++++++++++ README.md | 109 ++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 178 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1a20b0..f44aa94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,62 @@ plus bug fixes and the first real HTTP-layer test coverage. ### Added +- **Write path (Arc 2027.01.1 parity).** + - `ArcPartialWriteError` (`dropped`, `others_written`) for Arc's + `400 "partial write"`. It is never retried, and buffered writers discard + that batch and report it once, so a resend cannot duplicate stored rows. + - `ArcIngestRefusedError` (`reason`, `retry_safe`) for 503 refusals + (`X-Arc-Ingest-Fenced`, `X-Arc-Ingest-Refused`). Single-measurement + writes are resent automatically except for `wal_not_durable`. Bodies + spanning several measurements are never resent, because Arc re-checks + its fence per measurement. + - `precision=` on `write_line_protocol`/`write_point`. + - `tag_columns=` on `write_columnar`/`write_dataframe` and buffered writes, + sent as msgpack `tag_keys`. +- **Query path.** + - `profile=True` on `query`/`query_msgpack`, returned as `QueryResponse.profile`. + - `dictionary=`/`compression=` Arrow wire options. + - `QueryResponse.query_id`, and `arc.query_id` in the Arrow schema metadata, + for `client.queries.kill()`. + - `client.query.query_measurement()` for `GET /api/v1/query/:measurement`. +- **Measurement schema**: `databases.get_measurement_schema()` and + `rebuild_measurement_schema()`. +- **Readiness**: `client.readiness()` and `client.write_readiness()` return + `ReadyResponse` (`status`, `reason`, `tier`). A 503 is an answer, not an + error, and is not retried. +- **Compaction**: + - `compaction.cycles()`, `compaction.cycle(id)`, `compaction.too_wide()`, + `compaction.clear_too_wide()` and `history(cycle_id=)` (Arc 2027.01.1+). + - `trigger(measurement=)` (Arc 2026.09.2+). +- **Backup**: `create(databases=[...])`, `list(include_foreign=True)` and + `restore(mode="merge"|"replace")` (Arc 2027.01.1+). Restore still requires + `confirm=True`. +- **Delete**: `allow_unreachable=True` on `delete.delete()` and + `databases.delete()` (Arc 2027.01.1+). A cluster delete then proceeds + despite unreachable files or unflushed peers, leaves those rows in place, + and names them in the response. +- **Continuous queries**: `ExecuteCQResponse.watermark_status` (Arc 2027.01.1+). +- **Cluster operator controls** (admin): + - `delete_file`, `demote_writer`, `assign_compactor`, `converge_voters` + (Arc 2026.09.2+). + - `stream_claims`, `release_stream_claim`, `abort_compaction_job`, + `policies`, `adopt_policies` (Arc 2027.01.1+). + - Destructive controls require `confirm=True`. `converge_voters` only + plans unless `dry_run=False, confirm=True`. +- **Reconciliation** (`client.reconciliation`): `status()`, `trigger()` and + `get_run()`. `trigger()` is a dry run by default; acting requires + `dry_run=False, confirm=True`. +- **Operational diagnostics** (`client.diagnostics`): + - Public JSON metrics, Prometheus text and timeseries. + - Admin `logs`, `memstats`, `duckdb_memory` and + `free_os_memory(confirm=True)`. + - `auth.cache_stats()` and `auth.invalidate_cache()`. +- **Edge sync administration** (`client.edge_sync`): + - Hub: spoke registry and air-gap bundle import. + - Spoke: status, ledger, sync, requeue/dismiss, and bundle export/ack. + - `rotate_spoke_secret`, `delete_spoke` and `dismiss_failed` require + `confirm=True`. + - **Installation ID for Arc's client telemetry.** Requests carry a random per-installation `Arc-Client-Installation-Id` and a `User-Agent` that names the Python version and platform, so Arc's opt-out telemetry can count SDK @@ -67,6 +123,15 @@ plus bug fixes and the first real HTTP-layer test coverage. ### Changed +- MessagePack queries are stable. A stream that fails after the 200 raises + `ArcQueryError` (incomplete body) instead of returning partial rows. +- `ready()` is a single probe: a 503 "not ready" is no longer retried with backoff. +- `httpcore` is bounded `>=1.0.9,<1.1` instead of pinned `==1.0.9`. +- Compaction docstrings state that every `/api/v1/compaction` route, reads + included, requires an admin token, as Arc enforces. +- `User-Agent` now includes the Python version and platform + (`arc-client-python/0.2.0 (python/3.12.4; linux/amd64)`). + - Default write compression is now **zstd** (was gzip). Passing `compression=True` still maps to gzip for backward compatibility. - Writes no longer send a `Content-Encoding` header; Arc auto-detects the codec @@ -83,6 +148,20 @@ plus bug fixes and the first real HTTP-layer test coverage. ### Fixed +- Query POSTs (`query`, `query_arrow`, `query_msgpack`, `estimate`) now retry + 429/503 with `Retry-After`. Arc sends these during replication catch-up, + for a vanished partition, and when the cold tier is unavailable. A + transport failure mid-response is still not replayed. +- Query and write clients no longer hide typed errors. Auth, permission, + scoped-access, license, rate-limit and not-supported errors surface as + themselves. +- `create_token(permissions=[])` and `update_token(permissions=[])` send the + empty list, creating an RBAC-only token, instead of silently producing a + read/write token. +- `HealthResponse` models Arc's real `/health` (`time`, `uptime_sec`, + `storage`, `license`). +- `dataframe_to_columnar(tag_columns=...)` is no longer a no-op. + - MessagePack queries preserve cap metadata and exact timestamp nanoseconds. DELETE results retain partial-failure, cold-storage and peer diagnostics. - Retention and continuous-query clients accept Arc's list/null/flat response diff --git a/README.md b/README.md index 5355452..6bdfb6c 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,28 @@ tle = """ISS (ZARYA) client.write.write_tle(tle, measurement="satellite_tle") ``` +### Precision, Tags and Partial Writes + +```python +client.write.write_line_protocol("cpu,host=a v=1 1700000000", precision="s") +client.write.write_columnar("cpu", cols, tag_columns=["host", "region"]) +client.write.write_dataframe(df, "cpu", tag_columns=["host"]) + +from arc_client import ArcPartialWriteError, ArcIngestRefusedError +try: + client.write.write_line_protocol(lines) +except ArcPartialWriteError as e: # never resend: the rest was stored + print(e.dropped, e.others_written) +except ArcIngestRefusedError as e: # retries exhausted, or unsafe to resend + print(e.reason, e.retry_safe) +``` + +Arc refuses writes with a 503 during failover or under disk pressure. The SDK +resends single-measurement writes that Arc reports as "nothing was written". +It never resends a partial write, a `wal_not_durable` refusal, or a refusal +of a body spanning several measurements: Arc may already have stored part of +those. + ## Bulk Import Import files directly (admin token required). Each returns a typed result with @@ -260,10 +282,12 @@ DataFrame conversions can drop schema metadata; inspect the Arrow table first when completeness matters. Trailer capture is tested with HTTPX 0.27/0.28, httpcore 1.0.9 and h11 0.16. -Those dependency bounds are deliberate: the adapter observes HTTP/1.1 parser -completion without replacing HTTPX's TLS, proxy, timeout or pooling behavior. +httpcore is bounded `>=1.0.9,<1.1`, because the internals the adapter reads are +unchanged across 1.0.x. If a release changes them, Arrow responses that +announce trailers fail with `ArcQueryError` rather than return unverified +metadata. -### MessagePack Response (experimental) +### MessagePack Response A compact columnar MessagePack response with native timestamp decoding. Returns the same `QueryResponse` shape as `query()` (row-major `data`), plus per-column @@ -274,9 +298,9 @@ result = client.query.query_msgpack("SELECT * FROM default.cpu LIMIT 1000") print(result.columns, result.types) ``` -> **Experimental.** This uses `/api/v1/query/msgpack`, which requires the Arc -> server to be built with the `duckdb_arrow` build tag. Against a server without -> it, the call raises `ArcNotSupportedError` (HTTP 501). +This uses `/api/v1/query/msgpack`, stable since Arc 2026.09.1. Older servers +built without the `duckdb_arrow` tag answer 501, which raises +`ArcNotSupportedError`. ### Query Estimation @@ -296,6 +320,25 @@ for m in measurements: print(f"{m.measurement}: {m.file_count} files, {m.total_size_mb:.1f} MB") ``` +### Profiling, Query IDs and Measurement Reads + +```python +r = client.query.query("SELECT * FROM cpu", profile=True) +print(r.profile.total_ms, r.profile.rows_scanned, r.query_id) +client.queries.kill(r.query_id) + +# Arrow wire encodings; the query ID is in the schema metadata +t = client.query.query_arrow("SELECT * FROM cpu", dictionary=True, compression="zstd") +t.schema.metadata[b"arc.query_id"] + +# Rows of one measurement without free-form SQL +client.query.query_measurement("cpu", "metrics", limit=100, order_by="time ASC", where="host = 'a'") + +# Stable measurement schema (admin rebuild is queued, 202) +s = client.databases.get_measurement_schema("metrics", "cpu") +client.databases.rebuild_measurement_schema("metrics", "cpu") +``` + ## Async Support All operations have async equivalents: @@ -450,14 +493,58 @@ client.audit.logs(event_type="query", since="2024-01-01T00:00:00Z", limit=100) client.queries.active() client.queries.kill("query-id") -# Cluster / compaction / tiering / backup / MQTT +# Cluster / compaction / tiering / backup / MQTT (admin) client.cluster.status() -client.compaction.trigger(tier="hourly,daily") +client.cluster.converge_voters() # plan only +client.cluster.converge_voters(dry_run=False, confirm=True) +client.cluster.demote_writer("writer-1", confirm=True) +client.cluster.policies(kind="retention") # on the Raft leader + +client.compaction.trigger(database="metrics", measurement="cpu") +client.compaction.cycles(limit=5) +client.compaction.too_wide() + client.tiering.set_policy("metrics", hot_only=True, hot_max_age_days=7) -client.backup.create() + +client.backup.create(databases=["metrics"]) # scoped backup +client.backup.list(include_foreign=True) +client.backup.restore("backup-id", confirm=True, mode="merge") + client.mqtt.list_subscriptions() ``` +## Operations and Diagnostics + +```python +# Readiness: should this node get traffic / writes, and if not, why? +r = client.readiness(); print(r.ready, r.reason, r.tier) +client.write_readiness().status # "ready" | "not_ready" | "not_write_target" + +# Metrics (public) and logs (admin) +client.diagnostics.metrics() +client.diagnostics.timeseries("api", duration_minutes=15) +client.diagnostics.logs(level="warn", since_minutes=30, limit=200) +client.auth.cache_stats() + +# Reconciliation (enterprise clustering): dry run first, then act +run = client.reconciliation.trigger() # dry run +client.reconciliation.trigger(dry_run=False, confirm=True) + +# Edge sync, on the hub +secret = client.edge_sync.register_spoke("rocket-01", name="Rocket") +print(secret.secret) # shown only once +client.edge_sync.import_bundle("/mnt/drive/bundle-1") + +# Edge sync, on the spoke +client.edge_sync.spoke_status() +client.edge_sync.run_sync() # long-running: use a large client timeout +client.edge_sync.requeue_failed(all_failed=True) +``` + +Destructive or disruptive calls (`reconciliation.trigger(dry_run=False)`, +`diagnostics.free_os_memory`, `edge_sync.rotate_spoke_secret`, +`edge_sync.delete_spoke`, `edge_sync.dismiss_failed`) require `confirm=True`. + ## Configuration ```python @@ -494,6 +581,8 @@ from arc_client.exceptions import ( ArcLicenseError, # Enterprise feature not licensed (403) ArcQueryError, # Query execution errors ArcIngestionError, # Write failures + ArcPartialWriteError, # Some points dropped, others stored (400); never resend + ArcIngestRefusedError, # Write refused (503); .reason, .retry_safe ArcValidationError, # Invalid input ArcNotFoundError, # Resource not found (404) ArcRateLimitError, # Rate limited (429) @@ -562,7 +651,7 @@ Both sync and async clients provide the same methods. CQ replacements include `tag_columns`; retention replacements include `database`, `measurement`, and `buffer_days`. Query results retain `truncated`, `truncation_reason`, `rows_capped`, and `row_cap` -when their wire format supplies them. Experimental `query_msgpack()` returns UTC +when their wire format supplies them. `query_msgpack()` returns UTC `datetime` values when a timestamp is exactly representable at microsecond resolution. Submicrosecond or out-of-range timestamps remain `msgpack.Timestamp`, preserving `.seconds`, `.nanoseconds`, and `.to_unix_nano()`. Such cells require an explicit From 75512fbed67b56cfbf3c20fbf253155b0c87c65f Mon Sep 17 00:00:00 2001 From: Ignacio Van Droogenbroeck <64545348+xe-nvdk@users.noreply.github.com> Date: Mon, 2 Nov 2026 09:31:36 -0600 Subject: [PATCH 19/20] fix: address review findings on the 0.2.0 parity branch - Buffered writers no longer lose partial-write errors: they are queued, reported before any terminal failure, and several are folded into one ArcPartialWriteError (dropped summed). A second background partial no longer overwrites the first, and close()/flush() report them even when another measurement fails. - A retry-safe refusal (fenced, wal_disk) keeps the batch and leaves the buffered writer usable; auto-flushes inside write() leave it for the next flush. Only wal_not_durable and other uncertain failures are terminal. - Buffered write() validates record tags before buffering, so a bad tag name raises at once instead of poisoning the writer. - MessagePack refusals are resent again for multi-measurement records: Arc checks the fence once per msgpack batch (ArrowBuffer.Write). Only multi-measurement line protocol stays gated, and that check now runs only when Arc actually refuses a write. - Query POSTs retry 503 only. Arc's query 429 is governance (rate limit or quota) without Retry-After and does not clear within the backoff. - The installation id never fails a request: an unresolvable home directory falls back to an in-memory id. It is resolved once per HTTP client, so no env lookups or first-run file I/O run per request or on an event loop. --- CHANGELOG.md | 10 +- README.md | 4 +- src/arc_client/_installation.py | 13 ++- src/arc_client/exceptions.py | 8 +- src/arc_client/http/async_http.py | 9 +- src/arc_client/http/base.py | 16 ++- src/arc_client/http/sync_http.py | 9 +- src/arc_client/ingestion/async_buffered.py | 108 ++++++++++++-------- src/arc_client/ingestion/async_writer.py | 31 +++--- src/arc_client/ingestion/buffered.py | 110 +++++++++++++-------- src/arc_client/ingestion/write_support.py | 22 +++-- src/arc_client/ingestion/writer.py | 31 +++--- src/arc_client/query/async_executor.py | 5 +- src/arc_client/query/executor.py | 5 +- tests/unit/http/test_installation_id.py | 11 +++ tests/unit/http/test_query_parity.py | 25 ++++- tests/unit/http/test_write_parity.py | 24 +++-- tests/unit/test_buffered_partial_write.py | 87 +++++++++++++++- 18 files changed, 369 insertions(+), 159 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f44aa94..92b4433 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,10 +17,10 @@ plus bug fixes and the first real HTTP-layer test coverage. `400 "partial write"`. It is never retried, and buffered writers discard that batch and report it once, so a resend cannot duplicate stored rows. - `ArcIngestRefusedError` (`reason`, `retry_safe`) for 503 refusals - (`X-Arc-Ingest-Fenced`, `X-Arc-Ingest-Refused`). Single-measurement - writes are resent automatically except for `wal_not_durable`. Bodies - spanning several measurements are never resent, because Arc re-checks - its fence per measurement. + (`X-Arc-Ingest-Fenced`, `X-Arc-Ingest-Refused`). Writes are resent + automatically except for `wal_not_durable` and line-protocol bodies + spanning several measurements, where Arc re-checks its fence per + measurement. A buffered writer keeps a refused batch and stays usable. - `precision=` on `write_line_protocol`/`write_point`. - `tag_columns=` on `write_columnar`/`write_dataframe` and buffered writes, sent as msgpack `tag_keys`. @@ -149,7 +149,7 @@ plus bug fixes and the first real HTTP-layer test coverage. ### Fixed - Query POSTs (`query`, `query_arrow`, `query_msgpack`, `estimate`) now retry - 429/503 with `Retry-After`. Arc sends these during replication catch-up, + 503 with `Retry-After`. A governance 429 is raised at once. Arc sends these during replication catch-up, for a vanished partition, and when the cold tier is unavailable. A transport failure mid-response is still not replayed. - Query and write clients no longer hide typed errors. Auth, permission, diff --git a/README.md b/README.md index 6bdfb6c..f284f1e 100644 --- a/README.md +++ b/README.md @@ -173,8 +173,8 @@ except ArcIngestRefusedError as e: # retries exhausted, or unsafe to resend Arc refuses writes with a 503 during failover or under disk pressure. The SDK resends single-measurement writes that Arc reports as "nothing was written". It never resends a partial write, a `wal_not_durable` refusal, or a refusal -of a body spanning several measurements: Arc may already have stored part of -those. +of a line-protocol body spanning several measurements: Arc may already have +stored part of those. ## Bulk Import diff --git a/src/arc_client/_installation.py b/src/arc_client/_installation.py index 01b2655..f0bcf7c 100644 --- a/src/arc_client/_installation.py +++ b/src/arc_client/_installation.py @@ -103,7 +103,12 @@ def _valid(value: str) -> Optional[str]: def _load_or_create() -> str: - path = _config_dir() / _FILE_NAME + try: + # Path.home() raises when no home directory resolves (an arbitrary + # container UID without a passwd entry and no HOME). + path = _config_dir() / _FILE_NAME + except Exception: + return str(uuid.uuid4()) try: existing = _valid(path.read_text(encoding="ascii")) if existing: @@ -138,7 +143,11 @@ def installation_id() -> str: global _cached_id with _lock: if _cached_id is None: - _cached_id = _load_or_create() + try: + _cached_id = _load_or_create() + except Exception: + # Never let identity fail a request (see module docstring). + _cached_id = str(uuid.uuid4()) return _cached_id diff --git a/src/arc_client/exceptions.py b/src/arc_client/exceptions.py index 7553d54..6589652 100644 --- a/src/arc_client/exceptions.py +++ b/src/arc_client/exceptions.py @@ -106,10 +106,10 @@ class ArcIngestRefusedError(ArcIngestionError): may already be stored, so a resend may duplicate rows. The SDK never retries it, and ``retry_safe`` is ``False``. - A refusal of a body spanning several measurements (multi-measurement line - protocol, row-format records) is also never resent and has - ``retry_safe=False``: Arc re-checks its fence per measurement, so the - refusal may follow measurements it already stored. + A refusal of a line-protocol body spanning several measurements is also + never resent and has ``retry_safe=False``: Arc re-checks its fence per + measurement there, so the refusal may follow measurements it already + stored. MessagePack writes are checked once per request and are resent. Attributes: reason: Arc's refusal reason (``"fenced"`` for a fence, otherwise the diff --git a/src/arc_client/http/async_http.py b/src/arc_client/http/async_http.py index 446639f..df70882 100644 --- a/src/arc_client/http/async_http.py +++ b/src/arc_client/http/async_http.py @@ -60,7 +60,7 @@ async def _request( Reads (GET/HEAD/OPTIONS) retry transient transport failures and 429/503 responses up to ``config.max_retries`` times. Requests the caller marks ``idempotent=True`` (Arc's query POSTs, which only read) - retry 429/503 like a read; for transport failures they keep the + retry 503 (not 429, see below); for transport failures they keep the mutation rule (only connection establishment failures), because a timeout or a connection lost mid-response means Arc already ran the query, and silently re-running it would multiply its cost and hide a @@ -97,8 +97,11 @@ async def _request( if retry_if is not None: retry_status = retry_if(response) else: - retry_status = (read_only or idempotent) and is_retryable_status( - response.status_code + # Idempotent POSTs (queries) retry only 503: Arc's query 429 + # is governance (rate limit or quota) with no Retry-After, and + # does not clear within a few seconds of backoff. + retry_status = (read_only and is_retryable_status(response.status_code)) or ( + idempotent and response.status_code == 503 ) if retry_status and attempt < self.config.max_retries: retry_after = parse_retry_after(response) diff --git a/src/arc_client/http/base.py b/src/arc_client/http/base.py index ecdf6aa..24c8ad1 100644 --- a/src/arc_client/http/base.py +++ b/src/arc_client/http/base.py @@ -33,9 +33,15 @@ def build_headers( - config: ClientConfig, extra_headers: Optional[dict[str, str]] = None + config: ClientConfig, + extra_headers: Optional[dict[str, str]] = None, + installation_id: Optional[str] = None, ) -> dict[str, str]: - """Build request headers with authentication.""" + """Build request headers with authentication. + + ``installation_id`` is resolved once per HTTP client (see + ``arc_client._installation``) and omitted when None. + """ headers: dict[str, str] = { "User-Agent": USER_AGENT, } @@ -43,7 +49,6 @@ def build_headers( headers["Authorization"] = f"Bearer {config.token}" if config.database: headers["x-arc-database"] = config.database - installation_id = outbound_installation_id(config.send_installation_id) if installation_id: headers[HEADER_INSTALLATION_ID] = installation_id if extra_headers: @@ -316,6 +321,9 @@ class HTTPClientBase: def __init__(self, config: ClientConfig) -> None: self.config = config self._base_url = config.base_url + # Resolved once: env opt-outs are read when the client is created, and + # the first-run file I/O never happens on an event loop. + self._installation_id = outbound_installation_id(config.send_installation_id) def _build_url(self, path: str) -> str: """Build full URL from path.""" @@ -329,7 +337,7 @@ def _prepare_request_kwargs( **kwargs: Any, ) -> dict[str, Any]: """Prepare common request kwargs.""" - request_headers = build_headers(self.config, headers) + request_headers = build_headers(self.config, headers, self._installation_id) return { "headers": request_headers, "timeout": self.config.timeout, diff --git a/src/arc_client/http/sync_http.py b/src/arc_client/http/sync_http.py index b128d79..ea780bb 100644 --- a/src/arc_client/http/sync_http.py +++ b/src/arc_client/http/sync_http.py @@ -60,7 +60,7 @@ def _request( Reads (GET/HEAD/OPTIONS) retry transient transport failures and 429/503 responses up to ``config.max_retries`` times. Requests the caller marks ``idempotent=True`` (Arc's query POSTs, which only read) - retry 429/503 like a read; for transport failures they keep the + retry 503 (not 429, see below); for transport failures they keep the mutation rule (only connection establishment failures), because a timeout or a connection lost mid-response means Arc already ran the query, and silently re-running it would multiply its cost and hide a @@ -97,8 +97,11 @@ def _request( if retry_if is not None: retry_status = retry_if(response) else: - retry_status = (read_only or idempotent) and is_retryable_status( - response.status_code + # Idempotent POSTs (queries) retry only 503: Arc's query 429 + # is governance (rate limit or quota) with no Retry-After, and + # does not clear within a few seconds of backoff. + retry_status = (read_only and is_retryable_status(response.status_code)) or ( + idempotent and response.status_code == 503 ) if retry_status and attempt < self.config.max_retries: retry_after = parse_retry_after(response) diff --git a/src/arc_client/ingestion/async_buffered.py b/src/arc_client/ingestion/async_buffered.py index 797c575..3c177e1 100644 --- a/src/arc_client/ingestion/async_buffered.py +++ b/src/arc_client/ingestion/async_buffered.py @@ -7,8 +7,9 @@ from collections import defaultdict from typing import TYPE_CHECKING, Any, Optional -from arc_client.exceptions import ArcPartialWriteError +from arc_client.exceptions import ArcIngestRefusedError, ArcPartialWriteError from arc_client.ingestion.msgpack import validate_tag_keys +from arc_client.ingestion.write_support import combine_partial_writes if TYPE_CHECKING: from arc_client.ingestion.async_writer import AsyncWriteClient @@ -61,7 +62,7 @@ def __init__( self._flush_error: BaseException | None = None # A partial write from a background flush, reported once by the next # foreground call (see the class docstring). - self._partial_error: Optional[ArcPartialWriteError] = None + self._partial_errors: list[ArcPartialWriteError] = [] # Columns sent as tag_keys, per measurement, in first-seen order. self._tag_keys: dict[str, dict[str, None]] = defaultdict(dict) @@ -90,6 +91,7 @@ async def write(self, record: dict[str, Any]) -> None: columns[key] = [value] for key, value in tags.items(): columns[key] = [value] + validate_tag_keys(list(tags), columns) async with self._lock: self._raise_if_failed() @@ -100,11 +102,17 @@ async def write(self, record: dict[str, Any]) -> None: for key in tags: self._tag_keys[measurement][key] = None - if self._record_counts[measurement] >= self._batch_size: - await self._flush_measurement_unlocked(measurement) - - if time.monotonic() - self._last_flush_time >= self._flush_interval: - await self._flush_all_unlocked() + try: + if self._record_counts[measurement] >= self._batch_size: + await self._flush_measurement_unlocked(measurement) + if time.monotonic() - self._last_flush_time >= self._flush_interval: + await self._flush_all_unlocked() + except ArcIngestRefusedError as error: + # Arc applied nothing and the batch is still buffered; the + # next flush resends it. flush()/close() report it if it lasts. + if not error.retry_safe: + raise + self._raise_pending_partial() async def write_columnar( self, @@ -128,11 +136,17 @@ async def write_columnar( for key in tag_columns or (): self._tag_keys[measurement][key] = None - if self._record_counts[measurement] >= self._batch_size: - await self._flush_measurement_unlocked(measurement) - - if time.monotonic() - self._last_flush_time >= self._flush_interval: - await self._flush_all_unlocked() + try: + if self._record_counts[measurement] >= self._batch_size: + await self._flush_measurement_unlocked(measurement) + if time.monotonic() - self._last_flush_time >= self._flush_interval: + await self._flush_all_unlocked() + except ArcIngestRefusedError as error: + # Arc applied nothing and the batch is still buffered; the + # next flush resends it. flush()/close() report it if it lasts. + if not error.retry_safe: + raise + self._raise_pending_partial() async def flush(self) -> None: """Flush all buffered data, waiting for any in-flight write. @@ -141,12 +155,22 @@ async def flush(self) -> None: subsequent flush/close calls raise that failure without retrying data whose server outcome may be unknown. Producers wait while I/O runs. - A partial write (``ArcPartialWriteError``) is the exception: its batch - is discarded and the error raised once, after the other measurements - were flushed. + Two exceptions: + + * A partial write (``ArcPartialWriteError``): its batch is discarded and + the error raised once, after the other measurements were flushed. + Several partial writes are folded into one error. + * A retry-safe refusal (``ArcIngestRefusedError`` with + ``retry_safe=True``, e.g. during a failover): Arc applied nothing, so + the batch stays buffered and the writer stays usable; flush again. """ async with self._lock: - await self._flush_all_unlocked() + try: + await self._flush_all_unlocked() + except BaseException: + # Report dropped points first; a terminal failure stays sticky. + self._raise_pending_partial() + raise self._raise_pending_partial() async def _flush_loop(self) -> None: @@ -167,25 +191,26 @@ async def _flush_loop(self) -> None: ): try: await self._flush_all_unlocked() - except ArcPartialWriteError as error: - # Batch already discarded; report it once from the - # foreground and keep the timer running. - self._partial_error = error + except ArcIngestRefusedError as error: + # Nothing applied, batch kept: resend next tick. + if not error.retry_safe: + raise except Exception as error: # Foreground flush/close will surface this error. Do not retry an # uncertain outcome, or leave a task exception unobserved. self._flush_error = error def _raise_if_failed(self) -> None: - """Raise a terminal failure, then a pending partial write (once).""" + """Raise pending partial writes (once), then a terminal failure.""" + self._raise_pending_partial() if self._flush_error is not None: raise self._flush_error - self._raise_pending_partial() def _raise_pending_partial(self) -> None: - error, self._partial_error = self._partial_error, None - if error is not None: - raise error + """Raise every partial write not yet reported, folded into one error.""" + errors, self._partial_errors = self._partial_errors, [] + if errors: + raise combine_partial_writes(errors) def _discard(self, measurement: str) -> None: """Drop a measurement's buffered batch. Must hold lock.""" @@ -207,10 +232,17 @@ async def _flush_measurement_unlocked(self, measurement: str) -> None: await self._client.write_columnar(measurement, merged, tag_columns=tags) else: await self._client.write_columnar(measurement, merged) - except ArcPartialWriteError: - # Arc stored what it could; resending would duplicate it. + except ArcPartialWriteError as error: + # Arc stored what it could; resending would duplicate it. Queue the + # error so a later failure in this flush cannot lose it. self._discard(measurement) self._last_flush_time = time.monotonic() + self._partial_errors.append(error) + return + except ArcIngestRefusedError as error: + if not error.retry_safe: + self._flush_error = error + # Retry-safe: Arc applied nothing; keep the batch for a resend. raise except BaseException as error: self._flush_error = error @@ -222,21 +254,14 @@ async def _flush_measurement_unlocked(self, measurement: str) -> None: async def _flush_all_unlocked(self) -> None: """Flush all measurements. Must hold lock. - A partial write does not stop the other measurements from flushing; the - first one is raised after they have been sent. + A partial write does not stop the other measurements from flushing: it + is queued and raised by the caller via ``_raise_pending_partial``. """ if self._flush_error is not None: raise self._flush_error - partial: Optional[ArcPartialWriteError] = None - measurements = list(self._buffers.keys()) - for measurement in measurements: + for measurement in list(self._buffers.keys()): if self._buffers[measurement]: - try: - await self._flush_measurement_unlocked(measurement) - except ArcPartialWriteError as error: - partial = partial or error - if partial is not None: - raise partial + await self._flush_measurement_unlocked(measurement) def _merge_columnar(self, batches: list[dict[str, list[Any]]]) -> dict[str, list[Any]]: """Merge multiple columnar batches into one. @@ -285,9 +310,10 @@ async def close(self) -> None: async with self._lock: try: await self._flush_all_unlocked() - except ArcPartialWriteError: - # Everything was flushed or discarded; the writer is done. - self._closed = True + except BaseException: + # Report dropped points first; the failure itself stays sticky + # (or, for a retry-safe refusal, close() can be called again). + self._raise_pending_partial() raise self._closed = True self._raise_pending_partial() diff --git a/src/arc_client/ingestion/async_writer.py b/src/arc_client/ingestion/async_writer.py index b8b0e14..3df5df3 100644 --- a/src/arc_client/ingestion/async_writer.py +++ b/src/arc_client/ingestion/async_writer.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Optional, Union from arc_client.compression import Compression from arc_client.config import ClientConfig @@ -13,7 +13,7 @@ ArcValidationError, ) from arc_client.http.async_http import AsyncHTTPClient -from arc_client.http.base import is_retryable_write_refusal, never_retry_status +from arc_client.http.base import is_retryable_write_refusal from arc_client.ingestion.compression import compress as compress_data from arc_client.ingestion.line_protocol import format_line_protocol from arc_client.ingestion.msgpack import ( @@ -25,7 +25,6 @@ as_write_error, refusal_without_resend, single_lp_measurement, - single_record_measurement, tle_stats_body, validate_precision, ) @@ -105,7 +104,7 @@ async def write_records( except Exception as e: raise ArcValidationError(f"Failed to encode records: {e}") from e - await self._write_msgpack(data, database, compress, single_record_measurement(records)) + await self._write_msgpack(data, database, compress) async def write_dataframe( self, @@ -215,7 +214,7 @@ async def _post_write( compress: CompressArg, extra_headers: Optional[dict[str, str]] = None, params: Optional[dict[str, Any]] = None, - resend_refusals: bool = True, + resend_refusals: Callable[[], bool] = lambda: True, ) -> None: """Compress (if requested) and POST a write payload. @@ -229,9 +228,10 @@ async def _post_write( ``wal_not_durable`` refusal or any other failure may have stored data. Errors are mapped by ``as_write_error``. - ``resend_refusals=False`` is for bodies spanning several measurements: - Arc re-checks its fence per measurement, so a refusal may follow - stored measurements. Such a refusal is raised with ``retry_safe=False``. + ``resend_refusals`` is asked only when Arc refuses the write. It answers + False for multi-measurement line protocol: Arc re-checks its fence per + measurement there, so a refusal may follow stored measurements. Such a + refusal is raised with ``retry_safe=False``. """ codec = self._resolve_codec(compress) data = compress_data(data, codec) @@ -248,11 +248,11 @@ async def _post_write( content=data, headers=headers, params=params, - retry_if=is_retryable_write_refusal if resend_refusals else never_retry_status, + retry_if=lambda r: is_retryable_write_refusal(r) and resend_refusals(), ) except Exception as e: mapped = as_write_error(e) - if not resend_refusals and isinstance(mapped, ArcIngestRefusedError): + if isinstance(mapped, ArcIngestRefusedError) and not resend_refusals(): mapped = refusal_without_resend(mapped) if mapped is e: raise @@ -269,16 +269,19 @@ async def _write_msgpack( data: bytes, database: Optional[str], compress: CompressArg, - resend_refusals: bool = True, ) -> None: - """Send MessagePack data to Arc.""" + """Send MessagePack data to Arc. + + Arc checks its ingest fence once for the whole msgpack batch + (internal/ingest/arrow_writer.go ArrowBuffer.Write), so a refusal + applied nothing and is safe to resend, whatever the measurements. + """ await self._post_write( "/api/v1/write/msgpack", data, "application/msgpack", database, compress, - resend_refusals=resend_refusals, ) async def _write_line_protocol( @@ -296,5 +299,5 @@ async def _write_line_protocol( database, compress, params={"precision": precision} if precision else None, - resend_refusals=single_lp_measurement(data), + resend_refusals=lambda: single_lp_measurement(data), ) diff --git a/src/arc_client/ingestion/buffered.py b/src/arc_client/ingestion/buffered.py index e699918..e0e8cd3 100644 --- a/src/arc_client/ingestion/buffered.py +++ b/src/arc_client/ingestion/buffered.py @@ -11,8 +11,9 @@ from collections import defaultdict from typing import TYPE_CHECKING, Any, Optional -from arc_client.exceptions import ArcPartialWriteError +from arc_client.exceptions import ArcIngestRefusedError, ArcPartialWriteError from arc_client.ingestion.msgpack import validate_tag_keys +from arc_client.ingestion.write_support import combine_partial_writes if TYPE_CHECKING: from arc_client.ingestion.writer import WriteClient @@ -82,7 +83,7 @@ def __init__( self._flush_error: BaseException | None = None # A partial write from a background flush, reported once by the next # foreground call (see the class docstring). - self._partial_error: Optional[ArcPartialWriteError] = None + self._partial_errors: list[ArcPartialWriteError] = [] # Columns sent as tag_keys, per measurement, in first-seen order. self._tag_keys: dict[str, dict[str, None]] = defaultdict(dict) @@ -124,6 +125,7 @@ def write(self, record: dict[str, Any]) -> None: columns[key] = [value] for key, value in tags.items(): columns[key] = [value] + validate_tag_keys(list(tags), columns) with self._lock: self._raise_if_failed() @@ -135,12 +137,17 @@ def write(self, record: dict[str, Any]) -> None: self._tag_keys[measurement][key] = None # Check if we should flush this measurement - if self._record_counts[measurement] >= self._batch_size: - self._flush_measurement(measurement) - - # Check time-based flush - if time.monotonic() - self._last_flush_time >= self._flush_interval: - self._flush_all() + try: + if self._record_counts[measurement] >= self._batch_size: + self._flush_measurement(measurement) + if time.monotonic() - self._last_flush_time >= self._flush_interval: + self._flush_all() + except ArcIngestRefusedError as error: + # Arc applied nothing and the batch is still buffered; the + # next flush resends it. flush()/close() report it if it lasts. + if not error.retry_safe: + raise + self._raise_pending_partial() def write_columnar( self, @@ -172,12 +179,17 @@ def write_columnar( self._tag_keys[measurement][key] = None # Check if we should flush this measurement - if self._record_counts[measurement] >= self._batch_size: - self._flush_measurement(measurement) - - # Check time-based flush - if time.monotonic() - self._last_flush_time >= self._flush_interval: - self._flush_all() + try: + if self._record_counts[measurement] >= self._batch_size: + self._flush_measurement(measurement) + if time.monotonic() - self._last_flush_time >= self._flush_interval: + self._flush_all() + except ArcIngestRefusedError as error: + # Arc applied nothing and the batch is still buffered; the + # next flush resends it. flush()/close() report it if it lasts. + if not error.retry_safe: + raise + self._raise_pending_partial() def flush(self) -> None: """Flush all buffered data, waiting for any in-flight write. @@ -186,12 +198,22 @@ def flush(self) -> None: subsequent flush/close calls raise that failure without retrying data whose server outcome may be unknown. Producers wait while I/O runs. - A partial write (``ArcPartialWriteError``) is the exception: its batch - is discarded and the error raised once, after the other measurements - were flushed. + Two exceptions: + + * A partial write (``ArcPartialWriteError``): its batch is discarded and + the error raised once, after the other measurements were flushed. + Several partial writes are folded into one error. + * A retry-safe refusal (``ArcIngestRefusedError`` with + ``retry_safe=True``, e.g. during a failover): Arc applied nothing, so + the batch stays buffered and the writer stays usable; flush again. """ with self._lock: - self._flush_all() + try: + self._flush_all() + except BaseException: + # Report dropped points first; a terminal failure stays sticky. + self._raise_pending_partial() + raise self._raise_pending_partial() def _start_flush_thread(self) -> None: @@ -217,10 +239,10 @@ def _flush_loop(self) -> None: ): try: self._flush_all() - except ArcPartialWriteError as error: - # Batch already discarded; report it once from the - # foreground and keep the timer running. - self._partial_error = error + except ArcIngestRefusedError as error: + # Nothing applied, batch kept: resend next tick. + if not error.retry_safe: + raise except Exception as error: # Keep the error observable on foreground operations; never retry # an uncertain write automatically from the background thread. @@ -228,15 +250,16 @@ def _flush_loop(self) -> None: self._flush_error = error def _raise_if_failed(self) -> None: - """Raise a terminal failure, then a pending partial write (once).""" + """Raise pending partial writes (once), then a terminal failure.""" + self._raise_pending_partial() if self._flush_error is not None: raise self._flush_error - self._raise_pending_partial() def _raise_pending_partial(self) -> None: - error, self._partial_error = self._partial_error, None - if error is not None: - raise error + """Raise every partial write not yet reported, folded into one error.""" + errors, self._partial_errors = self._partial_errors, [] + if errors: + raise combine_partial_writes(errors) def _discard(self, measurement: str) -> None: """Drop a measurement's buffered batch. Must hold lock.""" @@ -260,10 +283,17 @@ def _flush_measurement(self, measurement: str) -> None: self._client.write_columnar(measurement, merged, tag_columns=tags) else: self._client.write_columnar(measurement, merged) - except ArcPartialWriteError: - # Arc stored what it could; resending would duplicate it. + except ArcPartialWriteError as error: + # Arc stored what it could; resending would duplicate it. Queue the + # error so a later failure in this flush cannot lose it. self._discard(measurement) self._last_flush_time = time.monotonic() + self._partial_errors.append(error) + return + except ArcIngestRefusedError as error: + if not error.retry_safe: + self._flush_error = error + # Retry-safe: Arc applied nothing; keep the batch for a resend. raise except BaseException as error: self._flush_error = error @@ -275,21 +305,14 @@ def _flush_measurement(self, measurement: str) -> None: def _flush_all(self) -> None: """Flush all measurements. Must hold lock. - A partial write does not stop the other measurements from flushing; the - first one is raised after they have been sent. + A partial write does not stop the other measurements from flushing: it + is queued and raised by the caller via ``_raise_pending_partial``. """ if self._flush_error is not None: raise self._flush_error - partial: Optional[ArcPartialWriteError] = None - measurements = list(self._buffers.keys()) - for measurement in measurements: + for measurement in list(self._buffers.keys()): if self._buffers[measurement]: - try: - self._flush_measurement(measurement) - except ArcPartialWriteError as error: - partial = partial or error - if partial is not None: - raise partial + self._flush_measurement(measurement) def _merge_columnar(self, batches: list[dict[str, list[Any]]]) -> dict[str, list[Any]]: """Merge multiple columnar batches into one. @@ -338,9 +361,10 @@ def close(self) -> None: with self._lock: try: self._flush_all() - except ArcPartialWriteError: - # Everything was flushed or discarded; the writer is done. - self._closed = True + except BaseException: + # Report dropped points first; the failure itself stays sticky + # (or, for a retry-safe refusal, close() can be called again). + self._raise_pending_partial() raise self._closed = True self._raise_pending_partial() diff --git a/src/arc_client/ingestion/write_support.py b/src/arc_client/ingestion/write_support.py index 1b36ccc..9094576 100644 --- a/src/arc_client/ingestion/write_support.py +++ b/src/arc_client/ingestion/write_support.py @@ -53,17 +53,12 @@ def single_lp_measurement(data: bytes) -> bool: return True -def single_record_measurement(records: list[dict[str, Any]]) -> bool: - """True when every row-format record targets the same measurement (see above).""" - return len({r.get("measurement") for r in records}) <= 1 - - def refusal_without_resend(error: ArcIngestRefusedError) -> ArcIngestRefusedError: """Re-issue a refusal for a multi-measurement body as not safe to resend.""" if not error.retry_safe: return error refused = ArcIngestRefusedError( - f"{error} (not resent: the request spans several measurements, and Arc " + f"{error} (not resent: the line-protocol body spans several measurements, and Arc " "may have stored the ones before the refusal)", reason=error.reason, retry_safe=False, @@ -72,6 +67,21 @@ def refusal_without_resend(error: ArcIngestRefusedError) -> ArcIngestRefusedErro return refused +def combine_partial_writes(errors: list[ArcPartialWriteError]) -> ArcPartialWriteError: + """Fold the partial writes of several buffered batches into one error.""" + if len(errors) == 1: + return errors[0] + dropped = sum(e.dropped for e in errors) + combined = ArcPartialWriteError( + f"Arc dropped {dropped} point(s) across {len(errors)} buffered batches: " + + "; ".join(str(e) for e in errors), + dropped=dropped, + others_written=any(e.others_written for e in errors), + ) + combined.__cause__ = errors[0] + return combined + + def validate_precision(precision: Optional[str]) -> Optional[str]: """Return ``precision`` if Arc accepts it, raising ArcValidationError otherwise.""" if precision is None: diff --git a/src/arc_client/ingestion/writer.py b/src/arc_client/ingestion/writer.py index 9ef2a20..6bf65d2 100644 --- a/src/arc_client/ingestion/writer.py +++ b/src/arc_client/ingestion/writer.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Optional, Union from arc_client.compression import Compression from arc_client.config import ClientConfig @@ -12,7 +12,7 @@ ArcIngestRefusedError, ArcValidationError, ) -from arc_client.http.base import is_retryable_write_refusal, never_retry_status +from arc_client.http.base import is_retryable_write_refusal from arc_client.http.sync_http import SyncHTTPClient from arc_client.ingestion.compression import compress as compress_data from arc_client.ingestion.line_protocol import format_line_protocol @@ -25,7 +25,6 @@ as_write_error, refusal_without_resend, single_lp_measurement, - single_record_measurement, tle_stats_body, validate_precision, ) @@ -150,7 +149,7 @@ def write_records( except Exception as e: raise ArcValidationError(f"Failed to encode records: {e}") from e - self._write_msgpack(data, database, compress, single_record_measurement(records)) + self._write_msgpack(data, database, compress) def write_dataframe( self, @@ -360,7 +359,7 @@ def _post_write( compress: CompressArg, extra_headers: Optional[dict[str, str]] = None, params: Optional[dict[str, Any]] = None, - resend_refusals: bool = True, + resend_refusals: Callable[[], bool] = lambda: True, ) -> None: """Compress (if requested) and POST a write payload. @@ -374,9 +373,10 @@ def _post_write( ``wal_not_durable`` refusal or any other failure may have stored data. Errors are mapped by ``as_write_error``. - ``resend_refusals=False`` is for bodies spanning several measurements: - Arc re-checks its fence per measurement, so a refusal may follow - stored measurements. Such a refusal is raised with ``retry_safe=False``. + ``resend_refusals`` is asked only when Arc refuses the write. It answers + False for multi-measurement line protocol: Arc re-checks its fence per + measurement there, so a refusal may follow stored measurements. Such a + refusal is raised with ``retry_safe=False``. """ codec = self._resolve_codec(compress) data = compress_data(data, codec) @@ -393,11 +393,11 @@ def _post_write( content=data, headers=headers, params=params, - retry_if=is_retryable_write_refusal if resend_refusals else never_retry_status, + retry_if=lambda r: is_retryable_write_refusal(r) and resend_refusals(), ) except Exception as e: mapped = as_write_error(e) - if not resend_refusals and isinstance(mapped, ArcIngestRefusedError): + if isinstance(mapped, ArcIngestRefusedError) and not resend_refusals(): mapped = refusal_without_resend(mapped) if mapped is e: raise @@ -414,16 +414,19 @@ def _write_msgpack( data: bytes, database: Optional[str], compress: CompressArg, - resend_refusals: bool = True, ) -> None: - """Send MessagePack data to Arc.""" + """Send MessagePack data to Arc. + + Arc checks its ingest fence once for the whole msgpack batch + (internal/ingest/arrow_writer.go ArrowBuffer.Write), so a refusal + applied nothing and is safe to resend, whatever the measurements. + """ self._post_write( "/api/v1/write/msgpack", data, "application/msgpack", database, compress, - resend_refusals=resend_refusals, ) def _write_line_protocol( @@ -441,5 +444,5 @@ def _write_line_protocol( database, compress, params={"precision": precision} if precision else None, - resend_refusals=single_lp_measurement(data), + resend_refusals=lambda: single_lp_measurement(data), ) diff --git a/src/arc_client/query/async_executor.py b/src/arc_client/query/async_executor.py index a906174..6ea9417 100644 --- a/src/arc_client/query/async_executor.py +++ b/src/arc_client/query/async_executor.py @@ -63,9 +63,10 @@ async def query( This method returns structured JSON data. For large result sets, consider using query_arrow() or query_polars() for better performance. - Queries only read, so a 429 or 503 (Arc's replication catch-up gate, a + Queries only read, so a 503 (Arc's replication catch-up gate, a partition removed mid-query, an unavailable cold tier) is retried with - backoff, honoring ``Retry-After``, up to ``max_retries``. + backoff, honoring ``Retry-After``, up to ``max_retries``. A governance + 429 (rate limit or quota) is raised at once as ``ArcRateLimitError``. Args: sql: SQL query to execute. diff --git a/src/arc_client/query/executor.py b/src/arc_client/query/executor.py index c106c90..2b8a992 100644 --- a/src/arc_client/query/executor.py +++ b/src/arc_client/query/executor.py @@ -63,9 +63,10 @@ def query( This method returns structured JSON data. For large result sets, consider using query_arrow() or query_polars() for better performance. - Queries only read, so a 429 or 503 (Arc's replication catch-up gate, a + Queries only read, so a 503 (Arc's replication catch-up gate, a partition removed mid-query, an unavailable cold tier) is retried with - backoff, honoring ``Retry-After``, up to ``max_retries``. + backoff, honoring ``Retry-After``, up to ``max_retries``. A governance + 429 (rate limit or quota) is raised at once as ``ArcRateLimitError``. Args: sql: SQL query to execute. diff --git a/tests/unit/http/test_installation_id.py b/tests/unit/http/test_installation_id.py index b591a9e..835740f 100644 --- a/tests/unit/http/test_installation_id.py +++ b/tests/unit/http/test_installation_id.py @@ -134,3 +134,14 @@ def test_user_agent_shape() -> None: # Arc keeps only printable ASCII up to 32 chars per field. for part in m.groups(): assert 0 < len(part) <= 32 and part.isascii() and part.isprintable() + + +def test_unresolvable_home_never_fails_a_request(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("XDG_CONFIG_HOME") + monkeypatch.delenv("APPDATA") + + def no_home() -> Path: + raise RuntimeError("Could not determine home directory.") + + monkeypatch.setattr(Path, "home", staticmethod(no_home)) + assert UUID_RE.match(_installation.installation_id()) diff --git a/tests/unit/http/test_query_parity.py b/tests/unit/http/test_query_parity.py index 441f873..ca19e9e 100644 --- a/tests/unit/http/test_query_parity.py +++ b/tests/unit/http/test_query_parity.py @@ -79,20 +79,19 @@ def ok_response(httpx_mock: HTTPXMock, endpoint: str) -> None: httpx_mock.add_response(url=url, json=JSON_OK) -# --- Bug 1: query POSTs are reads and retry 429/503 ------------------------ +# --- Bug 1: query POSTs are reads and retry 503 (not governance 429) -------- @MODES @pytest.mark.parametrize("endpoint", list(ENDPOINTS)) -@pytest.mark.parametrize("status", [429, 503]) -async def test_query_post_retries_retryable_status( - httpx_mock: HTTPXMock, async_mode: bool, endpoint: str, status: int +async def test_query_post_retries_503( + httpx_mock: HTTPXMock, async_mode: bool, endpoint: str ) -> None: path, run = ENDPOINTS[endpoint] # Arc's replication catch-up gate (query.go checkReplicationReady). httpx_mock.add_response( url=f"http://h:8000{path}", - status_code=status, + status_code=503, headers={"Retry-After": "0"}, json={"success": False, "error": "replication_catch_up_in_progress"}, ) @@ -102,6 +101,22 @@ async def test_query_post_retries_retryable_status( assert len(httpx_mock.get_requests()) == 2 +@MODES +@pytest.mark.parametrize("endpoint", list(ENDPOINTS)) +async def test_query_post_does_not_retry_governance_429( + httpx_mock: HTTPXMock, async_mode: bool, endpoint: str +) -> None: + # query.go checkQueryGovernance: 429 with no Retry-After (rate limit or quota). + path, run = ENDPOINTS[endpoint] + httpx_mock.add_response( + url=f"http://h:8000{path}", status_code=429, json={"error": "rate limit exceeded"} + ) + async with make_client(async_mode, max_retries=2) as client: + with pytest.raises(ArcRateLimitError): + await call(run(client)) + assert len(httpx_mock.get_requests()) == 1 + + @MODES async def test_query_503_exhausted_raises_query_error_with_status( httpx_mock: HTTPXMock, async_mode: bool diff --git a/tests/unit/http/test_write_parity.py b/tests/unit/http/test_write_parity.py index 620222c..fcac8b3 100644 --- a/tests/unit/http/test_write_parity.py +++ b/tests/unit/http/test_write_parity.py @@ -94,10 +94,6 @@ def write_records_multi(client: Any) -> Any: @MODES -@pytest.mark.parametrize( - "write", - [pytest.param(write_lp_multi, id="lp"), pytest.param(write_records_multi, id="records")], -) @pytest.mark.parametrize( "headers", [ @@ -105,9 +101,10 @@ def write_records_multi(client: Any) -> Any: pytest.param({"X-Arc-Ingest-Fenced": "1"}, id="fenced"), ], ) -async def test_multi_measurement_refusal_is_not_resent( - httpx_mock: HTTPXMock, async_mode: bool, write: Any, headers: dict[str, str] +async def test_multi_measurement_lp_refusal_is_not_resent( + httpx_mock: HTTPXMock, async_mode: bool, headers: dict[str, str] ) -> None: + write = write_lp_multi # Arc re-checks the fence per measurement (ingest.checkIngestGate), so the # refusal may follow measurements it already buffered: a resend duplicates. httpx_mock.add_response( @@ -123,6 +120,21 @@ async def test_multi_measurement_refusal_is_not_resent( assert len(httpx_mock.get_requests()) == 1 +@MODES +async def test_multi_measurement_records_refusal_is_resent( + httpx_mock: HTTPXMock, async_mode: bool +) -> None: + # The msgpack path checks the fence once for the whole batch + # (arrow_writer.go ArrowBuffer.Write): nothing was applied. + httpx_mock.add_response( + status_code=503, headers={"X-Arc-Ingest-Fenced": "1", "Retry-After": "0"}, json={} + ) + httpx_mock.add_response(status_code=204) + async with make_client(async_mode, max_retries=2, **client_kwargs()) as client: + await call(write_records_multi(client)) + assert len(httpx_mock.get_requests()) == 2 + + @pytest.mark.parametrize( ("body", "single"), [ diff --git a/tests/unit/test_buffered_partial_write.py b/tests/unit/test_buffered_partial_write.py index dfd2f7e..e727b05 100644 --- a/tests/unit/test_buffered_partial_write.py +++ b/tests/unit/test_buffered_partial_write.py @@ -9,7 +9,7 @@ import pytest from arc_client import ArcPartialWriteError -from arc_client.exceptions import ArcServerError +from arc_client.exceptions import ArcIngestRefusedError, ArcServerError, ArcValidationError from arc_client.ingestion.async_buffered import AsyncBufferedWriter from arc_client.ingestion.buffered import BufferedWriter @@ -103,7 +103,7 @@ def test_sync_background_partial_is_reported_once_by_next_call() -> None: # Wait until the timer has recorded the partial write. for _ in range(100): with buffer._lock: - if buffer._partial_error is not None: + if buffer._partial_errors: break threading.Event().wait(0.01) with pytest.raises(ArcPartialWriteError): @@ -160,10 +160,91 @@ async def test_async_background_partial_is_reported_once_by_next_call() -> None: buffer._buffers["cpu"].append({"time": [1], "v": [1.0]}) buffer._record_counts["cpu"] += 1 for _ in range(200): - if buffer._partial_error is not None: + if buffer._partial_errors: break await asyncio.sleep(0.01) with pytest.raises(ArcPartialWriteError): await buffer.write({**ROW, "timestamp": 2}) await buffer.write({**ROW, "timestamp": 3}) assert [c[1]["time"] for c in fake.calls] == [[1], [3]] + + +# --- Review fixes: no lost partials, retry-safe refusals, tag validation ----- + + +def refused(retry_safe: bool = True) -> ArcIngestRefusedError: + return ArcIngestRefusedError("refused", reason="fenced", retry_safe=retry_safe) + + +def make(async_mode: bool, fail: dict[int, BaseException]) -> tuple[Any, Any]: + if async_mode: + fake: Any = AsyncFakeWriter(fail) + return fake, AsyncBufferedWriter(fake, batch_size=100, flush_interval=3600) + fake = FakeWriter(fail) + return fake, BufferedWriter(fake, batch_size=100, flush_interval=3600) + + +async def run(result: Any) -> Any: + return await result if asyncio.iscoroutine(result) else result + + +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_partial_survives_a_later_failure_in_the_same_flush(async_mode: bool) -> None: + # cpu: partial write (discarded); mem: server error (terminal). + fake, buffer = make(async_mode, {0: partial(), 1: ArcServerError("boom", status_code=500)}) + await run(buffer.write(ROW)) + await run(buffer.write({**ROW, "measurement": "mem"})) + + with pytest.raises(ArcPartialWriteError): + await run(buffer.flush()) + with pytest.raises(ArcServerError): + await run(buffer.flush()) + + +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_several_partials_are_folded_into_one(async_mode: bool) -> None: + fake, buffer = make(async_mode, {0: partial(), 1: partial()}) + await run(buffer.write(ROW)) + await run(buffer.write({**ROW, "measurement": "mem"})) + + with pytest.raises(ArcPartialWriteError) as info: + await run(buffer.flush()) + assert info.value.dropped == 2 + await run(buffer.flush()) # reported once + + +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_retry_safe_refusal_keeps_batch_and_writer(async_mode: bool) -> None: + fake, buffer = make(async_mode, {0: refused()}) + await run(buffer.write(ROW)) + + with pytest.raises(ArcIngestRefusedError): + await run(buffer.flush()) + assert buffer.pending_count == 1 + + await run(buffer.flush()) # resent, accepted + assert buffer.pending_count == 0 + await run(buffer.close()) + assert [c[1]["time"] for c in fake.calls] == [[1], [1]] + + +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_unsafe_refusal_is_terminal(async_mode: bool) -> None: + fake, buffer = make(async_mode, {0: refused(retry_safe=False)}) + await run(buffer.write(ROW)) + for _ in range(2): + with pytest.raises(ArcIngestRefusedError): + await run(buffer.flush()) + assert len(fake.calls) == 1 + + +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_bad_tag_is_rejected_before_buffering(async_mode: bool) -> None: + fake, buffer = make(async_mode, {}) + with pytest.raises(ArcValidationError): + await run(buffer.write({**ROW, "tags": {"time": "x"}})) + assert buffer.pending_count == 0 + + await run(buffer.write(ROW)) # writer is still usable + await run(buffer.close()) + assert len(fake.calls) == 1 From cb1d3f7daa572db2f4c6d285487c9391263783bb Mon Sep 17 00:00:00 2001 From: Ignacio Van Droogenbroeck <64545348+xe-nvdk@users.noreply.github.com> Date: Mon, 2 Nov 2026 09:32:13 -0600 Subject: [PATCH 20/20] docs(changelog): clarify query retry entry --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92b4433..339d046 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -149,9 +149,10 @@ plus bug fixes and the first real HTTP-layer test coverage. ### Fixed - Query POSTs (`query`, `query_arrow`, `query_msgpack`, `estimate`) now retry - 503 with `Retry-After`. A governance 429 is raised at once. Arc sends these during replication catch-up, - for a vanished partition, and when the cold tier is unavailable. A - transport failure mid-response is still not replayed. + 503 with `Retry-After`, which Arc sends during replication catch-up, for a + vanished partition, and when the cold tier is unavailable. A governance 429 + (rate limit or quota) is raised at once. A transport failure mid-response + is still not replayed. - Query and write clients no longer hide typed errors. Auth, permission, scoped-access, license, rate-limit and not-supported errors surface as themselves.