From 72a4c0258751b04a4f043bdcd4e77fecbcfb931f Mon Sep 17 00:00:00 2001 From: Gustavo Caso Date: Tue, 1 Sep 2026 12:58:56 +0200 Subject: [PATCH 1/3] eops-378(fix): Fix using --all display a warning letting the user know there are more records if we hit the limit --- src/dualentry_cli/client.py | 42 +++++++++++++---- src/dualentry_cli/commands/__init__.py | 47 ++++++++++++++++++- tests/test_client.py | 62 ++++++++++++++++++++++++++ tests/test_commands.py | 34 ++++++++++++++ 4 files changed, 175 insertions(+), 10 deletions(-) diff --git a/src/dualentry_cli/client.py b/src/dualentry_cli/client.py index 280f9c9..48348de 100644 --- a/src/dualentry_cli/client.py +++ b/src/dualentry_cli/client.py @@ -33,6 +33,10 @@ _MAX_RETRY_AFTER = 60 +# Hard ceiling for --all crawls: page_size 100 x 1000 pages = 100_000 items. +# Truncation must warn; never report the truncated length as the API total. +_MAX_PAGES = 1000 + def _retry_after_seconds(response: httpx.Response) -> int | None: """Seconds from the Retry-After header, or None if absent or unusable.""" @@ -206,27 +210,47 @@ def _request(self, method: str, path: str, **kwargs) -> dict: def get(self, path: str, params: dict[str, Any] | None = None) -> dict: return self._request("GET", path, params=params) - def paginate(self, path: str, params: dict[str, Any] | None = None, page_size: int = 100, max_items: int | None = None) -> dict: - """Fetch all pages and return combined {items: [...], count: N}.""" + def paginate( + self, + path: str, + params: dict[str, Any] | None = None, + page_size: int = 100, + max_items: int | None = None, + *, + start_offset: int = 0, + ) -> dict: + """ + Fetch pages and return {items, count}. + + ``count`` is the API total. When the page ceiling (or ``max_items``) stops + the crawl early, ``next_offset`` is set so callers can resume. + """ params = dict(params or {}) params["limit"] = page_size - params["offset"] = 0 - all_items = [] - max_pages = 1000 + params["offset"] = start_offset + all_items: list = [] + total = 0 + truncated = False - for _ in range(max_pages): + for _ in range(_MAX_PAGES): data = self.get(path, params=params) items = data.get("items", []) all_items.extend(items) - total = data.get("count", len(items)) + total = data.get("count", start_offset + len(all_items)) if max_items and len(all_items) >= max_items: all_items = all_items[:max_items] + truncated = start_offset + len(all_items) < total break - if len(all_items) >= total or not items: + if start_offset + len(all_items) >= total or not items: break params["offset"] += page_size + else: + truncated = start_offset + len(all_items) < total - return {"items": all_items, "count": len(all_items)} + result: dict[str, Any] = {"items": all_items, "count": total} + if truncated: + result["next_offset"] = start_offset + len(all_items) + return result def post(self, path: str, json: dict[str, Any] | None = None) -> dict: return self._request("POST", path, json=json) diff --git a/src/dualentry_cli/commands/__init__.py b/src/dualentry_cli/commands/__init__.py index 41f6db1..2d4d23b 100644 --- a/src/dualentry_cli/commands/__init__.py +++ b/src/dualentry_cli/commands/__init__.py @@ -72,15 +72,60 @@ def _build_filter_params( return params +# Map _do_list filter kwargs to CLI flags for the --all resume hint. +_FILTER_CLI_FLAGS = ( + ("search", "--search"), + ("status", "--status"), + ("start_date", "--start-date"), + ("end_date", "--end-date"), + ("company_id", "--company"), + ("customer_id", "--customer"), + ("vendor_id", "--vendor"), +) + + +def _resume_all_command(path: str, next_offset: int, filters: dict) -> str: + """Build a copy-paste dualentry list --all command that continues from next_offset.""" + parts = ["dualentry", *path.split("/"), "list", "--all", "--offset", str(next_offset)] + for key, flag in _FILTER_CLI_FLAGS: + value = filters.get(key) + if value is not None: + parts.extend([flag, str(value)]) + return " ".join(parts) + + +def _warn_all_truncated(path: str, *, fetched_through: int, total: int, next_offset: int, filters: dict) -> None: + """Tell the user --all stopped early and how to continue.""" + from dualentry_cli.client import _MAX_PAGES + + cmd = _resume_all_command(path, next_offset, filters) + typer.secho( + f"Warning: fetched {fetched_through} of {total} items; stopped at the {_MAX_PAGES}-page limit.\nTo continue, re-run with the same filters:\n {cmd}", + fg=typer.colors.YELLOW, + err=True, + ) + + def _do_list(client, path: str, resource: str, *, limit: int, offset: int, all_pages: bool, output: str, **filters): """Shared list logic for all resources.""" params = _build_filter_params(**filters) + next_offset = None if all_pages: - data = client.paginate(f"/{path}/", params=params) + data = client.paginate(f"/{path}/", params=params, start_offset=offset) + next_offset = data.pop("next_offset", None) else: params.update({"limit": limit, "offset": offset}) data = client.get(f"/{path}/", params=params) format_output(data, resource=resource, fmt=output) + # After the table so the resume hint is visible without scrolling up. + if next_offset is not None: + _warn_all_truncated( + path, + fetched_through=next_offset, + total=data.get("count", next_offset), + next_offset=next_offset, + filters=filters, + ) def _load_json_file(file: Path) -> dict: diff --git a/tests/test_client.py b/tests/test_client.py index 18974a7..aa552fa 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -501,3 +501,65 @@ def test_backoff_table_and_retry_count_cannot_drift(self): from dualentry_cli.client import _MAX_RETRIES, _RETRY_DELAYS assert len(_RETRY_DELAYS) == _MAX_RETRIES + + +class TestPaginate: + BASE = "https://api.dualentry.com/public/v2" + + @staticmethod + def _client(): + from dualentry_cli.client import DualEntryClient + + return DualEntryClient(api_url="https://api.dualentry.com", api_key="test_key") + + @respx.mock + def test_complete_crawl_has_no_next_offset(self): + respx.get(f"{self.BASE}/invoices/").mock( + side_effect=[ + httpx.Response(200, json={"items": [{"id": 1}, {"id": 2}], "count": 3}), + httpx.Response(200, json={"items": [{"id": 3}], "count": 3}), + ] + ) + + data = self._client().paginate("/invoices/", page_size=2) + + assert data["items"] == [{"id": 1}, {"id": 2}, {"id": 3}] + assert data["count"] == 3 + assert "next_offset" not in data + + @respx.mock + def test_page_cap_sets_next_offset_and_keeps_api_count(self, monkeypatch): + monkeypatch.setattr("dualentry_cli.client._MAX_PAGES", 2) + + def _page(request: httpx.Request) -> httpx.Response: + offset = int(request.url.params.get("offset", "0")) + return httpx.Response(200, json={"items": [{"id": offset}, {"id": offset + 1}], "count": 10}) + + respx.get(f"{self.BASE}/invoices/").mock(side_effect=_page) + + data = self._client().paginate("/invoices/", page_size=2) + + assert len(data["items"]) == 4 + assert data["count"] == 10 + assert data["next_offset"] == 4 + + @respx.mock + def test_start_offset_is_sent_on_first_request(self): + route = respx.get(f"{self.BASE}/invoices/").mock(return_value=httpx.Response(200, json={"items": [{"id": 5}], "count": 5})) + + data = self._client().paginate("/invoices/", page_size=2, start_offset=4) + + assert data["items"] == [{"id": 5}] + assert data["count"] == 5 + assert "next_offset" not in data + assert route.calls[0].request.url.params["offset"] == "4" + + @respx.mock + def test_max_items_truncation_sets_next_offset(self): + respx.get(f"{self.BASE}/invoices/").mock(return_value=httpx.Response(200, json={"items": [{"id": 1}, {"id": 2}, {"id": 3}], "count": 9})) + + data = self._client().paginate("/invoices/", page_size=3, max_items=2) + + assert data["items"] == [{"id": 1}, {"id": 2}] + assert data["count"] == 9 + assert data["next_offset"] == 2 diff --git a/tests/test_commands.py b/tests/test_commands.py index 40447ba..4929069 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -207,3 +207,37 @@ def test_unmatchable_command_still_shows_help(self): assert result.exit_code == 2 assert "Unknown command 'zzzzzz'" in result.output assert "Did you mean" not in result.output + + +class TestListAllTruncation: + def test_all_passes_offset_to_paginate(self, mock_get_client): + mock_get_client.paginate.return_value = {"items": [], "count": 0} + result = runner.invoke(app, ["invoices", "list", "--all", "--offset", "100"]) + assert result.exit_code == 0 + mock_get_client.paginate.assert_called_once_with("/invoices/", params={}, start_offset=100) + + def test_all_truncation_prints_resume_command(self, mock_get_client): + mock_get_client.paginate.return_value = { + "items": [{"internal_id": 1, "number": 1}], + "count": 250000, + "next_offset": 100000, + } + result = runner.invoke(app, ["invoices", "list", "--all", "--search", "acme", "--company", "9"]) + assert result.exit_code == 0 + assert "fetched 100000 of 250000" in result.output + assert "dualentry invoices list --all --offset 100000" in result.output + assert "--search acme" in result.output + assert "--company 9" in result.output + # Warning must come after the list so users see it without scrolling up. + assert result.output.index("Showing") < result.output.index("fetched 100000 of 250000") + mock_get_client.paginate.assert_called_once_with( + "/invoices/", + params={"search": "acme", "company_id": "9"}, + start_offset=0, + ) + + def test_resume_command_helper_for_nested_path(self): + from dualentry_cli.commands import _resume_all_command + + cmd = _resume_all_command("recurring/invoices", 200, {"status": "posted"}) + assert cmd == "dualentry recurring invoices list --all --offset 200 --status posted" From 317c7998c05c509e01c7433512f206d79eb8de37 Mon Sep 17 00:00:00 2001 From: Warkanlock Date: Tue, 1 Sep 2026 15:17:24 -0400 Subject: [PATCH 2/3] fix(cli): simplify --all truncation after review Keep API total as count. One truncate check after the page cap. Refs EOPS-378. --- src/dualentry_cli/client.py | 13 +++++-------- src/dualentry_cli/commands/__init__.py | 23 +++++++++++------------ tests/test_commands.py | 10 ++++++++++ 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/dualentry_cli/client.py b/src/dualentry_cli/client.py index 48348de..da16788 100644 --- a/src/dualentry_cli/client.py +++ b/src/dualentry_cli/client.py @@ -220,17 +220,17 @@ def paginate( start_offset: int = 0, ) -> dict: """ - Fetch pages and return {items, count}. + Fetch pages and return {items, count, next_offset?}. - ``count`` is the API total. When the page ceiling (or ``max_items``) stops - the crawl early, ``next_offset`` is set so callers can resume. + ``count`` is always the API total, regardless of truncation. + When the page ceiling (or ``max_items``) stops the crawl early, + ``next_offset`` is set so callers can resume. """ params = dict(params or {}) params["limit"] = page_size params["offset"] = start_offset all_items: list = [] total = 0 - truncated = False for _ in range(_MAX_PAGES): data = self.get(path, params=params) @@ -239,16 +239,13 @@ def paginate( total = data.get("count", start_offset + len(all_items)) if max_items and len(all_items) >= max_items: all_items = all_items[:max_items] - truncated = start_offset + len(all_items) < total break if start_offset + len(all_items) >= total or not items: break params["offset"] += page_size - else: - truncated = start_offset + len(all_items) < total result: dict[str, Any] = {"items": all_items, "count": total} - if truncated: + if start_offset + len(all_items) < total: result["next_offset"] = start_offset + len(all_items) return result diff --git a/src/dualentry_cli/commands/__init__.py b/src/dualentry_cli/commands/__init__.py index 2d4d23b..77b7cb9 100644 --- a/src/dualentry_cli/commands/__init__.py +++ b/src/dualentry_cli/commands/__init__.py @@ -9,6 +9,7 @@ import typer from dualentry_cli.cli import HelpfulGroup +from dualentry_cli.client import _MAX_PAGES from dualentry_cli.output import _RECORD_PREFIX, format_output # ── Shared option defaults ────────────────────────────────────────── @@ -73,21 +74,21 @@ def _build_filter_params( # Map _do_list filter kwargs to CLI flags for the --all resume hint. -_FILTER_CLI_FLAGS = ( - ("search", "--search"), - ("status", "--status"), - ("start_date", "--start-date"), - ("end_date", "--end-date"), - ("company_id", "--company"), - ("customer_id", "--customer"), - ("vendor_id", "--vendor"), -) +_FILTER_CLI_FLAGS = { + "search": "--search", + "status": "--status", + "start_date": "--start-date", + "end_date": "--end-date", + "company_id": "--company", + "customer_id": "--customer", + "vendor_id": "--vendor", +} def _resume_all_command(path: str, next_offset: int, filters: dict) -> str: """Build a copy-paste dualentry list --all command that continues from next_offset.""" parts = ["dualentry", *path.split("/"), "list", "--all", "--offset", str(next_offset)] - for key, flag in _FILTER_CLI_FLAGS: + for key, flag in _FILTER_CLI_FLAGS.items(): value = filters.get(key) if value is not None: parts.extend([flag, str(value)]) @@ -96,8 +97,6 @@ def _resume_all_command(path: str, next_offset: int, filters: dict) -> str: def _warn_all_truncated(path: str, *, fetched_through: int, total: int, next_offset: int, filters: dict) -> None: """Tell the user --all stopped early and how to continue.""" - from dualentry_cli.client import _MAX_PAGES - cmd = _resume_all_command(path, next_offset, filters) typer.secho( f"Warning: fetched {fetched_through} of {total} items; stopped at the {_MAX_PAGES}-page limit.\nTo continue, re-run with the same filters:\n {cmd}", diff --git a/tests/test_commands.py b/tests/test_commands.py index 4929069..51cec44 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -241,3 +241,13 @@ def test_resume_command_helper_for_nested_path(self): cmd = _resume_all_command("recurring/invoices", 200, {"status": "posted"}) assert cmd == "dualentry recurring invoices list --all --offset 200 --status posted" + + def test_complete_all_does_not_warn(self, mock_get_client): + mock_get_client.paginate.return_value = { + "items": [{"internal_id": 1, "number": 1}], + "count": 1, + } + result = runner.invoke(app, ["invoices", "list", "--all"]) + assert result.exit_code == 0 + assert "Warning" not in result.output + assert "stopped at the" not in result.output From 41fdfee37dd29e1c4a4e86f12a80d29c1eaca946 Mon Sep 17 00:00:00 2001 From: Warkanlock Date: Tue, 1 Sep 2026 15:22:02 -0400 Subject: [PATCH 3/3] =?UTF-8?q?fix(qa):=20ISSUE-001=20=E2=80=94=20say=20re?= =?UTF-8?q?ached,=20not=20fetched?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resume --all reports the cursor, not this-run size. --- src/dualentry_cli/commands/__init__.py | 2 +- tests/test_commands.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dualentry_cli/commands/__init__.py b/src/dualentry_cli/commands/__init__.py index 77b7cb9..cd4c215 100644 --- a/src/dualentry_cli/commands/__init__.py +++ b/src/dualentry_cli/commands/__init__.py @@ -99,7 +99,7 @@ def _warn_all_truncated(path: str, *, fetched_through: int, total: int, next_off """Tell the user --all stopped early and how to continue.""" cmd = _resume_all_command(path, next_offset, filters) typer.secho( - f"Warning: fetched {fetched_through} of {total} items; stopped at the {_MAX_PAGES}-page limit.\nTo continue, re-run with the same filters:\n {cmd}", + f"Warning: reached {fetched_through} of {total} items; stopped at the {_MAX_PAGES}-page limit.\nTo continue, re-run with the same filters:\n {cmd}", fg=typer.colors.YELLOW, err=True, ) diff --git a/tests/test_commands.py b/tests/test_commands.py index 51cec44..e7abf22 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -224,12 +224,12 @@ def test_all_truncation_prints_resume_command(self, mock_get_client): } result = runner.invoke(app, ["invoices", "list", "--all", "--search", "acme", "--company", "9"]) assert result.exit_code == 0 - assert "fetched 100000 of 250000" in result.output + assert "reached 100000 of 250000" in result.output assert "dualentry invoices list --all --offset 100000" in result.output assert "--search acme" in result.output assert "--company 9" in result.output # Warning must come after the list so users see it without scrolling up. - assert result.output.index("Showing") < result.output.index("fetched 100000 of 250000") + assert result.output.index("Showing") < result.output.index("reached 100000 of 250000") mock_get_client.paginate.assert_called_once_with( "/invoices/", params={"search": "acme", "company_id": "9"},