diff --git a/seam/paginator.py b/seam/paginator.py index 6f2ad9d..635155b 100644 --- a/seam/paginator.py +++ b/seam/paginator.py @@ -87,28 +87,32 @@ def next_page( def flatten_to_list(self) -> List[Any]: """Fetches all pages and returns all items as a single list.""" all_items = [] - current_items, pagination = self.first_page() - - if current_items: + for current_items in self._walk(): all_items.extend(current_items) - while pagination and pagination.has_next_page and pagination.next_page_cursor: - current_items, pagination = self.next_page(pagination.next_page_cursor) - if current_items: - all_items.extend(current_items) - return all_items def flatten(self) -> Generator[Any, None, None]: """Fetches all pages and yields items one by one using a generator.""" - current_items, pagination = self.first_page() - if current_items: + for current_items in self._walk(): yield from current_items + def _walk(self) -> Generator[List[Any], None, None]: + """Yields each page once, stopping if the server repeats a cursor.""" + current_items, pagination = self.first_page() + yield current_items or [] + + seen_cursors = set() + while pagination and pagination.has_next_page and pagination.next_page_cursor: - current_items, pagination = self.next_page(pagination.next_page_cursor) - if current_items: - yield from current_items + cursor = pagination.next_page_cursor + + if cursor in seen_cursors: + return + seen_cursors.add(cursor) + + current_items, pagination = self.next_page(cursor) + yield current_items or [] class AsyncSeamPaginator: @@ -161,29 +165,30 @@ async def next_page( async def flatten_to_list(self) -> List[Any]: """Fetches all pages and returns all items as a single list.""" all_items = [] - current_items, pagination = await self.first_page() - - if current_items: + async for current_items in self._walk(): all_items.extend(current_items) - while pagination and pagination.has_next_page and pagination.next_page_cursor: - current_items, pagination = await self.next_page( - pagination.next_page_cursor - ) - if current_items: - all_items.extend(current_items) - return all_items async def flatten(self) -> AsyncGenerator[Any, None]: """Fetches all pages and yields items one by one using an async generator.""" + async for current_items in self._walk(): + for item in current_items: + yield item + + async def _walk(self) -> AsyncGenerator[List[Any], None]: + """Yields each page once, stopping if the server repeats a cursor.""" current_items, pagination = await self.first_page() - for item in current_items or []: - yield item + yield current_items or [] + + seen_cursors = set() while pagination and pagination.has_next_page and pagination.next_page_cursor: - current_items, pagination = await self.next_page( - pagination.next_page_cursor - ) - for item in current_items or []: - yield item + cursor = pagination.next_page_cursor + + if cursor in seen_cursors: + return + seen_cursors.add(cursor) + + current_items, pagination = await self.next_page(cursor) + yield current_items or [] diff --git a/test/paginator_isolation_test.py b/test/paginator_isolation_test.py index fa6d806..47922d2 100644 --- a/test/paginator_isolation_test.py +++ b/test/paginator_isolation_test.py @@ -92,3 +92,48 @@ async def test_a_missing_pagination_envelope_raises_async(recording_server): "pagination object", ): await paginator.first_page() + + +PINNED_CURSOR_PAGE = { + "devices": [{"device_id": "33333333-3333-3333-3333-333333333333"}], + "pagination": { + "has_next_page": True, + "next_page_cursor": "pinned-cursor", + "next_page_url": "https://example.com/devices/list?page_cursor=pinned-cursor", + }, +} + + +def test_paginator_stops_when_a_cursor_repeats(recording_server): + with recording_server([(200, PINNED_CURSOR_PAGE)]) as (endpoint, requests): + seam = Seam.from_api_key("seam_apikey_token", endpoint=endpoint) + paginator = seam.create_paginator(seam.devices.list) + + devices = paginator.flatten_to_list() + + # The server pins one cursor, so the paginator fetches the first + # page, follows the cursor once, sees it repeat, and stops. + assert len(requests) == 2 + assert len(devices) == 2 + + +def test_paginator_flatten_stops_when_a_cursor_repeats(recording_server): + with recording_server([(200, PINNED_CURSOR_PAGE)]) as (endpoint, requests): + seam = Seam.from_api_key("seam_apikey_token", endpoint=endpoint) + paginator = seam.create_paginator(seam.devices.list) + + devices = list(paginator.flatten()) + + assert len(requests) == 2 + assert len(devices) == 2 + + +async def test_paginator_stops_when_a_cursor_repeats_async(recording_server): + with recording_server([(200, PINNED_CURSOR_PAGE)]) as (endpoint, requests): + async with AsyncSeam(api_key="seam_apikey_token", endpoint=endpoint) as seam: + paginator = seam.create_paginator(seam.devices.list) + + devices = await paginator.flatten_to_list() + + assert len(requests) == 2 + assert len(devices) == 2