From a472d24639f94907807b3f9a0eddd0b681490d1c Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 20 Jul 2026 21:18:02 +0200 Subject: [PATCH 1/4] fix: Keep pagination iterators advancing past fully-filtered pages --- src/apify_client/_pagination.py | 31 +++++++----- tests/unit/test_client_pagination.py | 76 ++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 12 deletions(-) diff --git a/src/apify_client/_pagination.py b/src/apify_client/_pagination.py index 3433c242..1e4f71b0 100644 --- a/src/apify_client/_pagination.py +++ b/src/apify_client/_pagination.py @@ -41,8 +41,11 @@ def get_items_iterator( is used for offset bookkeeping (the Apify API's `count` reflects items scanned, which can exceed items returned when filters are applied). - Iteration stops when a page returns no items or when the user-requested `limit` is reached. The `total` field is - intentionally not consulted, because it can change between calls. + Iteration stops when a page scans no items (`count` is `0`, or `items` is empty when `count` is absent) or when the + user-requested `limit` is reached. A page can scan items while returning none — filters like `clean` drop items from + `items` but still count toward `count` — so terminating on scanned rather than returned items keeps the iterator + advancing across fully-filtered pages. The `total` field is intentionally not consulted, because it can change + between calls. Args: callback: Function returning a single page of items. @@ -62,9 +65,10 @@ def get_items_iterator( ) yield from current_page.items - fetched_items += max(getattr(current_page, 'count', 0), len(current_page.items)) + page_scanned = max(getattr(current_page, 'count', 0), len(current_page.items)) + fetched_items += page_scanned - if not current_page.items or (initial_limit and fetched_items >= initial_limit): + if not page_scanned or (initial_limit and fetched_items >= initial_limit): break @@ -92,9 +96,10 @@ async def get_items_iterator_async( for item in current_page.items: yield item - fetched_items += max(getattr(current_page, 'count', 0), len(current_page.items)) + page_scanned = max(getattr(current_page, 'count', 0), len(current_page.items)) + fetched_items += page_scanned - if not current_page.items or (initial_limit and fetched_items >= initial_limit): + if not page_scanned or (initial_limit and fetched_items >= initial_limit): break @@ -124,8 +129,8 @@ def get_cursor_iterator( """Yield individual items from cursor-paginated API responses. Cursor pagination is restricted to the two API responses that expose it: `ListOfKeys` (for key-value store keys) and - `ListOfRequests` (for request queue requests). Iteration ends when a page returns no items, the next cursor is - `None`, or the user-requested `limit` is reached. + `ListOfRequests` (for request queue requests). Iteration ends when a page scans no items, the next cursor is `None`, + or the user-requested `limit` is reached. Args: callback: Function returning a single page of items. Receives `cursor` and `limit` kwargs. @@ -144,12 +149,13 @@ def get_cursor_iterator( ) yield from current_page.items - fetched_items += max(getattr(current_page, 'count', 0), len(current_page.items)) + page_scanned = max(getattr(current_page, 'count', 0), len(current_page.items)) + fetched_items += page_scanned cursor = ( current_page.next_exclusive_start_key if isinstance(current_page, ListOfKeys) else current_page.next_cursor ) - if not current_page.items or cursor is None or (initial_limit and fetched_items >= initial_limit): + if not page_scanned or cursor is None or (initial_limit and fetched_items >= initial_limit): break @@ -189,12 +195,13 @@ async def get_cursor_iterator_async( for item in current_page.items: yield item - fetched_items += max(getattr(current_page, 'count', 0), len(current_page.items)) + page_scanned = max(getattr(current_page, 'count', 0), len(current_page.items)) + fetched_items += page_scanned cursor = ( current_page.next_exclusive_start_key if isinstance(current_page, ListOfKeys) else current_page.next_cursor ) - if not current_page.items or cursor is None or (initial_limit and fetched_items >= initial_limit): + if not page_scanned or cursor is None or (initial_limit and fetched_items >= initial_limit): break diff --git a/tests/unit/test_client_pagination.py b/tests/unit/test_client_pagination.py index 5bf4e4a2..1fd92c1e 100644 --- a/tests/unit/test_client_pagination.py +++ b/tests/unit/test_client_pagination.py @@ -11,6 +11,12 @@ from apify_client import ApifyClient, ApifyClientAsync from apify_client import _models as _models_module +from apify_client._pagination import ( + get_cursor_iterator, + get_cursor_iterator_async, + get_items_iterator, + get_items_iterator_async, +) from apify_client._resource_clients import ( ActorCollectionClient, ActorCollectionClientAsync, @@ -617,3 +623,73 @@ async def test_rq_list_requests_iterable_async( client: RequestQueueClientAsync = _CLIENT_FACTORIES[client_name](_make_async_client(pagination_server)) returned_items = [dict(item) async for item in client.iterate_requests(**inputs)] assert returned_items == expected_items + + +class _FakeOffsetPage: + """Offset-paginated page whose `count` (items scanned) may exceed `len(items)` when filters drop items.""" + + def __init__(self, items: list[dict[str, int]], count: int) -> None: + self.items = items + self.count = count + + +class _FakeCursorPage: + """Cursor-paginated page whose `count` (items scanned) may exceed `len(items)` when filters drop items.""" + + def __init__(self, items: list[dict[str, int]], count: int, next_cursor: str | None) -> None: + self.items = items + self.count = count + self.next_cursor = next_cursor + + +def test_items_iterator_continues_past_fully_filtered_page() -> None: + """A fully-filtered page (`items=[]`, `count>0`) must not stop the offset iterator while more data was scanned.""" + pages = { + 0: _FakeOffsetPage(items=[], count=1000), + 1000: _FakeOffsetPage(items=[{'id': 1}, {'id': 2}], count=2), + } + + def _callback(*, limit: int | None = None, offset: int | None = None) -> _FakeOffsetPage: # noqa: ARG001 + return pages.get(offset or 0, _FakeOffsetPage(items=[], count=0)) + + assert list(get_items_iterator(_callback, chunk_size=1000)) == [{'id': 1}, {'id': 2}] + + +async def test_items_iterator_async_continues_past_fully_filtered_page() -> None: + """A fully-filtered page (`items=[]`, `count>0`) must not stop the async offset iterator while more was scanned.""" + pages = { + 0: _FakeOffsetPage(items=[], count=1000), + 1000: _FakeOffsetPage(items=[{'id': 1}, {'id': 2}], count=2), + } + + async def _callback(*, limit: int | None = None, offset: int | None = None) -> _FakeOffsetPage: # noqa: ARG001 + return pages.get(offset or 0, _FakeOffsetPage(items=[], count=0)) + + assert [item async for item in get_items_iterator_async(_callback, chunk_size=1000)] == [{'id': 1}, {'id': 2}] + + +def test_cursor_iterator_continues_past_fully_filtered_page() -> None: + """A fully-filtered page (`items=[]`, `count>0`) with a live cursor must not stop the cursor iterator.""" + pages = { + None: _FakeCursorPage(items=[], count=1000, next_cursor='c1'), + 'c1': _FakeCursorPage(items=[{'id': 1}, {'id': 2}], count=2, next_cursor=None), + } + + def _callback(*, limit: int | None = None, cursor: str | None = None) -> _FakeCursorPage: # noqa: ARG001 + return pages[cursor] + + assert list(get_cursor_iterator(_callback, chunk_size=1000)) == [{'id': 1}, {'id': 2}] # ty: ignore[no-matching-overload] + + +async def test_cursor_iterator_async_continues_past_fully_filtered_page() -> None: + """A fully-filtered page (`items=[]`, `count>0`) with a live cursor must not stop the async cursor iterator.""" + pages = { + None: _FakeCursorPage(items=[], count=1000, next_cursor='c1'), + 'c1': _FakeCursorPage(items=[{'id': 1}, {'id': 2}], count=2, next_cursor=None), + } + + async def _callback(*, limit: int | None = None, cursor: str | None = None) -> _FakeCursorPage: # noqa: ARG001 + return pages[cursor] + + collected = [item async for item in get_cursor_iterator_async(_callback, chunk_size=1000)] # ty: ignore[no-matching-overload] + assert collected == [{'id': 1}, {'id': 2}] From e81c2a96c893fcaabc13cf811506e3f3cc349d64 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 21 Jul 2026 08:36:07 +0200 Subject: [PATCH 2/4] fix: Stop cursor-paginated iterators on the cursor, not on empty pages --- src/apify_client/_pagination.py | 17 +++++++++-------- tests/unit/test_client_pagination.py | 17 ++++++++--------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/apify_client/_pagination.py b/src/apify_client/_pagination.py index 1e4f71b0..09b29cb5 100644 --- a/src/apify_client/_pagination.py +++ b/src/apify_client/_pagination.py @@ -129,8 +129,11 @@ def get_cursor_iterator( """Yield individual items from cursor-paginated API responses. Cursor pagination is restricted to the two API responses that expose it: `ListOfKeys` (for key-value store keys) and - `ListOfRequests` (for request queue requests). Iteration ends when a page scans no items, the next cursor is `None`, - or the user-requested `limit` is reached. + `ListOfRequests` (for request queue requests). Iteration ends when the next cursor is `None` or the user-requested + `limit` is reached. Emptiness alone does not stop iteration: server-side filters (such as the request-queue state + `filter`) can drop every item on a page while a live cursor still points at more data, so termination relies on the + cursor, not on whether a page returned items. Unlike offset responses, cursor responses expose no scanned-item + `count`, so `count` cannot be used to detect a fully-filtered page here. Args: callback: Function returning a single page of items. Receives `cursor` and `limit` kwargs. @@ -149,13 +152,12 @@ def get_cursor_iterator( ) yield from current_page.items - page_scanned = max(getattr(current_page, 'count', 0), len(current_page.items)) - fetched_items += page_scanned + fetched_items += len(current_page.items) cursor = ( current_page.next_exclusive_start_key if isinstance(current_page, ListOfKeys) else current_page.next_cursor ) - if not page_scanned or cursor is None or (initial_limit and fetched_items >= initial_limit): + if cursor is None or (initial_limit and fetched_items >= initial_limit): break @@ -195,13 +197,12 @@ async def get_cursor_iterator_async( for item in current_page.items: yield item - page_scanned = max(getattr(current_page, 'count', 0), len(current_page.items)) - fetched_items += page_scanned + fetched_items += len(current_page.items) cursor = ( current_page.next_exclusive_start_key if isinstance(current_page, ListOfKeys) else current_page.next_cursor ) - if not page_scanned or cursor is None or (initial_limit and fetched_items >= initial_limit): + if cursor is None or (initial_limit and fetched_items >= initial_limit): break diff --git a/tests/unit/test_client_pagination.py b/tests/unit/test_client_pagination.py index 1fd92c1e..a8fba574 100644 --- a/tests/unit/test_client_pagination.py +++ b/tests/unit/test_client_pagination.py @@ -634,11 +634,10 @@ def __init__(self, items: list[dict[str, int]], count: int) -> None: class _FakeCursorPage: - """Cursor-paginated page whose `count` (items scanned) may exceed `len(items)` when filters drop items.""" + """Cursor-paginated page mirroring `ListOfRequests`: no scanned-`count`, so a filtered page is just `items=[]`.""" - def __init__(self, items: list[dict[str, int]], count: int, next_cursor: str | None) -> None: + def __init__(self, items: list[dict[str, int]], next_cursor: str | None) -> None: self.items = items - self.count = count self.next_cursor = next_cursor @@ -669,10 +668,10 @@ async def _callback(*, limit: int | None = None, offset: int | None = None) -> _ def test_cursor_iterator_continues_past_fully_filtered_page() -> None: - """A fully-filtered page (`items=[]`, `count>0`) with a live cursor must not stop the cursor iterator.""" + """A fully-filtered page (`items=[]`) with a live cursor must not stop the cursor iterator.""" pages = { - None: _FakeCursorPage(items=[], count=1000, next_cursor='c1'), - 'c1': _FakeCursorPage(items=[{'id': 1}, {'id': 2}], count=2, next_cursor=None), + None: _FakeCursorPage(items=[], next_cursor='c1'), + 'c1': _FakeCursorPage(items=[{'id': 1}, {'id': 2}], next_cursor=None), } def _callback(*, limit: int | None = None, cursor: str | None = None) -> _FakeCursorPage: # noqa: ARG001 @@ -682,10 +681,10 @@ def _callback(*, limit: int | None = None, cursor: str | None = None) -> _FakeCu async def test_cursor_iterator_async_continues_past_fully_filtered_page() -> None: - """A fully-filtered page (`items=[]`, `count>0`) with a live cursor must not stop the async cursor iterator.""" + """A fully-filtered page (`items=[]`) with a live cursor must not stop the async cursor iterator.""" pages = { - None: _FakeCursorPage(items=[], count=1000, next_cursor='c1'), - 'c1': _FakeCursorPage(items=[{'id': 1}, {'id': 2}], count=2, next_cursor=None), + None: _FakeCursorPage(items=[], next_cursor='c1'), + 'c1': _FakeCursorPage(items=[{'id': 1}, {'id': 2}], next_cursor=None), } async def _callback(*, limit: int | None = None, cursor: str | None = None) -> _FakeCursorPage: # noqa: ARG001 From 964fe74fe2914000c52920cb43cf5eff482ef2ce Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 21 Jul 2026 09:33:53 +0200 Subject: [PATCH 3/4] style: Tidy pagination test helper names and normalize non-ASCII chars --- src/apify_client/_pagination.py | 6 ++-- tests/unit/test_client_pagination.py | 52 ++++++++++++++-------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/apify_client/_pagination.py b/src/apify_client/_pagination.py index 09b29cb5..4facbe58 100644 --- a/src/apify_client/_pagination.py +++ b/src/apify_client/_pagination.py @@ -19,7 +19,7 @@ class HasItems(Protocol[T]): """Structural contract for a single page of results from a paginated API endpoint. - Implementations must expose `items`. They may optionally expose `count` — the number of items scanned by the API for + Implementations must expose `items`. They may optionally expose `count` - the number of items scanned by the API for this page, which can exceed `len(items)` when filters drop items from the response. The iterator helpers consult `count` opportunistically via `getattr` for offset bookkeeping and fall back to `len(items)` when it is absent. """ @@ -42,8 +42,8 @@ def get_items_iterator( filters are applied). Iteration stops when a page scans no items (`count` is `0`, or `items` is empty when `count` is absent) or when the - user-requested `limit` is reached. A page can scan items while returning none — filters like `clean` drop items from - `items` but still count toward `count` — so terminating on scanned rather than returned items keeps the iterator + user-requested `limit` is reached. A page can scan items while returning none - filters like `clean` drop items from + `items` but still count toward `count` - so terminating on scanned rather than returned items keeps the iterator advancing across fully-filtered pages. The `total` field is intentionally not consulted, because it can change between calls. diff --git a/tests/unit/test_client_pagination.py b/tests/unit/test_client_pagination.py index a8fba574..84286647 100644 --- a/tests/unit/test_client_pagination.py +++ b/tests/unit/test_client_pagination.py @@ -119,7 +119,7 @@ ) # Outer wrappers that embed a relaxed list model via `.data`. Their compiled schema pins the inner's schema at -# construction time, so they need a forced rebuild to pick up the relaxation. The wrappers themselves are not mutated — +# construction time, so they need a forced rebuild to pick up the relaxation. The wrappers themselves are not mutated - # their own field annotations stay as-is. _REBUILT_RESPONSE_WRAPPERS = ( 'ListOfActorsInStoreResponse', @@ -144,9 +144,9 @@ def _relax_item_validation() -> Any: """Relax only the element type of `items` on paginated list models for the test run. Pagination tests feed synthetic `{'id': N}` items that don't satisfy the real API schemas (`ActorShort`, - `BuildShort`, `Request`, `EnvVar`, …). Instead of bypassing validation wholesale, each inner `ListOf*` model has its - `items` field swapped to `list[dict]` and rebuilt. Outer `.data` wrapping and every pagination-metadata field remain - validated. + `BuildShort`, `Request`, `EnvVar`, ...). Instead of bypassing validation wholesale, each inner `ListOf*` model + has its `items` field swapped to `list[dict]` and rebuilt. Outer `.data` wrapping and every pagination-metadata + field remain validated. """ relaxed_field = FieldInfo.from_annotation(list[dict]) originals: dict[type[BaseModel], FieldInfo] = {} @@ -177,7 +177,7 @@ def create_items(start: int, end: int, step: int | None = None) -> list[dict[str def _is_true(value: str | None) -> bool: - """Match the `'true'` wire form produced by the client's bool→string serialization.""" + """Match the `'true'` wire form produced by the client's bool->string serialization.""" return value == 'true' @@ -241,7 +241,7 @@ def _handle_cursor_pagination(request: Request) -> Response: """Serve a cursor-paginated Apify API response for KVS keys and RQ requests. Holds 2500 synthetic items whose integer `id` equals their position. Each page is capped at 1000 items. KVS uses - `exclusiveStartKey`; RQ uses the opaque `cursor`. Both values encode the last-seen item id as a string — the + `exclusiveStartKey`; RQ uses the opaque `cursor`. Both values encode the last-seen item id as a string - the next page starts at id + 1. """ params = request.args @@ -625,7 +625,7 @@ async def test_rq_list_requests_iterable_async( assert returned_items == expected_items -class _FakeOffsetPage: +class FakeOffsetPage: """Offset-paginated page whose `count` (items scanned) may exceed `len(items)` when filters drop items.""" def __init__(self, items: list[dict[str, int]], count: int) -> None: @@ -633,7 +633,7 @@ def __init__(self, items: list[dict[str, int]], count: int) -> None: self.count = count -class _FakeCursorPage: +class FakeCursorPage: """Cursor-paginated page mirroring `ListOfRequests`: no scanned-`count`, so a filtered page is just `items=[]`.""" def __init__(self, items: list[dict[str, int]], next_cursor: str | None) -> None: @@ -644,51 +644,51 @@ def __init__(self, items: list[dict[str, int]], next_cursor: str | None) -> None def test_items_iterator_continues_past_fully_filtered_page() -> None: """A fully-filtered page (`items=[]`, `count>0`) must not stop the offset iterator while more data was scanned.""" pages = { - 0: _FakeOffsetPage(items=[], count=1000), - 1000: _FakeOffsetPage(items=[{'id': 1}, {'id': 2}], count=2), + 0: FakeOffsetPage(items=[], count=1000), + 1000: FakeOffsetPage(items=[{'id': 1}, {'id': 2}], count=2), } - def _callback(*, limit: int | None = None, offset: int | None = None) -> _FakeOffsetPage: # noqa: ARG001 - return pages.get(offset or 0, _FakeOffsetPage(items=[], count=0)) + def callback(*, limit: int | None = None, offset: int | None = None) -> FakeOffsetPage: # noqa: ARG001 + return pages.get(offset or 0, FakeOffsetPage(items=[], count=0)) - assert list(get_items_iterator(_callback, chunk_size=1000)) == [{'id': 1}, {'id': 2}] + assert list(get_items_iterator(callback, chunk_size=1000)) == [{'id': 1}, {'id': 2}] async def test_items_iterator_async_continues_past_fully_filtered_page() -> None: """A fully-filtered page (`items=[]`, `count>0`) must not stop the async offset iterator while more was scanned.""" pages = { - 0: _FakeOffsetPage(items=[], count=1000), - 1000: _FakeOffsetPage(items=[{'id': 1}, {'id': 2}], count=2), + 0: FakeOffsetPage(items=[], count=1000), + 1000: FakeOffsetPage(items=[{'id': 1}, {'id': 2}], count=2), } - async def _callback(*, limit: int | None = None, offset: int | None = None) -> _FakeOffsetPage: # noqa: ARG001 - return pages.get(offset or 0, _FakeOffsetPage(items=[], count=0)) + async def callback(*, limit: int | None = None, offset: int | None = None) -> FakeOffsetPage: # noqa: ARG001 + return pages.get(offset or 0, FakeOffsetPage(items=[], count=0)) - assert [item async for item in get_items_iterator_async(_callback, chunk_size=1000)] == [{'id': 1}, {'id': 2}] + assert [item async for item in get_items_iterator_async(callback, chunk_size=1000)] == [{'id': 1}, {'id': 2}] def test_cursor_iterator_continues_past_fully_filtered_page() -> None: """A fully-filtered page (`items=[]`) with a live cursor must not stop the cursor iterator.""" pages = { - None: _FakeCursorPage(items=[], next_cursor='c1'), - 'c1': _FakeCursorPage(items=[{'id': 1}, {'id': 2}], next_cursor=None), + None: FakeCursorPage(items=[], next_cursor='c1'), + 'c1': FakeCursorPage(items=[{'id': 1}, {'id': 2}], next_cursor=None), } - def _callback(*, limit: int | None = None, cursor: str | None = None) -> _FakeCursorPage: # noqa: ARG001 + def callback(*, limit: int | None = None, cursor: str | None = None) -> FakeCursorPage: # noqa: ARG001 return pages[cursor] - assert list(get_cursor_iterator(_callback, chunk_size=1000)) == [{'id': 1}, {'id': 2}] # ty: ignore[no-matching-overload] + assert list(get_cursor_iterator(callback, chunk_size=1000)) == [{'id': 1}, {'id': 2}] # ty: ignore[no-matching-overload] async def test_cursor_iterator_async_continues_past_fully_filtered_page() -> None: """A fully-filtered page (`items=[]`) with a live cursor must not stop the async cursor iterator.""" pages = { - None: _FakeCursorPage(items=[], next_cursor='c1'), - 'c1': _FakeCursorPage(items=[{'id': 1}, {'id': 2}], next_cursor=None), + None: FakeCursorPage(items=[], next_cursor='c1'), + 'c1': FakeCursorPage(items=[{'id': 1}, {'id': 2}], next_cursor=None), } - async def _callback(*, limit: int | None = None, cursor: str | None = None) -> _FakeCursorPage: # noqa: ARG001 + async def callback(*, limit: int | None = None, cursor: str | None = None) -> FakeCursorPage: # noqa: ARG001 return pages[cursor] - collected = [item async for item in get_cursor_iterator_async(_callback, chunk_size=1000)] # ty: ignore[no-matching-overload] + collected = [item async for item in get_cursor_iterator_async(callback, chunk_size=1000)] # ty: ignore[no-matching-overload] assert collected == [{'id': 1}, {'id': 2}] From 90ad29562ce58b80488869d78036e06084119a07 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 21 Jul 2026 10:07:04 +0200 Subject: [PATCH 4/4] test: Drop noqa/type-ignore from pagination iterator tests --- tests/unit/test_client_pagination.py | 29 +++++++++++----------------- 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/tests/unit/test_client_pagination.py b/tests/unit/test_client_pagination.py index 84286647..ac9d92b5 100644 --- a/tests/unit/test_client_pagination.py +++ b/tests/unit/test_client_pagination.py @@ -11,6 +11,7 @@ from apify_client import ApifyClient, ApifyClientAsync from apify_client import _models as _models_module +from apify_client._models import ListOfRequests from apify_client._pagination import ( get_cursor_iterator, get_cursor_iterator_async, @@ -633,14 +634,6 @@ def __init__(self, items: list[dict[str, int]], count: int) -> None: self.count = count -class FakeCursorPage: - """Cursor-paginated page mirroring `ListOfRequests`: no scanned-`count`, so a filtered page is just `items=[]`.""" - - def __init__(self, items: list[dict[str, int]], next_cursor: str | None) -> None: - self.items = items - self.next_cursor = next_cursor - - def test_items_iterator_continues_past_fully_filtered_page() -> None: """A fully-filtered page (`items=[]`, `count>0`) must not stop the offset iterator while more data was scanned.""" pages = { @@ -648,7 +641,7 @@ def test_items_iterator_continues_past_fully_filtered_page() -> None: 1000: FakeOffsetPage(items=[{'id': 1}, {'id': 2}], count=2), } - def callback(*, limit: int | None = None, offset: int | None = None) -> FakeOffsetPage: # noqa: ARG001 + def callback(*, offset: int | None = None, **_kwargs: object) -> FakeOffsetPage: return pages.get(offset or 0, FakeOffsetPage(items=[], count=0)) assert list(get_items_iterator(callback, chunk_size=1000)) == [{'id': 1}, {'id': 2}] @@ -661,7 +654,7 @@ async def test_items_iterator_async_continues_past_fully_filtered_page() -> None 1000: FakeOffsetPage(items=[{'id': 1}, {'id': 2}], count=2), } - async def callback(*, limit: int | None = None, offset: int | None = None) -> FakeOffsetPage: # noqa: ARG001 + async def callback(*, offset: int | None = None, **_kwargs: object) -> FakeOffsetPage: return pages.get(offset or 0, FakeOffsetPage(items=[], count=0)) assert [item async for item in get_items_iterator_async(callback, chunk_size=1000)] == [{'id': 1}, {'id': 2}] @@ -670,25 +663,25 @@ async def callback(*, limit: int | None = None, offset: int | None = None) -> Fa def test_cursor_iterator_continues_past_fully_filtered_page() -> None: """A fully-filtered page (`items=[]`) with a live cursor must not stop the cursor iterator.""" pages = { - None: FakeCursorPage(items=[], next_cursor='c1'), - 'c1': FakeCursorPage(items=[{'id': 1}, {'id': 2}], next_cursor=None), + None: ListOfRequests(items=[], limit=1000, next_cursor='c1'), + 'c1': ListOfRequests(items=[{'id': 1}, {'id': 2}], limit=1000, next_cursor=None), } - def callback(*, limit: int | None = None, cursor: str | None = None) -> FakeCursorPage: # noqa: ARG001 + def callback(*, cursor: str | None = None, **_kwargs: object) -> ListOfRequests: return pages[cursor] - assert list(get_cursor_iterator(callback, chunk_size=1000)) == [{'id': 1}, {'id': 2}] # ty: ignore[no-matching-overload] + assert list(get_cursor_iterator(callback, chunk_size=1000)) == [{'id': 1}, {'id': 2}] async def test_cursor_iterator_async_continues_past_fully_filtered_page() -> None: """A fully-filtered page (`items=[]`) with a live cursor must not stop the async cursor iterator.""" pages = { - None: FakeCursorPage(items=[], next_cursor='c1'), - 'c1': FakeCursorPage(items=[{'id': 1}, {'id': 2}], next_cursor=None), + None: ListOfRequests(items=[], limit=1000, next_cursor='c1'), + 'c1': ListOfRequests(items=[{'id': 1}, {'id': 2}], limit=1000, next_cursor=None), } - async def callback(*, limit: int | None = None, cursor: str | None = None) -> FakeCursorPage: # noqa: ARG001 + async def callback(*, cursor: str | None = None, **_kwargs: object) -> ListOfRequests: return pages[cursor] - collected = [item async for item in get_cursor_iterator_async(callback, chunk_size=1000)] # ty: ignore[no-matching-overload] + collected = [item async for item in get_cursor_iterator_async(callback, chunk_size=1000)] assert collected == [{'id': 1}, {'id': 2}]