From bc092b861c424dd738f32e907bab4265d5cde832 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 20 Jul 2026 21:18:03 +0200 Subject: [PATCH 1/5] fix: Read file-like values before upload in the impit transport --- src/apify_client/_utils/encoding.py | 13 ++++--- tests/unit/test_key_value_store.py | 60 +++++++++++++++++++++++++++++ tests/unit/test_utils.py | 12 +++++- 3 files changed, 77 insertions(+), 8 deletions(-) create mode 100644 tests/unit/test_key_value_store.py diff --git a/src/apify_client/_utils/encoding.py b/src/apify_client/_utils/encoding.py index b4adf385..27eda1a9 100644 --- a/src/apify_client/_utils/encoding.py +++ b/src/apify_client/_utils/encoding.py @@ -22,19 +22,20 @@ def encode_key_value_store_record_value(value: Any, *, content_type: str | None Returns: A tuple of (encoded_value, content_type). """ + # Read file-like values into memory; the underlying HTTP transport only accepts bytes-like bodies, + # so a file object would otherwise reach it unread and raise a raw `TypeError`. + if isinstance(value, io.IOBase): + value = value.read() + if not content_type: - if isinstance(value, (bytes, bytearray, io.IOBase)): + if isinstance(value, (bytes, bytearray)): content_type = 'application/octet-stream' elif isinstance(value, str): content_type = 'text/plain; charset=utf-8' else: content_type = 'application/json; charset=utf-8' - if ( - 'application/json' in content_type - and not isinstance(value, (bytes, bytearray, io.IOBase)) - and not isinstance(value, str) - ): + if 'application/json' in content_type and not isinstance(value, (bytes, bytearray, str)): # Don't use indentation to reduce size. value = json.dumps( value, diff --git a/tests/unit/test_key_value_store.py b/tests/unit/test_key_value_store.py new file mode 100644 index 00000000..46dc209f --- /dev/null +++ b/tests/unit/test_key_value_store.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import gzip +import io +from typing import TYPE_CHECKING + +from werkzeug import Request, Response + +from apify_client import ApifyClient, ApifyClientAsync + +if TYPE_CHECKING: + from pytest_httpserver import HTTPServer + +_MOCKED_KVS_ID = 'test_kvs_id' +_RECORD_PATH = f'/v2/key-value-stores/{_MOCKED_KVS_ID}/records/f' + + +def _decode_body(request: Request) -> bytes: + raw = request.get_data() + return gzip.decompress(raw) if request.headers.get('Content-Encoding') == 'gzip' else raw + + +def test_set_record_reads_file_like_value_sync(httpserver: HTTPServer) -> None: + """Regression test: a file-like value is read and its bytes are uploaded, not passed through unread.""" + captured_requests: list[Request] = [] + + def capture_request(request: Request) -> Response: + captured_requests.append(request) + return Response(status=201) + + httpserver.expect_request(_RECORD_PATH, method='PUT').respond_with_handler(capture_request) + + api_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClient(token='test_token', api_url=api_url) + + client.key_value_store(_MOCKED_KVS_ID).set_record('f', io.BytesIO(b'buffer data')) + + assert len(captured_requests) == 1 + assert _decode_body(captured_requests[0]) == b'buffer data' + assert captured_requests[0].headers['content-type'] == 'application/octet-stream' + + +async def test_set_record_reads_file_like_value_async(httpserver: HTTPServer) -> None: + """Regression test: a file-like value is read and its bytes are uploaded, not passed through unread.""" + captured_requests: list[Request] = [] + + def capture_request(request: Request) -> Response: + captured_requests.append(request) + return Response(status=201) + + httpserver.expect_request(_RECORD_PATH, method='PUT').respond_with_handler(capture_request) + + api_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClientAsync(token='test_token', api_url=api_url) + + await client.key_value_store(_MOCKED_KVS_ID).set_record('f', io.BytesIO(b'buffer data')) + + assert len(captured_requests) == 1 + assert _decode_body(captured_requests[0]) == b'buffer data' + assert captured_requests[0].headers['content-type'] == 'application/octet-stream' diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index a52d3fb4..757437fd 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -249,13 +249,21 @@ def test_encode_key_value_store_record_value( def test_encode_key_value_store_record_value_bytesio() -> None: - """Test that BytesIO is encoded as octet-stream.""" + """Test that BytesIO is read into bytes and encoded as octet-stream.""" buffer = io.BytesIO(b'buffer data') value, content_type = encode_key_value_store_record_value(buffer) - assert value == buffer + assert value == b'buffer data' assert content_type == 'application/octet-stream' +def test_encode_key_value_store_record_value_stringio() -> None: + """Test that StringIO is read into text and encoded as text/plain.""" + buffer = io.StringIO('buffer data') + value, content_type = encode_key_value_store_record_value(buffer) + assert value == 'buffer data' + assert content_type == 'text/plain; charset=utf-8' + + def test_response_to_dict() -> None: """Test parsing response as dictionary.""" mock_response = Mock() From 4f1e9ff3267a82a8dc9acc0100f007fbf1dab7c3 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 21 Jul 2026 10:14:35 +0200 Subject: [PATCH 2/5] fix: Read duck-typed file-like values and encode KVS records off the event loop --- .../_resource_clients/key_value_store.py | 7 ++++++- src/apify_client/_utils/encoding.py | 10 ++++++---- tests/unit/test_key_value_store.py | 20 +++++++++++++++++++ tests/unit/test_utils.py | 12 +++++++++++ 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/src/apify_client/_resource_clients/key_value_store.py b/src/apify_client/_resource_clients/key_value_store.py index 391c0969..6ca19184 100644 --- a/src/apify_client/_resource_clients/key_value_store.py +++ b/src/apify_client/_resource_clients/key_value_store.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import re from contextlib import asynccontextmanager, contextmanager from http import HTTPStatus @@ -801,7 +802,11 @@ async def set_record( content_type: The content type of the saved value. timeout: Timeout for the API HTTP request. """ - value, content_type = encode_key_value_store_record_value(value, content_type=content_type) + # Encoding reads file-like values and may serialize large payloads, which is blocking; offload it to a + # worker thread so it does not stall the event loop (mirrors the transport's own body-prep offload). + value, content_type = await asyncio.to_thread( + encode_key_value_store_record_value, value, content_type=content_type + ) headers = {'content-type': content_type} diff --git a/src/apify_client/_utils/encoding.py b/src/apify_client/_utils/encoding.py index 27eda1a9..d3ac20a6 100644 --- a/src/apify_client/_utils/encoding.py +++ b/src/apify_client/_utils/encoding.py @@ -1,6 +1,5 @@ from __future__ import annotations -import io import json from base64 import b64encode from functools import cache @@ -23,9 +22,12 @@ def encode_key_value_store_record_value(value: Any, *, content_type: str | None A tuple of (encoded_value, content_type). """ # Read file-like values into memory; the underlying HTTP transport only accepts bytes-like bodies, - # so a file object would otherwise reach it unread and raise a raw `TypeError`. - if isinstance(value, io.IOBase): - value = value.read() + # so a file object would otherwise reach it unread and raise a raw `TypeError`. Detect them by a + # callable `read` rather than `io.IOBase` so duck-typed file-likes (upload wrappers, raw streams) + # are read too, instead of falling through to JSON serialization. + read = getattr(value, 'read', None) + if callable(read): + value = read() if not content_type: if isinstance(value, (bytes, bytearray)): diff --git a/tests/unit/test_key_value_store.py b/tests/unit/test_key_value_store.py index 46dc209f..ef0d2aae 100644 --- a/tests/unit/test_key_value_store.py +++ b/tests/unit/test_key_value_store.py @@ -58,3 +58,23 @@ def capture_request(request: Request) -> Response: assert len(captured_requests) == 1 assert _decode_body(captured_requests[0]) == b'buffer data' assert captured_requests[0].headers['content-type'] == 'application/octet-stream' + + +def test_set_record_reads_stringio_value_sync(httpserver: HTTPServer) -> None: + """Regression test: a text file-like value is read and uploaded as text/plain through the HTTP stack.""" + captured_requests: list[Request] = [] + + def capture_request(request: Request) -> Response: + captured_requests.append(request) + return Response(status=201) + + httpserver.expect_request(_RECORD_PATH, method='PUT').respond_with_handler(capture_request) + + api_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClient(token='test_token', api_url=api_url) + + client.key_value_store(_MOCKED_KVS_ID).set_record('f', io.StringIO('buffer data')) + + assert len(captured_requests) == 1 + assert _decode_body(captured_requests[0]) == b'buffer data' + assert captured_requests[0].headers['content-type'] == 'text/plain; charset=utf-8' diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 757437fd..88963b1f 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -264,6 +264,18 @@ def test_encode_key_value_store_record_value_stringio() -> None: assert content_type == 'text/plain; charset=utf-8' +def test_encode_key_value_store_record_value_duck_typed_file_like() -> None: + """Test that a duck-typed file-like value (a callable `read`, not an `io.IOBase`) is read into bytes.""" + + class _Reader: + def read(self) -> bytes: + return b'buffer data' + + value, content_type = encode_key_value_store_record_value(_Reader()) + assert value == b'buffer data' + assert content_type == 'application/octet-stream' + + def test_response_to_dict() -> None: """Test parsing response as dictionary.""" mock_response = Mock() From c0f62ed7fc15136b3836d1f96a0865101d2ebd20 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 21 Jul 2026 10:56:46 +0200 Subject: [PATCH 3/5] style: Drop underscore prefix from test helpers and tighten comments --- src/apify_client/_resource_clients/key_value_store.py | 3 +-- src/apify_client/_utils/encoding.py | 6 ++---- tests/unit/test_key_value_store.py | 8 ++++---- tests/unit/test_utils.py | 4 ++-- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/apify_client/_resource_clients/key_value_store.py b/src/apify_client/_resource_clients/key_value_store.py index 6ca19184..13af637f 100644 --- a/src/apify_client/_resource_clients/key_value_store.py +++ b/src/apify_client/_resource_clients/key_value_store.py @@ -802,8 +802,7 @@ async def set_record( content_type: The content type of the saved value. timeout: Timeout for the API HTTP request. """ - # Encoding reads file-like values and may serialize large payloads, which is blocking; offload it to a - # worker thread so it does not stall the event loop (mirrors the transport's own body-prep offload). + # Encoding may read a file or serialize a large payload (blocking), so run it off the event loop. value, content_type = await asyncio.to_thread( encode_key_value_store_record_value, value, content_type=content_type ) diff --git a/src/apify_client/_utils/encoding.py b/src/apify_client/_utils/encoding.py index d3ac20a6..3f14fef8 100644 --- a/src/apify_client/_utils/encoding.py +++ b/src/apify_client/_utils/encoding.py @@ -21,10 +21,8 @@ def encode_key_value_store_record_value(value: Any, *, content_type: str | None Returns: A tuple of (encoded_value, content_type). """ - # Read file-like values into memory; the underlying HTTP transport only accepts bytes-like bodies, - # so a file object would otherwise reach it unread and raise a raw `TypeError`. Detect them by a - # callable `read` rather than `io.IOBase` so duck-typed file-likes (upload wrappers, raw streams) - # are read too, instead of falling through to JSON serialization. + # Read file-like values into memory; the transport only accepts bytes-like bodies. Detect them by a + # callable `read` (not `io.IOBase`) so duck-typed file-likes are read, not JSON-serialized. read = getattr(value, 'read', None) if callable(read): value = read() diff --git a/tests/unit/test_key_value_store.py b/tests/unit/test_key_value_store.py index ef0d2aae..a5abcd83 100644 --- a/tests/unit/test_key_value_store.py +++ b/tests/unit/test_key_value_store.py @@ -15,7 +15,7 @@ _RECORD_PATH = f'/v2/key-value-stores/{_MOCKED_KVS_ID}/records/f' -def _decode_body(request: Request) -> bytes: +def decode_body(request: Request) -> bytes: raw = request.get_data() return gzip.decompress(raw) if request.headers.get('Content-Encoding') == 'gzip' else raw @@ -36,7 +36,7 @@ def capture_request(request: Request) -> Response: client.key_value_store(_MOCKED_KVS_ID).set_record('f', io.BytesIO(b'buffer data')) assert len(captured_requests) == 1 - assert _decode_body(captured_requests[0]) == b'buffer data' + assert decode_body(captured_requests[0]) == b'buffer data' assert captured_requests[0].headers['content-type'] == 'application/octet-stream' @@ -56,7 +56,7 @@ def capture_request(request: Request) -> Response: await client.key_value_store(_MOCKED_KVS_ID).set_record('f', io.BytesIO(b'buffer data')) assert len(captured_requests) == 1 - assert _decode_body(captured_requests[0]) == b'buffer data' + assert decode_body(captured_requests[0]) == b'buffer data' assert captured_requests[0].headers['content-type'] == 'application/octet-stream' @@ -76,5 +76,5 @@ def capture_request(request: Request) -> Response: client.key_value_store(_MOCKED_KVS_ID).set_record('f', io.StringIO('buffer data')) assert len(captured_requests) == 1 - assert _decode_body(captured_requests[0]) == b'buffer data' + assert decode_body(captured_requests[0]) == b'buffer data' assert captured_requests[0].headers['content-type'] == 'text/plain; charset=utf-8' diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 88963b1f..803e11e2 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -267,11 +267,11 @@ def test_encode_key_value_store_record_value_stringio() -> None: def test_encode_key_value_store_record_value_duck_typed_file_like() -> None: """Test that a duck-typed file-like value (a callable `read`, not an `io.IOBase`) is read into bytes.""" - class _Reader: + class Reader: def read(self) -> bytes: return b'buffer data' - value, content_type = encode_key_value_store_record_value(_Reader()) + value, content_type = encode_key_value_store_record_value(Reader()) assert value == b'buffer data' assert content_type == 'application/octet-stream' From 6010aa2eed488ae5ffe0a7ac4d25735818f4ba85 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 21 Jul 2026 11:14:16 +0200 Subject: [PATCH 4/5] test: Parametrize key-value store record upload tests over gzip and brotli --- tests/unit/test_key_value_store.py | 47 +++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_key_value_store.py b/tests/unit/test_key_value_store.py index a5abcd83..cbe05568 100644 --- a/tests/unit/test_key_value_store.py +++ b/tests/unit/test_key_value_store.py @@ -4,6 +4,8 @@ import io from typing import TYPE_CHECKING +import brotli +import pytest from werkzeug import Request, Response from apify_client import ApifyClient, ApifyClientAsync @@ -11,17 +13,39 @@ if TYPE_CHECKING: from pytest_httpserver import HTTPServer + from apify_client.types import HttpCompressionAlgorithm + _MOCKED_KVS_ID = 'test_kvs_id' _RECORD_PATH = f'/v2/key-value-stores/{_MOCKED_KVS_ID}/records/f' +@pytest.fixture( + params=[ + pytest.param(('gzip', 'gzip'), id='gzip'), + pytest.param(('brotli', 'br'), id='brotli'), + ] +) +def compression_case(request: pytest.FixtureRequest) -> tuple[HttpCompressionAlgorithm, str]: + """Run each test over both supported request-body compression algorithms, as (algorithm, content-encoding).""" + return request.param + + def decode_body(request: Request) -> bytes: + """Decompress a captured request body according to its `Content-Encoding`.""" raw = request.get_data() - return gzip.decompress(raw) if request.headers.get('Content-Encoding') == 'gzip' else raw + encoding = request.headers.get('Content-Encoding') + if encoding == 'gzip': + return gzip.decompress(raw) + if encoding == 'br': + return brotli.decompress(raw) + return raw -def test_set_record_reads_file_like_value_sync(httpserver: HTTPServer) -> None: +def test_set_record_reads_file_like_value_sync( + httpserver: HTTPServer, compression_case: tuple[HttpCompressionAlgorithm, str] +) -> None: """Regression test: a file-like value is read and its bytes are uploaded, not passed through unread.""" + algorithm, content_encoding = compression_case captured_requests: list[Request] = [] def capture_request(request: Request) -> Response: @@ -31,17 +55,21 @@ def capture_request(request: Request) -> Response: httpserver.expect_request(_RECORD_PATH, method='PUT').respond_with_handler(capture_request) api_url = httpserver.url_for('/').removesuffix('/') - client = ApifyClient(token='test_token', api_url=api_url) + client = ApifyClient(token='test_token', api_url=api_url, compression=algorithm) client.key_value_store(_MOCKED_KVS_ID).set_record('f', io.BytesIO(b'buffer data')) assert len(captured_requests) == 1 + assert captured_requests[0].headers['content-encoding'] == content_encoding assert decode_body(captured_requests[0]) == b'buffer data' assert captured_requests[0].headers['content-type'] == 'application/octet-stream' -async def test_set_record_reads_file_like_value_async(httpserver: HTTPServer) -> None: +async def test_set_record_reads_file_like_value_async( + httpserver: HTTPServer, compression_case: tuple[HttpCompressionAlgorithm, str] +) -> None: """Regression test: a file-like value is read and its bytes are uploaded, not passed through unread.""" + algorithm, content_encoding = compression_case captured_requests: list[Request] = [] def capture_request(request: Request) -> Response: @@ -51,17 +79,21 @@ def capture_request(request: Request) -> Response: httpserver.expect_request(_RECORD_PATH, method='PUT').respond_with_handler(capture_request) api_url = httpserver.url_for('/').removesuffix('/') - client = ApifyClientAsync(token='test_token', api_url=api_url) + client = ApifyClientAsync(token='test_token', api_url=api_url, compression=algorithm) await client.key_value_store(_MOCKED_KVS_ID).set_record('f', io.BytesIO(b'buffer data')) assert len(captured_requests) == 1 + assert captured_requests[0].headers['content-encoding'] == content_encoding assert decode_body(captured_requests[0]) == b'buffer data' assert captured_requests[0].headers['content-type'] == 'application/octet-stream' -def test_set_record_reads_stringio_value_sync(httpserver: HTTPServer) -> None: +def test_set_record_reads_stringio_value_sync( + httpserver: HTTPServer, compression_case: tuple[HttpCompressionAlgorithm, str] +) -> None: """Regression test: a text file-like value is read and uploaded as text/plain through the HTTP stack.""" + algorithm, content_encoding = compression_case captured_requests: list[Request] = [] def capture_request(request: Request) -> Response: @@ -71,10 +103,11 @@ def capture_request(request: Request) -> Response: httpserver.expect_request(_RECORD_PATH, method='PUT').respond_with_handler(capture_request) api_url = httpserver.url_for('/').removesuffix('/') - client = ApifyClient(token='test_token', api_url=api_url) + client = ApifyClient(token='test_token', api_url=api_url, compression=algorithm) client.key_value_store(_MOCKED_KVS_ID).set_record('f', io.StringIO('buffer data')) assert len(captured_requests) == 1 + assert captured_requests[0].headers['content-encoding'] == content_encoding assert decode_body(captured_requests[0]) == b'buffer data' assert captured_requests[0].headers['content-type'] == 'text/plain; charset=utf-8' From 890246410887fca56c67e82081b1ecb86bfc6e3a Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 21 Jul 2026 11:18:42 +0200 Subject: [PATCH 5/5] test: Drop redundant 'Regression test:' prefix from record upload test docstrings --- tests/unit/test_key_value_store.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_key_value_store.py b/tests/unit/test_key_value_store.py index cbe05568..cbf8d6c3 100644 --- a/tests/unit/test_key_value_store.py +++ b/tests/unit/test_key_value_store.py @@ -44,7 +44,7 @@ def decode_body(request: Request) -> bytes: def test_set_record_reads_file_like_value_sync( httpserver: HTTPServer, compression_case: tuple[HttpCompressionAlgorithm, str] ) -> None: - """Regression test: a file-like value is read and its bytes are uploaded, not passed through unread.""" + """A file-like value is read and its bytes are uploaded, not passed through unread.""" algorithm, content_encoding = compression_case captured_requests: list[Request] = [] @@ -68,7 +68,7 @@ def capture_request(request: Request) -> Response: async def test_set_record_reads_file_like_value_async( httpserver: HTTPServer, compression_case: tuple[HttpCompressionAlgorithm, str] ) -> None: - """Regression test: a file-like value is read and its bytes are uploaded, not passed through unread.""" + """A file-like value is read and its bytes are uploaded, not passed through unread.""" algorithm, content_encoding = compression_case captured_requests: list[Request] = [] @@ -92,7 +92,7 @@ def capture_request(request: Request) -> Response: def test_set_record_reads_stringio_value_sync( httpserver: HTTPServer, compression_case: tuple[HttpCompressionAlgorithm, str] ) -> None: - """Regression test: a text file-like value is read and uploaded as text/plain through the HTTP stack.""" + """A text file-like value is read and uploaded as text/plain through the HTTP stack.""" algorithm, content_encoding = compression_case captured_requests: list[Request] = []