From bc092b861c424dd738f32e907bab4265d5cde832 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 20 Jul 2026 21:18:03 +0200 Subject: [PATCH 1/9] fix: Read file-like values before upload in the impit transport --- src/apify_client/_utils/encoding.py | 13 ++++--- tests/unit/test_key_value_store.py | 60 +++++++++++++++++++++++++++++ tests/unit/test_utils.py | 12 +++++- 3 files changed, 77 insertions(+), 8 deletions(-) create mode 100644 tests/unit/test_key_value_store.py diff --git a/src/apify_client/_utils/encoding.py b/src/apify_client/_utils/encoding.py index b4adf385..27eda1a9 100644 --- a/src/apify_client/_utils/encoding.py +++ b/src/apify_client/_utils/encoding.py @@ -22,19 +22,20 @@ def encode_key_value_store_record_value(value: Any, *, content_type: str | None Returns: A tuple of (encoded_value, content_type). """ + # Read file-like values into memory; the underlying HTTP transport only accepts bytes-like bodies, + # so a file object would otherwise reach it unread and raise a raw `TypeError`. + if isinstance(value, io.IOBase): + value = value.read() + if not content_type: - if isinstance(value, (bytes, bytearray, io.IOBase)): + if isinstance(value, (bytes, bytearray)): content_type = 'application/octet-stream' elif isinstance(value, str): content_type = 'text/plain; charset=utf-8' else: content_type = 'application/json; charset=utf-8' - if ( - 'application/json' in content_type - and not isinstance(value, (bytes, bytearray, io.IOBase)) - and not isinstance(value, str) - ): + if 'application/json' in content_type and not isinstance(value, (bytes, bytearray, str)): # Don't use indentation to reduce size. value = json.dumps( value, diff --git a/tests/unit/test_key_value_store.py b/tests/unit/test_key_value_store.py new file mode 100644 index 00000000..46dc209f --- /dev/null +++ b/tests/unit/test_key_value_store.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import gzip +import io +from typing import TYPE_CHECKING + +from werkzeug import Request, Response + +from apify_client import ApifyClient, ApifyClientAsync + +if TYPE_CHECKING: + from pytest_httpserver import HTTPServer + +_MOCKED_KVS_ID = 'test_kvs_id' +_RECORD_PATH = f'/v2/key-value-stores/{_MOCKED_KVS_ID}/records/f' + + +def _decode_body(request: Request) -> bytes: + raw = request.get_data() + return gzip.decompress(raw) if request.headers.get('Content-Encoding') == 'gzip' else raw + + +def test_set_record_reads_file_like_value_sync(httpserver: HTTPServer) -> None: + """Regression test: a file-like value is read and its bytes are uploaded, not passed through unread.""" + captured_requests: list[Request] = [] + + def capture_request(request: Request) -> Response: + captured_requests.append(request) + return Response(status=201) + + httpserver.expect_request(_RECORD_PATH, method='PUT').respond_with_handler(capture_request) + + api_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClient(token='test_token', api_url=api_url) + + client.key_value_store(_MOCKED_KVS_ID).set_record('f', io.BytesIO(b'buffer data')) + + assert len(captured_requests) == 1 + assert _decode_body(captured_requests[0]) == b'buffer data' + assert captured_requests[0].headers['content-type'] == 'application/octet-stream' + + +async def test_set_record_reads_file_like_value_async(httpserver: HTTPServer) -> None: + """Regression test: a file-like value is read and its bytes are uploaded, not passed through unread.""" + captured_requests: list[Request] = [] + + def capture_request(request: Request) -> Response: + captured_requests.append(request) + return Response(status=201) + + httpserver.expect_request(_RECORD_PATH, method='PUT').respond_with_handler(capture_request) + + api_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClientAsync(token='test_token', api_url=api_url) + + await client.key_value_store(_MOCKED_KVS_ID).set_record('f', io.BytesIO(b'buffer data')) + + assert len(captured_requests) == 1 + assert _decode_body(captured_requests[0]) == b'buffer data' + assert captured_requests[0].headers['content-type'] == 'application/octet-stream' diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index a52d3fb4..757437fd 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -249,13 +249,21 @@ def test_encode_key_value_store_record_value( def test_encode_key_value_store_record_value_bytesio() -> None: - """Test that BytesIO is encoded as octet-stream.""" + """Test that BytesIO is read into bytes and encoded as octet-stream.""" buffer = io.BytesIO(b'buffer data') value, content_type = encode_key_value_store_record_value(buffer) - assert value == buffer + assert value == b'buffer data' assert content_type == 'application/octet-stream' +def test_encode_key_value_store_record_value_stringio() -> None: + """Test that StringIO is read into text and encoded as text/plain.""" + buffer = io.StringIO('buffer data') + value, content_type = encode_key_value_store_record_value(buffer) + assert value == 'buffer data' + assert content_type == 'text/plain; charset=utf-8' + + def test_response_to_dict() -> None: """Test parsing response as dictionary.""" mock_response = Mock() From 4f1e9ff3267a82a8dc9acc0100f007fbf1dab7c3 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 21 Jul 2026 10:14:35 +0200 Subject: [PATCH 2/9] fix: Read duck-typed file-like values and encode KVS records off the event loop --- .../_resource_clients/key_value_store.py | 7 ++++++- src/apify_client/_utils/encoding.py | 10 ++++++---- tests/unit/test_key_value_store.py | 20 +++++++++++++++++++ tests/unit/test_utils.py | 12 +++++++++++ 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/src/apify_client/_resource_clients/key_value_store.py b/src/apify_client/_resource_clients/key_value_store.py index 391c0969..6ca19184 100644 --- a/src/apify_client/_resource_clients/key_value_store.py +++ b/src/apify_client/_resource_clients/key_value_store.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import re from contextlib import asynccontextmanager, contextmanager from http import HTTPStatus @@ -801,7 +802,11 @@ async def set_record( content_type: The content type of the saved value. timeout: Timeout for the API HTTP request. """ - value, content_type = encode_key_value_store_record_value(value, content_type=content_type) + # Encoding reads file-like values and may serialize large payloads, which is blocking; offload it to a + # worker thread so it does not stall the event loop (mirrors the transport's own body-prep offload). + value, content_type = await asyncio.to_thread( + encode_key_value_store_record_value, value, content_type=content_type + ) headers = {'content-type': content_type} diff --git a/src/apify_client/_utils/encoding.py b/src/apify_client/_utils/encoding.py index 27eda1a9..d3ac20a6 100644 --- a/src/apify_client/_utils/encoding.py +++ b/src/apify_client/_utils/encoding.py @@ -1,6 +1,5 @@ from __future__ import annotations -import io import json from base64 import b64encode from functools import cache @@ -23,9 +22,12 @@ def encode_key_value_store_record_value(value: Any, *, content_type: str | None A tuple of (encoded_value, content_type). """ # Read file-like values into memory; the underlying HTTP transport only accepts bytes-like bodies, - # so a file object would otherwise reach it unread and raise a raw `TypeError`. - if isinstance(value, io.IOBase): - value = value.read() + # so a file object would otherwise reach it unread and raise a raw `TypeError`. Detect them by a + # callable `read` rather than `io.IOBase` so duck-typed file-likes (upload wrappers, raw streams) + # are read too, instead of falling through to JSON serialization. + read = getattr(value, 'read', None) + if callable(read): + value = read() if not content_type: if isinstance(value, (bytes, bytearray)): diff --git a/tests/unit/test_key_value_store.py b/tests/unit/test_key_value_store.py index 46dc209f..ef0d2aae 100644 --- a/tests/unit/test_key_value_store.py +++ b/tests/unit/test_key_value_store.py @@ -58,3 +58,23 @@ def capture_request(request: Request) -> Response: assert len(captured_requests) == 1 assert _decode_body(captured_requests[0]) == b'buffer data' assert captured_requests[0].headers['content-type'] == 'application/octet-stream' + + +def test_set_record_reads_stringio_value_sync(httpserver: HTTPServer) -> None: + """Regression test: a text file-like value is read and uploaded as text/plain through the HTTP stack.""" + captured_requests: list[Request] = [] + + def capture_request(request: Request) -> Response: + captured_requests.append(request) + return Response(status=201) + + httpserver.expect_request(_RECORD_PATH, method='PUT').respond_with_handler(capture_request) + + api_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClient(token='test_token', api_url=api_url) + + client.key_value_store(_MOCKED_KVS_ID).set_record('f', io.StringIO('buffer data')) + + assert len(captured_requests) == 1 + assert _decode_body(captured_requests[0]) == b'buffer data' + assert captured_requests[0].headers['content-type'] == 'text/plain; charset=utf-8' diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 757437fd..88963b1f 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -264,6 +264,18 @@ def test_encode_key_value_store_record_value_stringio() -> None: assert content_type == 'text/plain; charset=utf-8' +def test_encode_key_value_store_record_value_duck_typed_file_like() -> None: + """Test that a duck-typed file-like value (a callable `read`, not an `io.IOBase`) is read into bytes.""" + + class _Reader: + def read(self) -> bytes: + return b'buffer data' + + value, content_type = encode_key_value_store_record_value(_Reader()) + assert value == b'buffer data' + assert content_type == 'application/octet-stream' + + def test_response_to_dict() -> None: """Test parsing response as dictionary.""" mock_response = Mock() From c0f62ed7fc15136b3836d1f96a0865101d2ebd20 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 21 Jul 2026 10:56:46 +0200 Subject: [PATCH 3/9] style: Drop underscore prefix from test helpers and tighten comments --- src/apify_client/_resource_clients/key_value_store.py | 3 +-- src/apify_client/_utils/encoding.py | 6 ++---- tests/unit/test_key_value_store.py | 8 ++++---- tests/unit/test_utils.py | 4 ++-- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/apify_client/_resource_clients/key_value_store.py b/src/apify_client/_resource_clients/key_value_store.py index 6ca19184..13af637f 100644 --- a/src/apify_client/_resource_clients/key_value_store.py +++ b/src/apify_client/_resource_clients/key_value_store.py @@ -802,8 +802,7 @@ async def set_record( content_type: The content type of the saved value. timeout: Timeout for the API HTTP request. """ - # Encoding reads file-like values and may serialize large payloads, which is blocking; offload it to a - # worker thread so it does not stall the event loop (mirrors the transport's own body-prep offload). + # Encoding may read a file or serialize a large payload (blocking), so run it off the event loop. value, content_type = await asyncio.to_thread( encode_key_value_store_record_value, value, content_type=content_type ) diff --git a/src/apify_client/_utils/encoding.py b/src/apify_client/_utils/encoding.py index d3ac20a6..3f14fef8 100644 --- a/src/apify_client/_utils/encoding.py +++ b/src/apify_client/_utils/encoding.py @@ -21,10 +21,8 @@ def encode_key_value_store_record_value(value: Any, *, content_type: str | None Returns: A tuple of (encoded_value, content_type). """ - # Read file-like values into memory; the underlying HTTP transport only accepts bytes-like bodies, - # so a file object would otherwise reach it unread and raise a raw `TypeError`. Detect them by a - # callable `read` rather than `io.IOBase` so duck-typed file-likes (upload wrappers, raw streams) - # are read too, instead of falling through to JSON serialization. + # Read file-like values into memory; the transport only accepts bytes-like bodies. Detect them by a + # callable `read` (not `io.IOBase`) so duck-typed file-likes are read, not JSON-serialized. read = getattr(value, 'read', None) if callable(read): value = read() diff --git a/tests/unit/test_key_value_store.py b/tests/unit/test_key_value_store.py index ef0d2aae..a5abcd83 100644 --- a/tests/unit/test_key_value_store.py +++ b/tests/unit/test_key_value_store.py @@ -15,7 +15,7 @@ _RECORD_PATH = f'/v2/key-value-stores/{_MOCKED_KVS_ID}/records/f' -def _decode_body(request: Request) -> bytes: +def decode_body(request: Request) -> bytes: raw = request.get_data() return gzip.decompress(raw) if request.headers.get('Content-Encoding') == 'gzip' else raw @@ -36,7 +36,7 @@ def capture_request(request: Request) -> Response: client.key_value_store(_MOCKED_KVS_ID).set_record('f', io.BytesIO(b'buffer data')) assert len(captured_requests) == 1 - assert _decode_body(captured_requests[0]) == b'buffer data' + assert decode_body(captured_requests[0]) == b'buffer data' assert captured_requests[0].headers['content-type'] == 'application/octet-stream' @@ -56,7 +56,7 @@ def capture_request(request: Request) -> Response: await client.key_value_store(_MOCKED_KVS_ID).set_record('f', io.BytesIO(b'buffer data')) assert len(captured_requests) == 1 - assert _decode_body(captured_requests[0]) == b'buffer data' + assert decode_body(captured_requests[0]) == b'buffer data' assert captured_requests[0].headers['content-type'] == 'application/octet-stream' @@ -76,5 +76,5 @@ def capture_request(request: Request) -> Response: client.key_value_store(_MOCKED_KVS_ID).set_record('f', io.StringIO('buffer data')) assert len(captured_requests) == 1 - assert _decode_body(captured_requests[0]) == b'buffer data' + assert decode_body(captured_requests[0]) == b'buffer data' assert captured_requests[0].headers['content-type'] == 'text/plain; charset=utf-8' diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 88963b1f..803e11e2 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -267,11 +267,11 @@ def test_encode_key_value_store_record_value_stringio() -> None: def test_encode_key_value_store_record_value_duck_typed_file_like() -> None: """Test that a duck-typed file-like value (a callable `read`, not an `io.IOBase`) is read into bytes.""" - class _Reader: + class Reader: def read(self) -> bytes: return b'buffer data' - value, content_type = encode_key_value_store_record_value(_Reader()) + value, content_type = encode_key_value_store_record_value(Reader()) assert value == b'buffer data' assert content_type == 'application/octet-stream' From 6010aa2eed488ae5ffe0a7ac4d25735818f4ba85 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 21 Jul 2026 11:14:16 +0200 Subject: [PATCH 4/9] test: Parametrize key-value store record upload tests over gzip and brotli --- tests/unit/test_key_value_store.py | 47 +++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_key_value_store.py b/tests/unit/test_key_value_store.py index a5abcd83..cbe05568 100644 --- a/tests/unit/test_key_value_store.py +++ b/tests/unit/test_key_value_store.py @@ -4,6 +4,8 @@ import io from typing import TYPE_CHECKING +import brotli +import pytest from werkzeug import Request, Response from apify_client import ApifyClient, ApifyClientAsync @@ -11,17 +13,39 @@ if TYPE_CHECKING: from pytest_httpserver import HTTPServer + from apify_client.types import HttpCompressionAlgorithm + _MOCKED_KVS_ID = 'test_kvs_id' _RECORD_PATH = f'/v2/key-value-stores/{_MOCKED_KVS_ID}/records/f' +@pytest.fixture( + params=[ + pytest.param(('gzip', 'gzip'), id='gzip'), + pytest.param(('brotli', 'br'), id='brotli'), + ] +) +def compression_case(request: pytest.FixtureRequest) -> tuple[HttpCompressionAlgorithm, str]: + """Run each test over both supported request-body compression algorithms, as (algorithm, content-encoding).""" + return request.param + + def decode_body(request: Request) -> bytes: + """Decompress a captured request body according to its `Content-Encoding`.""" raw = request.get_data() - return gzip.decompress(raw) if request.headers.get('Content-Encoding') == 'gzip' else raw + encoding = request.headers.get('Content-Encoding') + if encoding == 'gzip': + return gzip.decompress(raw) + if encoding == 'br': + return brotli.decompress(raw) + return raw -def test_set_record_reads_file_like_value_sync(httpserver: HTTPServer) -> None: +def test_set_record_reads_file_like_value_sync( + httpserver: HTTPServer, compression_case: tuple[HttpCompressionAlgorithm, str] +) -> None: """Regression test: a file-like value is read and its bytes are uploaded, not passed through unread.""" + algorithm, content_encoding = compression_case captured_requests: list[Request] = [] def capture_request(request: Request) -> Response: @@ -31,17 +55,21 @@ def capture_request(request: Request) -> Response: httpserver.expect_request(_RECORD_PATH, method='PUT').respond_with_handler(capture_request) api_url = httpserver.url_for('/').removesuffix('/') - client = ApifyClient(token='test_token', api_url=api_url) + client = ApifyClient(token='test_token', api_url=api_url, compression=algorithm) client.key_value_store(_MOCKED_KVS_ID).set_record('f', io.BytesIO(b'buffer data')) assert len(captured_requests) == 1 + assert captured_requests[0].headers['content-encoding'] == content_encoding assert decode_body(captured_requests[0]) == b'buffer data' assert captured_requests[0].headers['content-type'] == 'application/octet-stream' -async def test_set_record_reads_file_like_value_async(httpserver: HTTPServer) -> None: +async def test_set_record_reads_file_like_value_async( + httpserver: HTTPServer, compression_case: tuple[HttpCompressionAlgorithm, str] +) -> None: """Regression test: a file-like value is read and its bytes are uploaded, not passed through unread.""" + algorithm, content_encoding = compression_case captured_requests: list[Request] = [] def capture_request(request: Request) -> Response: @@ -51,17 +79,21 @@ def capture_request(request: Request) -> Response: httpserver.expect_request(_RECORD_PATH, method='PUT').respond_with_handler(capture_request) api_url = httpserver.url_for('/').removesuffix('/') - client = ApifyClientAsync(token='test_token', api_url=api_url) + client = ApifyClientAsync(token='test_token', api_url=api_url, compression=algorithm) await client.key_value_store(_MOCKED_KVS_ID).set_record('f', io.BytesIO(b'buffer data')) assert len(captured_requests) == 1 + assert captured_requests[0].headers['content-encoding'] == content_encoding assert decode_body(captured_requests[0]) == b'buffer data' assert captured_requests[0].headers['content-type'] == 'application/octet-stream' -def test_set_record_reads_stringio_value_sync(httpserver: HTTPServer) -> None: +def test_set_record_reads_stringio_value_sync( + httpserver: HTTPServer, compression_case: tuple[HttpCompressionAlgorithm, str] +) -> None: """Regression test: a text file-like value is read and uploaded as text/plain through the HTTP stack.""" + algorithm, content_encoding = compression_case captured_requests: list[Request] = [] def capture_request(request: Request) -> Response: @@ -71,10 +103,11 @@ def capture_request(request: Request) -> Response: httpserver.expect_request(_RECORD_PATH, method='PUT').respond_with_handler(capture_request) api_url = httpserver.url_for('/').removesuffix('/') - client = ApifyClient(token='test_token', api_url=api_url) + client = ApifyClient(token='test_token', api_url=api_url, compression=algorithm) client.key_value_store(_MOCKED_KVS_ID).set_record('f', io.StringIO('buffer data')) assert len(captured_requests) == 1 + assert captured_requests[0].headers['content-encoding'] == content_encoding assert decode_body(captured_requests[0]) == b'buffer data' assert captured_requests[0].headers['content-type'] == 'text/plain; charset=utf-8' From 890246410887fca56c67e82081b1ecb86bfc6e3a Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 21 Jul 2026 11:18:42 +0200 Subject: [PATCH 5/9] test: Drop redundant 'Regression test:' prefix from record upload test docstrings --- tests/unit/test_key_value_store.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_key_value_store.py b/tests/unit/test_key_value_store.py index cbe05568..cbf8d6c3 100644 --- a/tests/unit/test_key_value_store.py +++ b/tests/unit/test_key_value_store.py @@ -44,7 +44,7 @@ def decode_body(request: Request) -> bytes: def test_set_record_reads_file_like_value_sync( httpserver: HTTPServer, compression_case: tuple[HttpCompressionAlgorithm, str] ) -> None: - """Regression test: a file-like value is read and its bytes are uploaded, not passed through unread.""" + """A file-like value is read and its bytes are uploaded, not passed through unread.""" algorithm, content_encoding = compression_case captured_requests: list[Request] = [] @@ -68,7 +68,7 @@ def capture_request(request: Request) -> Response: async def test_set_record_reads_file_like_value_async( httpserver: HTTPServer, compression_case: tuple[HttpCompressionAlgorithm, str] ) -> None: - """Regression test: a file-like value is read and its bytes are uploaded, not passed through unread.""" + """A file-like value is read and its bytes are uploaded, not passed through unread.""" algorithm, content_encoding = compression_case captured_requests: list[Request] = [] @@ -92,7 +92,7 @@ def capture_request(request: Request) -> Response: def test_set_record_reads_stringio_value_sync( httpserver: HTTPServer, compression_case: tuple[HttpCompressionAlgorithm, str] ) -> None: - """Regression test: a text file-like value is read and uploaded as text/plain through the HTTP stack.""" + """A text file-like value is read and uploaded as text/plain through the HTTP stack.""" algorithm, content_encoding = compression_case captured_requests: list[Request] = [] From 849a4001b8f6403fc163b1c71e4e34d152887879 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 30 Jul 2026 09:35:46 +0200 Subject: [PATCH 6/9] refactor: Drop the thread hop from async set_record encoding --- src/apify_client/_resource_clients/key_value_store.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/apify_client/_resource_clients/key_value_store.py b/src/apify_client/_resource_clients/key_value_store.py index 13af637f..391c0969 100644 --- a/src/apify_client/_resource_clients/key_value_store.py +++ b/src/apify_client/_resource_clients/key_value_store.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import re from contextlib import asynccontextmanager, contextmanager from http import HTTPStatus @@ -802,10 +801,7 @@ async def set_record( content_type: The content type of the saved value. timeout: Timeout for the API HTTP request. """ - # Encoding may read a file or serialize a large payload (blocking), so run it off the event loop. - value, content_type = await asyncio.to_thread( - encode_key_value_store_record_value, value, content_type=content_type - ) + value, content_type = encode_key_value_store_record_value(value, content_type=content_type) headers = {'content-type': content_type} From 3a9a5d4786626bd034bf8d694f91519b3f054f6a Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 30 Jul 2026 10:00:36 +0200 Subject: [PATCH 7/9] fix: Reject KVS record values the encoder cannot turn into a request body --- src/apify_client/_utils/encoding.py | 35 ++++++++++++++++++++++++++--- tests/unit/test_utils.py | 31 +++++++++++++++++++++++-- 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/src/apify_client/_utils/encoding.py b/src/apify_client/_utils/encoding.py index 3f14fef8..0c5722f0 100644 --- a/src/apify_client/_utils/encoding.py +++ b/src/apify_client/_utils/encoding.py @@ -3,6 +3,7 @@ import json from base64 import b64encode from functools import cache +from inspect import isawaitable, iscoroutine from typing import TYPE_CHECKING, Any from apify_client._models import WebhookCreate, WebhookRepresentation @@ -11,22 +12,42 @@ from apify_client.types import WebhooksList -def encode_key_value_store_record_value(value: Any, *, content_type: str | None = None) -> tuple[Any, str]: +def encode_key_value_store_record_value( + value: Any, *, content_type: str | None = None +) -> tuple[bytes | bytearray | str, str]: """Encode a value for storage in a key-value store record. Args: - value: The value to encode (can be dict, str, bytes, or file-like object). + value: The value to encode. Anything exposing a callable `read` is treated as a file-like object: `read` + is called with no arguments, so the value is consumed from its current position and buffered in + memory whole - the object is neither rewound nor closed, and async file-like objects are rejected. + Any other value is JSON-serialized unless it is already bytes or a string. content_type: The content type; if None, it's inferred from the value type. Returns: A tuple of (encoded_value, content_type). + + Raises: + TypeError: If the value cannot be encoded into a body the transport accepts. """ # Read file-like values into memory; the transport only accepts bytes-like bodies. Detect them by a - # callable `read` (not `io.IOBase`) so duck-typed file-likes are read, not JSON-serialized. + # callable `read` (not `io.IOBase`) so duck-typed file-likes are read, not JSON-serialized. Impit exposes + # no streaming `content=` API, so the value has to be buffered whole. read = getattr(value, 'read', None) if callable(read): value = read() + if isawaitable(value): + if iscoroutine(value): + value.close() # Prevent a "coroutine was never awaited" warning. + raise TypeError( + 'Async file-like objects are not supported. Await the read yourself and pass the resulting ' + 'bytes or string.' + ) + + if not isinstance(value, (bytes, bytearray, str)): + raise TypeError(f'Reading the file-like value returned {type(value).__name__}, expected bytes or str.') + if not content_type: if isinstance(value, (bytes, bytearray)): content_type = 'application/octet-stream' @@ -44,6 +65,14 @@ def encode_key_value_store_record_value(value: Any, *, content_type: str | None default=str, ).encode('utf-8') + # A non-JSON content type skips the serialization above, so anything that is not bytes-like would reach the + # transport unencoded and fail there with an opaque error. + if not isinstance(value, (bytes, bytearray, str)): + raise TypeError( + f'Cannot encode a {type(value).__name__} value as {content_type!r}. Pass bytes, a string, or a ' + 'file-like object, or use a JSON content type.' + ) + return (value, content_type) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 803e11e2..0ed89bd0 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -223,8 +223,7 @@ def test__clean_json_payload(input_dict: dict, expected: dict) -> None: def test_encode_key_value_store_record_value_dict() -> None: """Test that dictionaries are encoded as JSON.""" value, content_type = encode_key_value_store_record_value({'key': 'value'}) - assert b'"key"' in value - assert b'"value"' in value + assert value == b'{"key": "value"}' assert content_type == 'application/json; charset=utf-8' @@ -276,6 +275,34 @@ def read(self) -> bytes: assert content_type == 'application/octet-stream' +def test_encode_key_value_store_record_value_async_file_like_raises() -> None: + """Test that an async file-like value is rejected instead of storing the repr of an un-awaited coroutine.""" + + class AsyncReader: + async def read(self) -> bytes: + return b'buffer data' + + with pytest.raises(TypeError, match='Async file-like objects are not supported'): + encode_key_value_store_record_value(AsyncReader()) + + +def test_encode_key_value_store_record_value_non_bytes_read_raises() -> None: + """Test that a `read` returning neither bytes nor str is rejected instead of being JSON-serialized.""" + + class EmptyNonBlockingReader: + def read(self) -> None: + """Mimic a non-blocking raw stream with no data available.""" + + with pytest.raises(TypeError, match='returned NoneType, expected bytes or str'): + encode_key_value_store_record_value(EmptyNonBlockingReader()) + + +def test_encode_key_value_store_record_value_non_encodable_with_explicit_content_type_raises() -> None: + """Test that a non-bytes-like value with a non-JSON content type is rejected before it reaches the transport.""" + with pytest.raises(TypeError, match="Cannot encode a dict value as 'image/png'"): + encode_key_value_store_record_value({'a': 1}, content_type='image/png') + + def test_response_to_dict() -> None: """Test parsing response as dictionary.""" mock_response = Mock() From 4ce034bae64b7f8ea5db8d160e8e53431d3475b5 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 30 Jul 2026 10:00:39 +0200 Subject: [PATCH 8/9] test: Cover all file-like record shapes in both sync and async set_record --- tests/unit/test_key_value_store.py | 117 +++++++++++++++-------------- 1 file changed, 62 insertions(+), 55 deletions(-) diff --git a/tests/unit/test_key_value_store.py b/tests/unit/test_key_value_store.py index cbf8d6c3..6af83f8b 100644 --- a/tests/unit/test_key_value_store.py +++ b/tests/unit/test_key_value_store.py @@ -2,7 +2,7 @@ import gzip import io -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import brotli import pytest @@ -11,6 +11,8 @@ from apify_client import ApifyClient, ApifyClientAsync if TYPE_CHECKING: + from collections.abc import Callable + from pytest_httpserver import HTTPServer from apify_client.types import HttpCompressionAlgorithm @@ -19,6 +21,22 @@ _RECORD_PATH = f'/v2/key-value-stores/{_MOCKED_KVS_ID}/records/f' +class DuckTypedReader: + """A file-like object that is not an `io.IOBase`, so only duck-typed detection picks it up.""" + + def read(self) -> bytes: + return b'buffer data' + + +# The values are built by a factory because reading consumes them, and each case runs once per compression +# algorithm. Each case is (value factory, expected uploaded body, expected content type). +_FILE_LIKE_VALUE_CASES = [ + pytest.param(lambda: io.BytesIO(b'buffer data'), b'buffer data', 'application/octet-stream', id='bytes io'), + pytest.param(lambda: io.StringIO('buffer data'), b'buffer data', 'text/plain; charset=utf-8', id='string io'), + pytest.param(DuckTypedReader, b'buffer data', 'application/octet-stream', id='duck-typed reader'), +] + + @pytest.fixture( params=[ pytest.param(('gzip', 'gzip'), id='gzip'), @@ -30,6 +48,25 @@ def compression_case(request: pytest.FixtureRequest) -> tuple[HttpCompressionAlg return request.param +@pytest.fixture +def api_url(httpserver: HTTPServer) -> str: + """The base URL of the mock server, in the form the clients expect.""" + return httpserver.url_for('/').removesuffix('/') + + +@pytest.fixture +def captured_records(httpserver: HTTPServer) -> list[Request]: + """Answer record uploads with a 201 and collect the requests the client sent.""" + requests: list[Request] = [] + + def capture_request(request: Request) -> Response: + requests.append(request) + return Response(status=201) + + httpserver.expect_request(_RECORD_PATH, method='PUT').respond_with_handler(capture_request) + return requests + + def decode_body(request: Request) -> bytes: """Decompress a captured request body according to its `Content-Encoding`.""" raw = request.get_data() @@ -41,73 +78,43 @@ def decode_body(request: Request) -> bytes: return raw +@pytest.mark.parametrize(('make_value', 'expected_body', 'expected_content_type'), _FILE_LIKE_VALUE_CASES) def test_set_record_reads_file_like_value_sync( - httpserver: HTTPServer, compression_case: tuple[HttpCompressionAlgorithm, str] + api_url: str, + captured_records: list[Request], + compression_case: tuple[HttpCompressionAlgorithm, str], + make_value: Callable[[], Any], + expected_body: bytes, + expected_content_type: str, ) -> None: """A file-like value is read and its bytes are uploaded, not passed through unread.""" algorithm, content_encoding = compression_case - captured_requests: list[Request] = [] - - def capture_request(request: Request) -> Response: - captured_requests.append(request) - return Response(status=201) - - httpserver.expect_request(_RECORD_PATH, method='PUT').respond_with_handler(capture_request) - - api_url = httpserver.url_for('/').removesuffix('/') client = ApifyClient(token='test_token', api_url=api_url, compression=algorithm) - client.key_value_store(_MOCKED_KVS_ID).set_record('f', io.BytesIO(b'buffer data')) + client.key_value_store(_MOCKED_KVS_ID).set_record('f', make_value()) - assert len(captured_requests) == 1 - assert captured_requests[0].headers['content-encoding'] == content_encoding - assert decode_body(captured_requests[0]) == b'buffer data' - assert captured_requests[0].headers['content-type'] == 'application/octet-stream' + assert len(captured_records) == 1 + assert captured_records[0].headers['content-encoding'] == content_encoding + assert decode_body(captured_records[0]) == expected_body + assert captured_records[0].headers['content-type'] == expected_content_type +@pytest.mark.parametrize(('make_value', 'expected_body', 'expected_content_type'), _FILE_LIKE_VALUE_CASES) async def test_set_record_reads_file_like_value_async( - httpserver: HTTPServer, compression_case: tuple[HttpCompressionAlgorithm, str] + api_url: str, + captured_records: list[Request], + compression_case: tuple[HttpCompressionAlgorithm, str], + make_value: Callable[[], Any], + expected_body: bytes, + expected_content_type: str, ) -> None: """A file-like value is read and its bytes are uploaded, not passed through unread.""" algorithm, content_encoding = compression_case - captured_requests: list[Request] = [] - - def capture_request(request: Request) -> Response: - captured_requests.append(request) - return Response(status=201) - - httpserver.expect_request(_RECORD_PATH, method='PUT').respond_with_handler(capture_request) - - api_url = httpserver.url_for('/').removesuffix('/') client = ApifyClientAsync(token='test_token', api_url=api_url, compression=algorithm) - await client.key_value_store(_MOCKED_KVS_ID).set_record('f', io.BytesIO(b'buffer data')) - - assert len(captured_requests) == 1 - assert captured_requests[0].headers['content-encoding'] == content_encoding - assert decode_body(captured_requests[0]) == b'buffer data' - assert captured_requests[0].headers['content-type'] == 'application/octet-stream' - - -def test_set_record_reads_stringio_value_sync( - httpserver: HTTPServer, compression_case: tuple[HttpCompressionAlgorithm, str] -) -> None: - """A text file-like value is read and uploaded as text/plain through the HTTP stack.""" - algorithm, content_encoding = compression_case - captured_requests: list[Request] = [] - - def capture_request(request: Request) -> Response: - captured_requests.append(request) - return Response(status=201) - - httpserver.expect_request(_RECORD_PATH, method='PUT').respond_with_handler(capture_request) - - api_url = httpserver.url_for('/').removesuffix('/') - client = ApifyClient(token='test_token', api_url=api_url, compression=algorithm) - - client.key_value_store(_MOCKED_KVS_ID).set_record('f', io.StringIO('buffer data')) + await client.key_value_store(_MOCKED_KVS_ID).set_record('f', make_value()) - assert len(captured_requests) == 1 - assert captured_requests[0].headers['content-encoding'] == content_encoding - assert decode_body(captured_requests[0]) == b'buffer data' - assert captured_requests[0].headers['content-type'] == 'text/plain; charset=utf-8' + assert len(captured_records) == 1 + assert captured_records[0].headers['content-encoding'] == content_encoding + assert decode_body(captured_records[0]) == expected_body + assert captured_records[0].headers['content-type'] == expected_content_type From 8fef0b9cba4dd14b61b6ab282b82f70001d206fa Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 30 Jul 2026 10:31:32 +0200 Subject: [PATCH 9/9] test: Make file-like set_record test parameters keyword-only --- tests/unit/test_key_value_store.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/test_key_value_store.py b/tests/unit/test_key_value_store.py index 6af83f8b..eb2176ef 100644 --- a/tests/unit/test_key_value_store.py +++ b/tests/unit/test_key_value_store.py @@ -80,6 +80,7 @@ def decode_body(request: Request) -> bytes: @pytest.mark.parametrize(('make_value', 'expected_body', 'expected_content_type'), _FILE_LIKE_VALUE_CASES) def test_set_record_reads_file_like_value_sync( + *, api_url: str, captured_records: list[Request], compression_case: tuple[HttpCompressionAlgorithm, str], @@ -101,6 +102,7 @@ def test_set_record_reads_file_like_value_sync( @pytest.mark.parametrize(('make_value', 'expected_body', 'expected_content_type'), _FILE_LIKE_VALUE_CASES) async def test_set_record_reads_file_like_value_async( + *, api_url: str, captured_records: list[Request], compression_case: tuple[HttpCompressionAlgorithm, str],