diff --git a/scripts/postprocess_generated_models.py b/scripts/postprocess_generated_models.py index 50a51f56..f3d728d8 100644 --- a/scripts/postprocess_generated_models.py +++ b/scripts/postprocess_generated_models.py @@ -1,5 +1,9 @@ """Post-process datamodel-codegen output to fix known issues and prune the TypedDict file. +Applied to both `_models.py` and `_typeddicts.py`: +- Reparent classes whose schema spells out the wire shape standalone instead of extending the base schema it + duplicates, so the generated class declares every field. + Applied to `_models.py`: - Fix discriminator field names that use camelCase instead of snake_case (known issue with discriminators on schemas referenced from array items). @@ -12,6 +16,7 @@ - Add `@docs_group('Models')` to every model class (plus the required import). Applied to `_typeddicts.py`: +- Drop the fields a reparented TypedDict inherits from its new base, which PEP 589 forbids it from redeclaring. - Keep only the TypedDicts actually used as resource-client method inputs (plus their transitive dependencies). The file is generated in full by datamodel-codegen; the trimming happens here. - Rename every kept class to add a `Dict` suffix so it doesn't clash with the Pydantic model name @@ -47,6 +52,14 @@ 'pricingModel': 'pricing_model', } +# Map of `{class name: base class it should inherit from}`, applied to both generated files. A schema that spells out +# only a few properties instead of extending the base schema carrying the rest generates a class whose `extra='allow'` +# absorbs the other fields - and `to_camel` doesn't alias extras, so sending it as a request body puts snake_case keys +# on the wire. Reparenting declares the full shape. +BASE_CLASS_FIXES: dict[str, str] = { + 'RequestDraft': 'RequestBase', +} + # TypedDicts accepted as inputs by resource-client methods. These are the roots of the reachability # walk over `_typeddicts.py`: anything not reachable from here (directly or transitively) # is dropped so only the TypedDicts that are part of the public input surface — plus their nested @@ -102,6 +115,65 @@ def _base_names(node: ast.ClassDef) -> set[str]: return {b.id for b in node.bases if isinstance(b, ast.Name)} +def reparent_classes(content: str) -> str: + """Replace the base class of every `BASE_CLASS_FIXES` entry with the mapped one. + + Rewrites the whole base list, so re-running is a no-op. A mapped class absent from `content` is skipped, since a + schema doesn't always yield a class in both files. The base is not looked up: naming one the file doesn't define + yields source that fails to import, which beats silently skipping the fix. + """ + for name, base in BASE_CLASS_FIXES.items(): + content = re.sub( + rf'^class {re.escape(name)}\([^)]*\):', + f'class {name}({base}):', + content, + flags=re.MULTILINE, + ) + return content + + +def _annotated_field_names(node: ast.ClassDef) -> set[str]: + """Return the names of every annotated field declared directly in `node`'s body.""" + return { + stmt.target.id for stmt in node.body if isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name) + } + + +def drop_inherited_typeddict_fields(content: str) -> str: + """Delete the fields a reparented TypedDict now inherits, along with their description docstrings. + + PEP 589 forbids redeclaring a base's key, even to turn a `NotRequired` one into a required one. The keys stay + required at runtime, where the Pydantic model keeps its own redeclarations. + """ + tree = ast.parse(content) + classes = {node.name: node for node in tree.body if isinstance(node, ast.ClassDef)} + drop_line_indices: set[int] = set() + + for name, base in BASE_CLASS_FIXES.items(): + node, base_node = classes.get(name), classes.get(base) + if node is None or base_node is None: + continue + inherited = _annotated_field_names(base_node) + for index, stmt in enumerate(node.body): + if not isinstance(stmt, ast.AnnAssign) or not isinstance(stmt.target, ast.Name): + continue + if stmt.target.id not in inherited: + continue + end_line = stmt.end_lineno + following = node.body[index + 1] if index + 1 < len(node.body) else None + if following is not None and _is_string_expr(following): + end_line = following.end_lineno + assert end_line is not None # noqa: S101 + drop_line_indices.update(range(stmt.lineno - 1, end_line)) + + if not drop_line_indices: + return content + + lines = content.split('\n') + kept = [line for i, line in enumerate(lines) if i not in drop_line_indices] + return _collapse_blank_lines('\n'.join(kept)) + + def fix_discriminators(content: str) -> str: """Replace camelCase discriminator values with their snake_case equivalents.""" for camel, snake in DISCRIMINATOR_FIXES.items(): @@ -645,7 +717,8 @@ def postprocess_models(models_path: Path, literals_path: Path) -> list[Path]: Returns the list of paths that were (re)written. """ original = models_path.read_text() - fixed = fix_discriminators(original) + fixed = reparent_classes(original) + fixed = fix_discriminators(fixed) fixed = absolutize_doc_links(fixed) fixed = convert_enums_to_literals(fixed) fixed = add_docs_group_decorators(fixed, 'Models') @@ -666,7 +739,9 @@ def postprocess_models(models_path: Path, literals_path: Path) -> list[Path]: def postprocess_typeddicts(path: Path, alias_map: dict[str, dict[str, str]]) -> bool: """Apply `_typeddicts.py`-specific fixes. Returns True if the file changed.""" original = path.read_text() - pruned, kept = prune_typeddicts(original, RESOURCE_INPUT_TYPEDDICTS) + # Reparenting comes first so the new base counts as a dependency of the input surface and survives pruning. + reparented = drop_inherited_typeddict_fields(reparent_classes(original)) + pruned, kept = prune_typeddicts(reparented, RESOURCE_INPUT_TYPEDDICTS) renamed = rename_with_dict_suffix(pruned, kept) flattened = flatten_empty_typeddicts(renamed) camelized = add_camel_case_typeddicts(flattened, alias_map) diff --git a/src/apify_client/_models.py b/src/apify_client/_models.py index 7e39f283..a1de1d3c 100644 --- a/src/apify_client/_models.py +++ b/src/apify_client/_models.py @@ -2486,7 +2486,7 @@ class Request(RequestBase): @docs_group('Models') -class RequestDraft(BaseModel): +class RequestDraft(RequestBase): """A request that failed to be processed during a request queue operation and can be retried.""" model_config = ConfigDict( diff --git a/src/apify_client/_resource_clients/request_queue.py b/src/apify_client/_resource_clients/request_queue.py index a9539bfb..21099e65 100644 --- a/src/apify_client/_resource_clients/request_queue.py +++ b/src/apify_client/_resource_clients/request_queue.py @@ -81,7 +81,7 @@ def _serialize_requests( return [ json.dumps( (request if isinstance(request, RequestDraft) else RequestDraft.model_validate(request)).model_dump( - by_alias=True, exclude_none=True + mode='json', by_alias=True, exclude_none=True, fallback=str ), ensure_ascii=False, allow_nan=False, @@ -238,7 +238,7 @@ def add_request( https://docs.apify.com/api/v2#/reference/request-queues/request-collection/add-request Args: - request: The request to add to the queue. + request: The request to add to the queue. Must carry a `unique_key` and a `url`. forefront: Whether to add the request to the head or the end of the queue. timeout: Timeout for the API HTTP request. @@ -253,7 +253,7 @@ def add_request( response = self._http_client.call( url=self._build_url('requests'), method='POST', - json=request.model_dump(by_alias=True, exclude_none=True), + json=request.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str), params=request_params, timeout=timeout, ) @@ -315,7 +315,7 @@ def update_request( response = self._http_client.call( url=self._build_url(f'requests/{request.id}'), method='PUT', - json=request.model_dump(by_alias=True, exclude_none=True), + json=request.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str), params=request_params, timeout=timeout, ) @@ -417,7 +417,7 @@ def batch_add_requests( https://docs.apify.com/api/v2#/reference/request-queues/batch-request-operations/add-requests Args: - requests: List of requests to be added to the queue. + requests: List of requests to be added to the queue. Each must carry a `unique_key` and a `url`. forefront: Whether to add requests to the front of the queue. max_parallel: Specifies the maximum number of parallel tasks for API calls. This is only applicable to the async client. For the sync client, this value must be set to 1, as parallel execution @@ -504,7 +504,7 @@ def batch_delete_requests( else RequestDraftDelete.model_validate( request, ) - ).model_dump(by_alias=True, exclude_none=True) + ).root.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str) for request in requests ] @@ -765,7 +765,7 @@ async def add_request( https://docs.apify.com/api/v2#/reference/request-queues/request-collection/add-request Args: - request: The request to add to the queue. + request: The request to add to the queue. Must carry a `unique_key` and a `url`. forefront: Whether to add the request to the head or the end of the queue. timeout: Timeout for the API HTTP request. @@ -780,7 +780,7 @@ async def add_request( response = await self._http_client.call( url=self._build_url('requests'), method='POST', - json=request.model_dump(by_alias=True, exclude_none=True), + json=request.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str), params=request_params, timeout=timeout, ) @@ -840,7 +840,7 @@ async def update_request( response = await self._http_client.call( url=self._build_url(f'requests/{request.id}'), method='PUT', - json=request.model_dump(by_alias=True, exclude_none=True), + json=request.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str), params=request_params, timeout=timeout, ) @@ -990,7 +990,7 @@ async def batch_add_requests( https://docs.apify.com/api/v2#/reference/request-queues/batch-request-operations/add-requests Args: - requests: List of requests to be added to the queue. + requests: List of requests to be added to the queue. Each must carry a `unique_key` and a `url`. forefront: Whether to add requests to the front of the queue. max_parallel: Specifies the maximum number of parallel tasks for API calls. This is only applicable to the async client. For the sync client, this value must be set to 1, as parallel execution @@ -1082,7 +1082,7 @@ async def batch_delete_requests( else RequestDraftDelete.model_validate( request, ) - ).model_dump(by_alias=True, exclude_none=True) + ).root.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str) for request in requests ] diff --git a/src/apify_client/_typeddicts.py b/src/apify_client/_typeddicts.py index adbad385..864b9a1b 100644 --- a/src/apify_client/_typeddicts.py +++ b/src/apify_client/_typeddicts.py @@ -112,41 +112,23 @@ class RequestCamelDict(RequestBaseCamelDict): @docs_group('Typed dicts') -class RequestDraftDict(TypedDict): +class RequestDraftDict(RequestBaseDict): """A request that failed to be processed during a request queue operation and can be retried.""" id: NotRequired[str] """ A unique identifier assigned to the request. """ - unique_key: str - """ - A unique key used for request de-duplication. Requests with the same unique key are considered identical. - """ - url: str - """ - The URL of the request. - """ - method: NotRequired[Literal['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'CONNECT', 'OPTIONS', 'TRACE', 'PATCH']] @docs_group('Typed dicts') -class RequestDraftCamelDict(TypedDict): +class RequestDraftCamelDict(RequestBaseCamelDict): """A request that failed to be processed during a request queue operation and can be retried.""" id: NotRequired[str] """ A unique identifier assigned to the request. """ - uniqueKey: str - """ - A unique key used for request de-duplication. Requests with the same unique key are considered identical. - """ - url: str - """ - The URL of the request. - """ - method: NotRequired[Literal['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'CONNECT', 'OPTIONS', 'TRACE', 'PATCH']] @docs_group('Typed dicts') diff --git a/tests/integration/test_request_queue.py b/tests/integration/test_request_queue.py index c09e2d7d..8be9454b 100644 --- a/tests/integration/test_request_queue.py +++ b/tests/integration/test_request_queue.py @@ -3,8 +3,10 @@ from __future__ import annotations from collections.abc import AsyncIterator, Iterator -from datetime import timedelta -from typing import TYPE_CHECKING +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING, Any + +import pytest from .._utils import ( collect_iterate_until_present, @@ -28,17 +30,75 @@ RequestRegistration, UnlockRequestsResult, ) +from apify_client.errors import ApifyApiError if TYPE_CHECKING: from apify_client import ApifyClient, ApifyClientAsync from apify_client._resource_clients.request_queue import RequestQueueClient, RequestQueueClientAsync from apify_client._typeddicts import ( RequestDict, + RequestDraftCamelDict, RequestDraftDeleteDict, RequestDraftDict, ) +# The wire format the API declares for `handled_at`, and the instant it denotes. The client validates the string +# into a datetime, so it only sends ISO 8601 back out while it serializes in JSON mode. +HANDLED_AT_ISO = '2019-06-16T10:23:31.607Z' +HANDLED_AT = datetime(2019, 6, 16, 10, 23, 31, 607000, tzinfo=UTC) + +# Every request field beyond `id`/`unique_key`/`url`, snake_cased. The API declares its write bodies with +# `additionalProperties: false`, so each of these has to reach it camelCased to be stored at all. +ALL_REQUEST_FIELDS: RequestDict = { + 'method': 'POST', + 'user_data': {'label': 'DETAIL', 'depth': 2}, + 'no_retry': True, + 'retry_count': 3, + 'headers': {'X-Test': 'yes'}, + 'payload': '{"a": 1}', + 'loaded_url': 'https://example.com/loaded', + 'error_messages': ['boom'], + 'handled_at': HANDLED_AT_ISO, +} + + +async def fetch_stored_request( + rq_client: RequestQueueClient | RequestQueueClientAsync, + request_id: str, +) -> Request: + """Poll until `request_id` is readable back from the queue, then return it.""" + + async def get_request() -> Request | None: + return await maybe_await(rq_client.get_request(request_id)) + + stored = await poll_until_condition(get_request, lambda request: request is not None) + assert isinstance(stored, Request) + return stored + + +def non_identity_fields(request: Request) -> dict[str, Any]: + """Return a stored request's fields without the ones identifying it, so two write paths can be compared.""" + dumped = request.model_dump(by_alias=True) + for key in ('id', 'uniqueKey', 'url'): + dumped.pop(key, None) + return dumped + + +def assert_all_fields_stored(request: Request) -> None: + """Assert the stored request carries every value of `ALL_REQUEST_FIELDS`.""" + assert request.method == 'POST' + assert request.user_data is not None + assert request.user_data.model_dump() == {'label': 'DETAIL', 'depth': 2} + assert request.no_retry is True + assert request.retry_count == 3 + assert request.headers == {'X-Test': 'yes'} + assert request.payload == '{"a": 1}' + assert str(request.loaded_url) == 'https://example.com/loaded' + assert request.error_messages == ['boom'] + assert request.handled_at == HANDLED_AT + + async def ensure_queue_is_populated( rq_client: RequestQueueClient | RequestQueueClientAsync, *, @@ -578,6 +638,144 @@ async def get_added_request() -> Request | None: await maybe_await(rq_client.delete()) +async def test_request_queue_add_request_round_trips_all_fields(client: ApifyClient | ApifyClientAsync) -> None: + """Every field of a snake_cased request survives `add_request` and comes back from `get_request`.""" + rq = await maybe_await(client.request_queues().get_or_create(name=get_random_resource_name('rq'))) + assert isinstance(rq, RequestQueue) + rq_client = client.request_queue(rq.id) + + try: + request_data: RequestDraftDict = { + 'unique_key': 'round-trip', + 'url': 'https://example.com/round-trip', + **ALL_REQUEST_FIELDS, + } + add_result = await maybe_await(rq_client.add_request(request_data)) + assert isinstance(add_result, RequestRegistration) + + stored = await fetch_stored_request(rq_client, add_result.request_id) + assert_all_fields_stored(stored) + finally: + await maybe_await(rq_client.delete()) + + +async def test_request_queue_batch_add_requests_round_trips_all_fields( + client: ApifyClient | ApifyClientAsync, +) -> None: + """Every field of a snake_cased request survives `batch_add_requests` and comes back from `get_request`.""" + rq = await maybe_await(client.request_queues().get_or_create(name=get_random_resource_name('rq'))) + assert isinstance(rq, RequestQueue) + rq_client = client.request_queue(rq.id) + + try: + requests_to_add: list[RequestDraftDict] = [ + { + 'unique_key': 'batch-round-trip', + 'url': 'https://example.com/batch-round-trip', + **ALL_REQUEST_FIELDS, + } + ] + batch_result = await maybe_await(rq_client.batch_add_requests(requests_to_add)) + assert isinstance(batch_result, BatchAddResult) + assert len(batch_result.unprocessed_requests) == 0 + assert len(batch_result.processed_requests) == 1 + + stored = await fetch_stored_request(rq_client, batch_result.processed_requests[0].request_id) + assert_all_fields_stored(stored) + finally: + await maybe_await(rq_client.delete()) + + +async def test_request_queue_add_and_update_request_store_identical_fields( + client: ApifyClient | ApifyClientAsync, +) -> None: + """The same field dict stored through `add_request` and through `update_request` lands identically.""" + rq = await maybe_await(client.request_queues().get_or_create(name=get_random_resource_name('rq'))) + assert isinstance(rq, RequestQueue) + rq_client = client.request_queue(rq.id) + + try: + added = await maybe_await( + rq_client.add_request( + {'unique_key': 'parity-add', 'url': 'https://example.com/parity-add', **ALL_REQUEST_FIELDS} + ) + ) + assert isinstance(added, RequestRegistration) + + seeded = await maybe_await( + rq_client.add_request({'unique_key': 'parity-update', 'url': 'https://example.com/parity-update'}) + ) + assert isinstance(seeded, RequestRegistration) + updated = await maybe_await( + rq_client.update_request( + { + 'id': seeded.request_id, + 'unique_key': 'parity-update', + 'url': 'https://example.com/parity-update', + **ALL_REQUEST_FIELDS, + } + ) + ) + assert isinstance(updated, RequestRegistration) + + from_add = await fetch_stored_request(rq_client, added.request_id) + from_update = await fetch_stored_request(rq_client, seeded.request_id) + assert non_identity_fields(from_add) == non_identity_fields(from_update) + assert_all_fields_stored(from_add) + assert_all_fields_stored(from_update) + finally: + await maybe_await(rq_client.delete()) + + +async def test_request_queue_add_request_accepts_camel_cased_fields(client: ApifyClient | ApifyClientAsync) -> None: + """A camelCased request dict stores the same fields as its snake_cased equivalent.""" + rq = await maybe_await(client.request_queues().get_or_create(name=get_random_resource_name('rq'))) + assert isinstance(rq, RequestQueue) + rq_client = client.request_queue(rq.id) + + try: + camel_request: RequestDraftCamelDict = { + 'uniqueKey': 'camel', + 'url': 'https://example.com/camel', + 'method': 'POST', + 'userData': {'label': 'DETAIL', 'depth': 2}, + 'noRetry': True, + 'retryCount': 3, + 'headers': {'X-Test': 'yes'}, + 'payload': '{"a": 1}', + 'loadedUrl': 'https://example.com/loaded', + 'errorMessages': ['boom'], + 'handledAt': HANDLED_AT_ISO, + } + add_result = await maybe_await(rq_client.add_request(camel_request)) + assert isinstance(add_result, RequestRegistration) + + stored = await fetch_stored_request(rq_client, add_result.request_id) + assert_all_fields_stored(stored) + finally: + await maybe_await(rq_client.delete()) + + +async def test_request_queue_add_request_rejects_undeclared_fields(client: ApifyClient | ApifyClientAsync) -> None: + """The API refuses a body key its schema does not declare, so a snake_cased field takes the whole write down.""" + rq = await maybe_await(client.request_queues().get_or_create(name=get_random_resource_name('rq'))) + assert isinstance(rq, RequestQueue) + rq_client = client.request_queue(rq.id) + + try: + # `model_validate` keeps the undeclared key as a model extra, which is serialized verbatim. + draft = RequestDraft.model_validate( + {'unique_key': 'undeclared', 'url': 'https://example.com/undeclared', 'undeclared_field': 'value'} + ) + with pytest.raises(ApifyApiError, match='not allowed by the schema'): + await maybe_await(rq_client.add_request(draft)) + + with pytest.raises(ApifyApiError, match='not allowed by the schema'): + await maybe_await(rq_client.batch_add_requests([draft])) + finally: + await maybe_await(rq_client.delete()) + + async def test_request_queue_collection_iterate(client: ApifyClient | ApifyClientAsync, *, is_async: bool) -> None: """Test paginated iteration over user request queues.""" created_ids: list[str] = [] diff --git a/tests/unit/test_client_request_queue.py b/tests/unit/test_client_request_queue.py index 8c34b025..a07f737e 100644 --- a/tests/unit/test_client_request_queue.py +++ b/tests/unit/test_client_request_queue.py @@ -6,9 +6,11 @@ from typing import TYPE_CHECKING import pytest +from pydantic import ValidationError from werkzeug.wrappers import Response from apify_client import ApifyClient, ApifyClientAsync +from apify_client._models import RequestDraftDelete from apify_client.errors import ApifyApiError if TYPE_CHECKING: @@ -17,7 +19,7 @@ from pytest_httpserver import HTTPServer from werkzeug.wrappers import Request - from apify_client._typeddicts import RequestDraftDict + from apify_client._typeddicts import RequestDict, RequestDraftDict # The Apify API limit on the payload size of a batch-add request, which the client's batching must respect. _API_MAX_PAYLOAD_SIZE_BYTES = 9 * 1024 * 1024 @@ -119,8 +121,11 @@ def _make_large_requests() -> list[RequestDraftDict]: ] -def _payload_capturing_handler(payloads: list[bytes]) -> Callable[[Request], Response]: - """Return a handler that records each POST body and responds with an empty batch result. +def _payload_capturing_handler( + payloads: list[bytes], + response_content: str = _EMPTY_BATCH_RESPONSE_CONTENT, +) -> Callable[[Request], Response]: + """Return a handler that records each request body and responds with `response_content`. Bodies below the client's compression threshold arrive uncompressed, so the recorded payload is decompressed only when the request says it was encoded. @@ -129,7 +134,7 @@ def _payload_capturing_handler(payloads: list[bytes]) -> Callable[[Request], Res def handler(request: Request) -> Response: body = request.get_data() payloads.append(gzip.decompress(body) if request.headers.get('Content-Encoding') == 'gzip' else body) - return Response(_EMPTY_BATCH_RESPONSE_CONTENT, status=200, content_type='application/json') + return Response(response_content, status=200, content_type='application/json') return handler @@ -253,3 +258,258 @@ def test_batch_processed_partially_sync(httpserver: HTTPServer) -> None: assert requests[0]['unique_key'] in {request.unique_key for request in batch_response.processed_requests} assert len(batch_response.unprocessed_requests) == 1 assert batch_response.unprocessed_requests[0].unique_key == requests[1]['unique_key'] + + +_REQUEST_REGISTRATION_RESPONSE_CONTENT = ( + '{"data": {"requestId": "YiKoxjkaS9gjGTqhF", "wasAlreadyPresent": false, "wasAlreadyHandled": false}}' +) + +_FULL_REQUEST_DICT: RequestDict = { + 'id': 'YiKoxjkaS9gjGTqhF', + 'unique_key': 'http://example.com/1', + 'url': 'http://example.com/1', + 'method': 'GET', + 'user_data': {'label': 'DETAIL'}, + 'no_retry': True, + 'retry_count': 2, + 'loaded_url': 'http://example.com/1/final', + 'headers': {'X-Test': 'value'}, + 'payload': 'body', + 'error_messages': ['boom'], + 'handled_at': '2019-06-16T10:23:31.607Z', +} + + +class Unserializable: + """A value no JSON serializer can encode without a stringification fallback.""" + + def __str__(self) -> str: + return 'unserializable' + + +async def test_add_request_matches_update_request_casing_async(httpserver: HTTPServer) -> None: + """The same snake_case request dict reaches the API camelCased on add_request just as on update_request.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClientAsync(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + payloads = list[bytes]() + httpserver.expect_request(re.compile(r'.*')).respond_with_handler( + _payload_capturing_handler(payloads, _REQUEST_REGISTRATION_RESPONSE_CONTENT) + ) + rq_client = client.request_queue(request_queue_id='whatever') + + await rq_client.add_request(_FULL_REQUEST_DICT) + await rq_client.update_request(_FULL_REQUEST_DICT) + + added, updated = (json.loads(payload) for payload in payloads) + assert added == updated + assert added['userData'] == {'label': 'DETAIL'} + assert [key for key in added if '_' in key] == [] + assert added['handledAt'] == '2019-06-16T10:23:31.607000Z' + + +def test_add_request_matches_update_request_casing_sync(httpserver: HTTPServer) -> None: + """The same snake_case request dict reaches the API camelCased on add_request just as on update_request.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClient(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + payloads = list[bytes]() + httpserver.expect_request(re.compile(r'.*')).respond_with_handler( + _payload_capturing_handler(payloads, _REQUEST_REGISTRATION_RESPONSE_CONTENT) + ) + rq_client = client.request_queue(request_queue_id='whatever') + + rq_client.add_request(_FULL_REQUEST_DICT) + rq_client.update_request(_FULL_REQUEST_DICT) + + added, updated = (json.loads(payload) for payload in payloads) + assert added == updated + assert added['userData'] == {'label': 'DETAIL'} + assert [key for key in added if '_' in key] == [] + assert added['handledAt'] == '2019-06-16T10:23:31.607000Z' + + +async def test_batch_add_requests_camel_cases_every_field_async(httpserver: HTTPServer) -> None: + """Every field of a snake_case request dict is camelCased in the batch-add payload.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClientAsync(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + payloads = list[bytes]() + httpserver.expect_request(re.compile(r'.*'), method='POST').respond_with_handler( + _payload_capturing_handler(payloads) + ) + rq_client = client.request_queue(request_queue_id='whatever') + + await rq_client.batch_add_requests(requests=[_FULL_REQUEST_DICT]) + + (sent_request,) = json.loads(payloads[0]) + assert sent_request['userData'] == {'label': 'DETAIL'} + assert [key for key in sent_request if '_' in key] == [] + assert sent_request['handledAt'] == '2019-06-16T10:23:31.607000Z' + + +def test_batch_add_requests_camel_cases_every_field_sync(httpserver: HTTPServer) -> None: + """Every field of a snake_case request dict is camelCased in the batch-add payload.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClient(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + payloads = list[bytes]() + httpserver.expect_request(re.compile(r'.*'), method='POST').respond_with_handler( + _payload_capturing_handler(payloads) + ) + rq_client = client.request_queue(request_queue_id='whatever') + + rq_client.batch_add_requests(requests=[_FULL_REQUEST_DICT]) + + (sent_request,) = json.loads(payloads[0]) + assert sent_request['userData'] == {'label': 'DETAIL'} + assert [key for key in sent_request if '_' in key] == [] + assert sent_request['handledAt'] == '2019-06-16T10:23:31.607000Z' + + +async def test_add_request_stringifies_unserializable_user_data_async(httpserver: HTTPServer) -> None: + """A `user_data` value JSON cannot represent is stringified instead of aborting the call.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClientAsync(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + payloads = list[bytes]() + httpserver.expect_request(re.compile(r'.*')).respond_with_handler( + _payload_capturing_handler(payloads, _REQUEST_REGISTRATION_RESPONSE_CONTENT) + ) + rq_client = client.request_queue(request_queue_id='whatever') + + await rq_client.add_request( + {'unique_key': 'http://example.com/1', 'url': 'http://example.com/1', 'user_data': {'tag': Unserializable()}} + ) + + assert json.loads(payloads[0])['userData'] == {'tag': 'unserializable'} + + +def test_add_request_stringifies_unserializable_user_data_sync(httpserver: HTTPServer) -> None: + """A `user_data` value JSON cannot represent is stringified instead of aborting the call.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClient(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + payloads = list[bytes]() + httpserver.expect_request(re.compile(r'.*')).respond_with_handler( + _payload_capturing_handler(payloads, _REQUEST_REGISTRATION_RESPONSE_CONTENT) + ) + rq_client = client.request_queue(request_queue_id='whatever') + + rq_client.add_request( + {'unique_key': 'http://example.com/1', 'url': 'http://example.com/1', 'user_data': {'tag': Unserializable()}} + ) + + assert json.loads(payloads[0])['userData'] == {'tag': 'unserializable'} + + +async def test_batch_add_requests_stringifies_unserializable_user_data_async(httpserver: HTTPServer) -> None: + """A `user_data` value JSON cannot represent is stringified instead of aborting the call.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClientAsync(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + payloads = list[bytes]() + httpserver.expect_request(re.compile(r'.*'), method='POST').respond_with_handler( + _payload_capturing_handler(payloads) + ) + rq_client = client.request_queue(request_queue_id='whatever') + + await rq_client.batch_add_requests( + requests=[ + { + 'unique_key': 'http://example.com/1', + 'url': 'http://example.com/1', + 'user_data': {'tag': Unserializable()}, + } + ] + ) + + (sent_request,) = json.loads(payloads[0]) + assert sent_request['userData'] == {'tag': 'unserializable'} + + +def test_batch_add_requests_stringifies_unserializable_user_data_sync(httpserver: HTTPServer) -> None: + """A `user_data` value JSON cannot represent is stringified instead of aborting the call.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClient(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + payloads = list[bytes]() + httpserver.expect_request(re.compile(r'.*'), method='POST').respond_with_handler( + _payload_capturing_handler(payloads) + ) + rq_client = client.request_queue(request_queue_id='whatever') + + rq_client.batch_add_requests( + requests=[ + { + 'unique_key': 'http://example.com/1', + 'url': 'http://example.com/1', + 'user_data': {'tag': Unserializable()}, + } + ] + ) + + (sent_request,) = json.loads(payloads[0]) + assert sent_request['userData'] == {'tag': 'unserializable'} + + +async def test_batch_delete_requests_stringifies_unserializable_extra_async(httpserver: HTTPServer) -> None: + """An extra field JSON cannot represent is stringified instead of aborting the call.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClientAsync(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + payloads = list[bytes]() + httpserver.expect_request(re.compile(r'.*'), method='DELETE').respond_with_handler( + _payload_capturing_handler(payloads) + ) + rq_client = client.request_queue(request_queue_id='whatever') + + request = RequestDraftDelete.model_validate({'id': 'YiKoxjkaS9gjGTqhF', 'weird': Unserializable()}) + await rq_client.batch_delete_requests(requests=[request]) + + (sent_request,) = json.loads(payloads[0]) + assert sent_request == {'id': 'YiKoxjkaS9gjGTqhF', 'weird': 'unserializable'} + + +def test_batch_delete_requests_stringifies_unserializable_extra_sync(httpserver: HTTPServer) -> None: + """An extra field JSON cannot represent is stringified instead of aborting the call.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClient(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + payloads = list[bytes]() + httpserver.expect_request(re.compile(r'.*'), method='DELETE').respond_with_handler( + _payload_capturing_handler(payloads) + ) + rq_client = client.request_queue(request_queue_id='whatever') + + request = RequestDraftDelete.model_validate({'id': 'YiKoxjkaS9gjGTqhF', 'weird': Unserializable()}) + rq_client.batch_delete_requests(requests=[request]) + + (sent_request,) = json.loads(payloads[0]) + assert sent_request == {'id': 'YiKoxjkaS9gjGTqhF', 'weird': 'unserializable'} + + +async def test_add_request_requires_unique_key_and_url_async(httpserver: HTTPServer) -> None: + """`RequestDraftDict` marks no key required, so Pydantic is the only guard left on `unique_key` and `url`.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClientAsync(token='placeholder_token', api_url=server_url, api_public_url=server_url) + rq_client = client.request_queue(request_queue_id='whatever') + + with pytest.raises(ValidationError) as exc_info: + await rq_client.add_request({'method': 'GET'}) + + assert {error['loc'][0] for error in exc_info.value.errors()} == {'uniqueKey', 'url'} + assert httpserver.log == [] + + +def test_add_request_requires_unique_key_and_url_sync(httpserver: HTTPServer) -> None: + """`RequestDraftDict` marks no key required, so Pydantic is the only guard left on `unique_key` and `url`.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClient(token='placeholder_token', api_url=server_url, api_public_url=server_url) + rq_client = client.request_queue(request_queue_id='whatever') + + with pytest.raises(ValidationError) as exc_info: + rq_client.add_request({'method': 'GET'}) + + assert {error['loc'][0] for error in exc_info.value.errors()} == {'uniqueKey', 'url'} + assert httpserver.log == [] diff --git a/tests/unit/test_postprocess_generated_models.py b/tests/unit/test_postprocess_generated_models.py index c652ba5a..14c0e4ed 100644 --- a/tests/unit/test_postprocess_generated_models.py +++ b/tests/unit/test_postprocess_generated_models.py @@ -7,11 +7,19 @@ add_docs_group_decorators, build_alias_map, convert_enums_to_literals, + drop_inherited_typeddict_fields, fix_discriminators, + reparent_classes, split_literals_to_file, ) -from apify_client._models import Request +from apify_client._models import Request, RequestBase, RequestDraft +from apify_client._typeddicts import ( + RequestBaseCamelDict, + RequestBaseDict, + RequestDraftCamelDict, + RequestDraftDict, +) # -- fix_discriminators ------------------------------------------------------- @@ -678,3 +686,82 @@ def test_add_camel_case_typeddicts_camel_validates_with_pydantic() -> None: 'user_data': {'tag': 'x'}, } assert Request.model_validate(camel_payload) == Request.model_validate(snake_payload) + + +def test_reparent_classes_swaps_base_class() -> None: + """A mapped class gets the configured base in place of the generated one.""" + content = 'class RequestDraft(BaseModel):\n url: str\n' + result = reparent_classes(content) + assert result == 'class RequestDraft(RequestBase):\n url: str\n' + + +def test_reparent_classes_is_idempotent() -> None: + """Re-running on already-reparented source leaves it unchanged.""" + content = 'class RequestDraft(RequestBase):\n url: str\n' + assert reparent_classes(content) == content + + +def test_reparent_classes_skips_absent_class() -> None: + """Source without any mapped class passes through unchanged.""" + content = 'class Foo(BaseModel):\n name: str\n' + assert reparent_classes(content) == content + + +def test_reparent_classes_leaves_name_prefixed_classes() -> None: + """A class whose name merely starts with a mapped name keeps its own base.""" + content = 'class RequestDraft(TypedDict):\n url: str\n\n\nclass RequestDraftDelete(TypedDict):\n id: str\n' + result = reparent_classes(content) + assert 'class RequestDraft(RequestBase):' in result + assert 'class RequestDraftDelete(TypedDict):' in result + + +def test_drop_inherited_typeddict_fields_removes_shadowed_keys() -> None: + """A reparented TypedDict keeps only the keys its new base does not declare, docstrings included.""" + content = textwrap.dedent("""\ + class RequestBase(TypedDict): + unique_key: NotRequired[str] + url: NotRequired[str] + + class RequestDraft(RequestBase): + id: NotRequired[str] + \""" + A unique identifier. + \""" + unique_key: str + \""" + A unique key. + \""" + url: str + """) + result = drop_inherited_typeddict_fields(content) + assert ' id: NotRequired[str]' in result + assert 'A unique identifier.' in result + assert ' unique_key: str' not in result + assert 'A unique key.' not in result + assert ' url: str' not in result + + +def test_drop_inherited_typeddict_fields_leaves_unreparented_source() -> None: + """Source whose mapped class has no base to inherit from passes through unchanged.""" + content = 'class RequestDraft(TypedDict):\n unique_key: str\n' + assert drop_inherited_typeddict_fields(content) == content + + +def test_reparented_draft_typeddicts_declare_full_shape() -> None: + """The reparented request TypedDicts expose every `RequestBase` key, snake_case and camelCase alike.""" + assert set(RequestBaseDict.__annotations__) < set(RequestDraftDict.__annotations__) + assert set(RequestBaseCamelDict.__annotations__) < set(RequestDraftCamelDict.__annotations__) + assert 'user_data' in RequestDraftDict.__annotations__ + assert 'userData' in RequestDraftCamelDict.__annotations__ + + +def test_reparented_draft_serializes_camel_case() -> None: + """`RequestDraft` inherits the full wire shape, so every field it accepts is aliased on serialization.""" + payload = {'unique_key': 'GET|abc', 'url': 'https://example.com', 'user_data': {'tag': 'x'}, 'no_retry': True} + assert issubclass(RequestDraft, RequestBase) + assert RequestDraft.model_validate(payload).model_dump(by_alias=True, exclude_none=True) == { + 'uniqueKey': 'GET|abc', + 'url': 'https://example.com', + 'userData': {'tag': 'x'}, + 'noRetry': True, + }