diff --git a/.gitignore b/.gitignore index 6835e1e631..ac4466df53 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,11 @@ node_modules # TODO: these are tmp files generated by unit tests. They should go to the /tmp directory. - + Template string evaluating when to stop paginating. Compare last_page_size against page_size rather than + against a hardcoded number: page_size is the page size that was actually requested, so the condition stays + correct when page_size_reduction shrinks it. It is only bound when this strategy declares page_size, and a + condition that reads it without one is rejected, so add page_size alongside the condition. Testing the page + for emptiness with last_page_size == 0 is equally safe and needs no page_size. A stream that enables + page_size_reduction is rejected when its stop condition compares last_page_size against anything else, + since a full page at a reduced size would then read as a short page. type: string interpolation_context: - config - headers + - last_page_size - last_record + - page_size - response examples: - "{{ response.data.has_more is false }}" - "{{ 'next' not in headers['link'] }}" + - "{{ last_page_size < page_size }}" $parameters: type: object additionalProperties: true @@ -2658,6 +2669,7 @@ definitions: - RESET_PAGINATION - RATE_LIMITED - REFRESH_TOKEN_THEN_RETRY + - REDUCE_PAGE_SIZE examples: - SUCCESS - FAIL @@ -2666,6 +2678,7 @@ definitions: - RESET_PAGINATION - RATE_LIMITED - REFRESH_TOKEN_THEN_RETRY + - REDUCE_PAGE_SIZE failure_type: title: Failure Type description: Failure type of traced exception if a response matches the filter. @@ -4223,6 +4236,15 @@ definitions: pagination_reset: description: Describes what triggers pagination reset and how to handle it. "$ref": "#/definitions/PaginationReset" + page_size_reduction: + description: >- + Describes how the page size is reduced when an error handler resolves to the REDUCE_PAGE_SIZE action. + Requires a DefaultPaginator that defines both page_size_option and a pagination strategy with a page_size. + Cannot be combined with query properties, a file uploader, or a parent stream read lazily through + lazy_read_pointer, because in those cases records of the failing page have already been emitted and + re-issuing the page would emit them twice. A page_token_option of type RequestPath is rejected as + well, because the next page is then a URL built by the API which already carries the page size. + "$ref": "#/definitions/PageSizeReduction" ignore_stream_slicer_parameters_on_paginated_requests: description: If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored. type: boolean @@ -4278,6 +4300,131 @@ definitions: enum: [PaginationResetLimits] number_of_records: type: integer + PageSizeReduction: + title: Page Size Reduction + description: >- + Describes how the page size is reduced when an error handler resolves to the REDUCE_PAGE_SIZE action. On such + a response, the connector re-issues the same page with a smaller page size instead of retrying the identical + request. Only supported on a DefaultPaginator that has a page_size_option and whose pagination_strategy + defines a page_size and is CursorPagination, OffsetIncrement, or a CustomPaginationStrategy whose + next_page_token accepts a page_size_override keyword argument. PageIncrement is not supported because a + smaller page size moves every following page boundary and would skip records. It is also rejected when the + stream uses query properties, a file uploader, or reads its parent stream lazily through lazy_read_pointer, + since re-issuing a page whose records were already emitted would duplicate them. A page_token_option of + type RequestPath is rejected too: the next page is then a URL built by the API which already carries the + page size, so the reduced page size would be sent next to the original one. Each reduction waits a + short, growing amount of time before re-issuing the page so that an endpoint failing at every page size + is not hit in a burst. That wait is the CDK's own: the error handler's backoff_strategies and any + Retry-After header are not consulted on this path, the same way they are not for RESET_PAGINATION, and + backoff_seconds is what sets its length. + type: object + required: + - type + properties: + type: + type: string + enum: [PageSizeReduction] + reduction_factor: + title: Reduction Factor + description: Divisor applied to the page size on each reduction. The new page size is floor(current page size / reduction factor). + type: number + default: 2 + exclusiveMinimum: 1 + examples: + - 2 + - 4 + minimum_page_size: + title: Minimum Page Size + description: >- + Page size below which the connector stops reducing and fails the sync. It must be smaller than the page + size configured on the pagination strategy, otherwise no reduction could ever be applied. It is one of + two bounds on the reduction and whichever is tighter wins: an unbroken run of failing pages divides the + page size by reduction_factor at most max_attempts times, so reaching this floor in a single run needs + max_attempts of at least log(page_size / minimum_page_size) / log(reduction_factor) - with the defaults, + a page size of 1000 bottoms out at 31 records per page and a floor of 10 is never reached. Pages that + succeed in between restart the max_attempts budget, so the floor is still reachable over a partition. + type: integer + default: 1 + minimum: 1 + examples: + - 1 + - 10 + max_attempts: + title: Maximum Reduction Attempts + description: >- + Maximum number of page size reductions made in a row without a single page succeeding, before the sync + fails with a transient error. Every reduction follows a request that failed, so at most + max_attempts + 1 failing requests are issued before giving up. The budget restarts after every page + that succeeds, under either reset_policy, so it bounds the reductions needed to get a single page + through and not the number of pages a partition may have: a stream that needs a reduction every now + and then reads to the end however long it is. Under NEVER the page size also strictly decreases, so + minimum_page_size bounds the reductions of the whole partition on its own. The wait between reduction + attempts is the CDK's own - it grows with each attempt - and does not consult the error handler's + backoff_strategies or a Retry-After header. + type: integer + default: 5 + minimum: 1 + examples: + - 5 + - 10 + backoff_seconds: + title: Backoff Seconds + description: >- + Base number of seconds to wait before the page is re-issued, multiplied by the number of attempts made in + a row, so the second attempt waits twice as long as the first. A REDUCE_PAGE_SIZE response never reaches + the error handler's retry budget, backoff_strategies or a Retry-After header, so this is the only thing + spacing those requests out. Raise it on an API whose error also means "we are briefly unwell" rather than + only "your page is too big", since the default spaces the whole run of attempts over a few seconds. + type: number + default: 0.5 + minimum: 0 + examples: + - 0.5 + - 5 + retries_at_minimum_page_size: + title: Retries At Minimum Page Size + description: >- + Number of times the same page is re-issued unchanged, each after the backoff wait, once the page size + cannot be shrunk any further, before the sync fails with a transient error. The default of 0 fails on the + first response received at minimum_page_size. Raise it when the API returns the same error for a page that + is too big and for a server-side hiccup: at the floor, reducing is no longer an option but waiting still + is, and without this budget those responses end the stream on the first one. This budget is separate from + max_attempts, which only counts reductions, and it restarts on every page that succeeds. It applies however the page size arrived at the floor, whether by + reduction or because page_size was already there; a page size that minimum_page_size blocks from ever + being reduced is still reported as a configuration error, but only once this budget is spent. + type: integer + default: 0 + minimum: 0 + examples: + - 0 + - 3 + failure_message: + title: Failure Message + description: >- + Sentence appended to the error message shown to the user when the connector runs out of reductions, + either because max_attempts was reached or because the page size is already at minimum_page_size. Use + it to tell the user what they can do about it in terms of this specific API, for instance which + filter narrows the query down. Without it the message only states that the API kept rejecting every + page size the connector asked for. + type: string + examples: + - Narrow the sync down by selecting fewer fields on this stream. + - Set a more recent start date so that each page covers less data. + reset_policy: + title: Reset Policy + description: >- + When to restore the page size configured on the pagination strategy. NEVER keeps the reduced page size for + the rest of the partition. AFTER_SUCCESSFUL_PAGE restores it as soon as one page succeeds, which means + hitting the same error again on every page - use it only when the reduction is worth one extra request per + page, for instance because the configured page size usually works and only some pages are too heavy. + It only controls the page size: the max_attempts budget restarts on every page that succeeds under both + policies, so there is no limit on how many reductions a partition may make in total. What is bounded is + the reductions that get no page through. + type: string + enum: + - NEVER + - AFTER_SUCCESSFUL_PAGE + default: NEVER GzipDecoder: title: gzip description: Select 'gzip' for response data that is compressed with gzip. Requires specifying an inner data type/decoder to parse the decompressed data. diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 21a92cc3be..1608767606 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -140,10 +140,11 @@ class CursorPagination(BaseModel): ) stop_condition: Optional[str] = Field( None, - description="Template string evaluating when to stop paginating.", + description="Template string evaluating when to stop paginating. Compare last_page_size against page_size rather than against a hardcoded number: page_size is the page size that was actually requested, so the condition stays correct when page_size_reduction shrinks it. It is only bound when this strategy declares page_size, and a condition that reads it without one is rejected, so add page_size alongside the condition. Testing the page for emptiness with last_page_size == 0 is equally safe and needs no page_size. A stream that enables page_size_reduction is rejected when its stop condition compares last_page_size against anything else, since a full page at a reduced size would then read as a short page.", examples=[ "{{ response.data.has_more is false }}", "{{ 'next' not in headers['link'] }}", + "{{ last_page_size < page_size }}", ], title="Stop Condition", ) @@ -714,6 +715,7 @@ class Action(Enum): RESET_PAGINATION = "RESET_PAGINATION" RATE_LIMITED = "RATE_LIMITED" REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" + REDUCE_PAGE_SIZE = "REDUCE_PAGE_SIZE" class FailureType(Enum): @@ -735,6 +737,7 @@ class HttpResponseFilter(BaseModel): "RESET_PAGINATION", "RATE_LIMITED", "REFRESH_TOKEN_THEN_RETRY", + "REDUCE_PAGE_SIZE", ], title="Action", ) @@ -1410,6 +1413,64 @@ class PaginationResetLimits(BaseModel): number_of_records: Optional[int] = None +class ResetPolicy(Enum): + NEVER = "NEVER" + AFTER_SUCCESSFUL_PAGE = "AFTER_SUCCESSFUL_PAGE" + + +class PageSizeReduction(BaseModel): + type: Literal["PageSizeReduction"] + reduction_factor: Optional[float] = Field( + 2, + description="Divisor applied to the page size on each reduction. The new page size is floor(current page size / reduction factor).", + examples=[2, 4], + gt=1.0, + title="Reduction Factor", + ) + minimum_page_size: Optional[int] = Field( + 1, + description="Page size below which the connector stops reducing and fails the sync. It must be smaller than the page size configured on the pagination strategy, otherwise no reduction could ever be applied. It is one of two bounds on the reduction and whichever is tighter wins: an unbroken run of failing pages divides the page size by reduction_factor at most max_attempts times, so reaching this floor in a single run needs max_attempts of at least log(page_size / minimum_page_size) / log(reduction_factor) - with the defaults, a page size of 1000 bottoms out at 31 records per page and a floor of 10 is never reached. Pages that succeed in between restart the max_attempts budget, so the floor is still reachable over a partition.", + examples=[1, 10], + ge=1, + title="Minimum Page Size", + ) + max_attempts: Optional[int] = Field( + 5, + description="Maximum number of page size reductions made in a row without a single page succeeding, before the sync fails with a transient error. Every reduction follows a request that failed, so at most max_attempts + 1 failing requests are issued before giving up. The budget restarts after every page that succeeds, under either reset_policy, so it bounds the reductions needed to get a single page through and not the number of pages a partition may have: a stream that needs a reduction every now and then reads to the end however long it is. Under NEVER the page size also strictly decreases, so minimum_page_size bounds the reductions of the whole partition on its own. The wait between reduction attempts is the CDK's own - it grows with each attempt - and does not consult the error handler's backoff_strategies or a Retry-After header.", + examples=[5, 10], + ge=1, + title="Maximum Reduction Attempts", + ) + backoff_seconds: Optional[float] = Field( + 0.5, + description='Base number of seconds to wait before the page is re-issued, multiplied by the number of attempts made in a row, so the second attempt waits twice as long as the first. A REDUCE_PAGE_SIZE response never reaches the error handler\'s retry budget, backoff_strategies or a Retry-After header, so this is the only thing spacing those requests out. Raise it on an API whose error also means "we are briefly unwell" rather than only "your page is too big", since the default spaces the whole run of attempts over a few seconds.', + examples=[0.5, 5], + ge=0.0, + title="Backoff Seconds", + ) + retries_at_minimum_page_size: Optional[int] = Field( + 0, + description="Number of times the same page is re-issued unchanged, each after the backoff wait, once the page size cannot be shrunk any further, before the sync fails with a transient error. The default of 0 fails on the first response received at minimum_page_size. Raise it when the API returns the same error for a page that is too big and for a server-side hiccup: at the floor, reducing is no longer an option but waiting still is, and without this budget those responses end the stream on the first one. This budget is separate from max_attempts, which only counts reductions, and it restarts on every page that succeeds. It applies however the page size arrived at the floor, whether by reduction or because page_size was already there; a page size that minimum_page_size blocks from ever being reduced is still reported as a configuration error, but only once this budget is spent.", + examples=[0, 3], + ge=0, + title="Retries At Minimum Page Size", + ) + failure_message: Optional[str] = Field( + None, + description="Sentence appended to the error message shown to the user when the connector runs out of reductions, either because max_attempts was reached or because the page size is already at minimum_page_size. Use it to tell the user what they can do about it in terms of this specific API, for instance which filter narrows the query down. Without it the message only states that the API kept rejecting every page size the connector asked for.", + examples=[ + "Narrow the sync down by selecting fewer fields on this stream.", + "Set a more recent start date so that each page covers less data.", + ], + title="Failure Message", + ) + reset_policy: Optional[ResetPolicy] = Field( + ResetPolicy.NEVER, + description="When to restore the page size configured on the pagination strategy. NEVER keeps the reduced page size for the rest of the partition. AFTER_SUCCESSFUL_PAGE restores it as soon as one page succeeds, which means hitting the same error again on every page - use it only when the reduction is worth one extra request per page, for instance because the configured page size usually works and only some pages are too heavy. It only controls the page size: the max_attempts budget restarts on every page that succeeds under both policies, so there is no limit on how many reductions a partition may make in total. What is bounded is the reductions that get no page through.", + title="Reset Policy", + ) + + class CsvDecoder(BaseModel): type: Literal["CsvDecoder"] encoding: Optional[str] = "utf-8" @@ -3222,6 +3283,10 @@ class SimpleRetriever(BaseModel): None, description="Describes what triggers pagination reset and how to handle it.", ) + page_size_reduction: Optional[PageSizeReduction] = Field( + None, + description="Describes how the page size is reduced when an error handler resolves to the REDUCE_PAGE_SIZE action. Requires a DefaultPaginator that defines both page_size_option and a pagination strategy with a page_size. Cannot be combined with query properties, a file uploader, or a parent stream read lazily through lazy_read_pointer, because in those cases records of the failing page have already been emitted and re-issuing the page would emit them twice. A page_token_option of type RequestPath is rejected as well, because the next page is then a URL built by the API which already carries the page size.", + ) ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( False, description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index aa546c73ab..6587b77ff0 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -9,6 +9,7 @@ import inspect import json import logging +import math import re from functools import partial from typing import ( @@ -138,6 +139,9 @@ DEPRECATION_LOGS_TAG, BaseModelWithDeprecations, ) +from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( + Action as HttpResponseFilterActionModel, +) from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( Action1 as PaginationResetActionModel, ) @@ -384,6 +388,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( PageIncrement as PageIncrementModel, ) +from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( + PageSizeReduction as PageSizeReductionModel, +) from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( PaginationReset as PaginationResetModel, ) @@ -493,6 +500,10 @@ AirbyteCustomCodeNotPermittedError, custom_code_execution_permitted, ) +from airbyte_cdk.sources.declarative.parsers.stop_condition_safety import ( + StopConditionSafety, + classify_stop_condition, +) from airbyte_cdk.sources.declarative.partition_routers import ( CartesianProductStreamSlicer, GroupingPartitionRouter, @@ -579,6 +590,10 @@ LocalFileSystemFileWriter, NoopFileWriter, ) +from airbyte_cdk.sources.declarative.retrievers.page_size_reducer import ( + PageSizeReduction, + PageSizeResetPolicy, +) from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker from airbyte_cdk.sources.declarative.schema import ( ComplexFieldType, @@ -1197,6 +1212,9 @@ def create_legacy_to_per_partition_state_migration( def create_session_token_authenticator( self, model: SessionTokenAuthenticatorModel, config: Config, name: str, **kwargs: Any ) -> Union[ApiKeyAuthenticator, BearerAuthenticator]: + self._reject_reduce_page_size_action( + model.login_requester, f"`login_requester` of the SessionTokenAuthenticator of {name}" + ) decoder = ( self._create_component_from_model(model=model.decoder, config=config) if model.decoder @@ -3614,14 +3632,30 @@ def _get_url(req: Requester) -> str: model.ignore_stream_slicer_parameters_on_paginated_requests or False ) - if ( + reads_parent_stream_lazily = bool( model.partition_router and isinstance(model.partition_router, SubstreamPartitionRouterModel) - and not bool(self._connector_state_manager.get_stream_state(name, None)) and any( parent_stream_config.lazy_read_pointer for parent_stream_config in model.partition_router.parent_stream_configs ) + ) + if reads_parent_stream_lazily and ( + model.page_size_reduction + or self._uses_reduce_page_size_action(getattr(model.requester, "error_handler", None)) + ): + # Checked outside of the LazySimpleRetriever branch below, which only applies on the first + # sync of a stream: gating it on the absence of state would accept the same manifest from the + # second sync onwards. LazySimpleRetriever paginates the parent's embedded pages, so there is + # no page of its own to re-issue with a smaller page size. + raise ValueError( + f"`page_size_reduction` and the REDUCE_PAGE_SIZE response action are not supported when " + f"reading a parent stream lazily. Remove either the page size reduction or the parent " + f"stream's `lazy_read_pointer` for stream {name}." + ) + + if reads_parent_stream_lazily and not bool( + self._connector_state_manager.get_stream_state(name, None) ): if incremental_sync: if incremental_sync.type != "DatetimeBasedCursor": @@ -3675,10 +3709,314 @@ def _get_url(req: Requester) -> str: pagination_tracker_factory=self._create_pagination_tracker_factory( model.pagination_reset, cursor ), + page_size_reduction=self._create_page_size_reduction( + model, name, query_properties, file_uploader + ), post_pagination_filter=post_pagination_filter, parameters=model.parameters or {}, ) + def _create_page_size_reduction( + self, + model: SimpleRetrieverModel, + name: str, + query_properties: Optional[QueryProperties], + file_uploader: Optional[DefaultFileUploader] = None, + ) -> Optional[PageSizeReduction]: + # A CustomRequester does not necessarily define an error handler. A CustomRequester that does define + # one keeps it as a raw dict rather than a typed model, so this returns False for it as well. + error_handler = getattr(model.requester, "error_handler", None) + uses_action = self._uses_reduce_page_size_action(error_handler) + if uses_action and not model.page_size_reduction: + raise ValueError( + f"Stream {name} has a response filter with the REDUCE_PAGE_SIZE action but the retriever does not " + f"define `page_size_reduction`. Add a `page_size_reduction` block to the retriever." + ) + + if not model.page_size_reduction: + return None + + if not uses_action: + # Not raised: a CustomErrorHandler can resolve to REDUCE_PAGE_SIZE without us being able to see + # it, so the only safe reaction to a block we cannot tie to an action is a warning. Without it a + # misspelled action leaves the feature silently dead on a stream that only exists because it + # would otherwise fail. + LOGGER.warning( + f"Stream {name} defines `page_size_reduction` but no response filter with the " + f"REDUCE_PAGE_SIZE action was found on its requester. The page size will never be reduced " + f"unless a custom error handler resolves to that action." + ) + + self._validate_page_size_reduction_is_supported( + model, name, query_properties, file_uploader + ) + + reset_policy = model.page_size_reduction.reset_policy + return PageSizeReduction( + reduction_factor=model.page_size_reduction.reduction_factor, # type: ignore[arg-type] # the schema defines a default + minimum_page_size=model.page_size_reduction.minimum_page_size, # type: ignore[arg-type] # the schema defines a default + max_attempts=model.page_size_reduction.max_attempts, # type: ignore[arg-type] # the schema defines a default + backoff_seconds=model.page_size_reduction.backoff_seconds, # type: ignore[arg-type] # the schema defines a default + retries_at_minimum_page_size=model.page_size_reduction.retries_at_minimum_page_size, # type: ignore[arg-type] # the schema defines a default + failure_message=model.page_size_reduction.failure_message, + reset_policy=PageSizeResetPolicy(reset_policy.value) + if reset_policy is not None + else PageSizeResetPolicy.NEVER, + ) + + def _validate_page_size_reduction_is_supported( + self, + model: SimpleRetrieverModel, + name: str, + query_properties: Optional[QueryProperties], + file_uploader: Optional[DefaultFileUploader] = None, + ) -> None: + """ + Page size reduction re-issues the same page with a smaller page size. That is only correct when the next + page does not depend on the page size, and it only has an effect when the paginator injects the page size + in the request. A custom pagination strategy is accepted when it can receive the reduced page size, which + is checked by inspecting its signature rather than by recognizing its type. + """ + if query_properties: + raise ValueError( + f"`page_size_reduction` cannot be used together with query properties on stream {name}. Records " + f"from the earlier property chunks have already been emitted when a chunk asks for a smaller page, " + f"so retrying the page would emit them twice." + ) + + if file_uploader: + raise ValueError( + f"`page_size_reduction` cannot be used together with a `file_uploader` on stream {name}. The " + f"file uploader sends one request per record from inside the page's record generator, so a " + f"reduction asked for halfway through a page would re-emit the records already yielded by it." + ) + + if not isinstance(model.paginator, DefaultPaginatorModel): + raise ValueError( + f"`page_size_reduction` requires a DefaultPaginator on stream {name} so that the connector can " + f"send a smaller page size." + ) + + if not model.paginator.page_size_option: + raise ValueError( + f"`page_size_reduction` requires `page_size_option` on the paginator of stream {name}: without it " + f"the connector cannot tell the API to send a smaller page." + ) + + if isinstance(model.paginator.page_token_option, RequestPathModel): + # A RequestPath page token is a full URL built by the API, and it already carries the page size the + # API echoed back. The reduced page size is injected as a request option on top of that URL, so the + # request goes out with the page size twice - the original one from the URL and the reduced one - + # and which of the two the API honors is up to the API. Every page after the first would then keep + # asking for the page size that just failed. + raise ValueError( + f"`page_size_reduction` does not support a `page_token_option` of type RequestPath on stream " + f"{name}. The next page is then requested through a URL returned by the API, which already " + f"carries the page size, so the reduced page size would be sent alongside the original one. Use " + f"a CursorPagination strategy with a `page_token_option` of type RequestOption instead." + ) + + strategy = model.paginator.pagination_strategy + if isinstance(strategy, PageIncrementModel): + raise ValueError( + f"`page_size_reduction` does not support the PageIncrement pagination strategy used by stream " + f"{name}. Pages are addressed as page number * page size, so a smaller page size shifts every " + f"following page boundary and would skip records. Use OffsetIncrement or CursorPagination." + ) + if isinstance(strategy, CustomPaginationStrategyModel): + # A custom strategy is written by the same person enabling the reduction, so the + # question is not whether we recognize it but whether it can be told the reduced + # page size. Checking the signature keeps a strategy that would raise TypeError + # mid-sync from being accepted at config time. + custom_class = self._get_class_from_fully_qualified_class_name(strategy.class_name) + if isinstance(custom_class, type) and issubclass(custom_class, PageIncrement): + # `PageIncrement.next_page_token` declares `page_size_override` only to reject it, so a + # subclass that does not override the method would pass the signature check below and only + # fail once the first reduction is requested, mid-sync. + raise ValueError( + f"`page_size_reduction` does not support the PageIncrement pagination strategy that the " + f"custom pagination strategy {strategy.class_name} used by stream {name} inherits from. " + f"Pages are addressed as page number * page size, so a smaller page size shifts every " + f"following page boundary and would skip records. Use OffsetIncrement or CursorPagination." + ) + try: + parameters = inspect.signature(custom_class.next_page_token).parameters + except (AttributeError, TypeError, ValueError) as exception: + raise ValueError( + f"`page_size_reduction` could not check the signature of `next_page_token` on the custom " + f"pagination strategy {strategy.class_name} used by stream {name}: {exception}. Make sure " + f"`class_name` points at a PaginationStrategy subclass whose `next_page_token` accepts a " + f"`page_size_override` keyword argument." + ) + accepts_override = "page_size_override" in parameters or any( + parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in parameters.values() + ) + if not accepts_override: + raise ValueError( + f"`page_size_reduction` requires the custom pagination strategy " + f"{strategy.class_name} used by stream {name} to accept a `page_size_override` keyword " + f"argument in `next_page_token`, so that it can honor the reduced page size. Add " + f"`page_size_override: Optional[int] = None` to its signature; a strategy that does not " + f"use its page size as a stop condition can ignore the value." + ) + elif not isinstance(strategy, (CursorPaginationModel, OffsetIncrementModel)): + raise ValueError( + f"`page_size_reduction` only supports the CursorPagination, OffsetIncrement and " + f"CustomPaginationStrategy pagination strategies. Stream {name} uses " + f"{type(strategy).__name__}." + ) + else: + # A CustomPaginationStrategy carries its page size in its own code, so this is only checkable + # for the strategies the CDK defines. `page_size` is optional on both of them, and without it + # `get_page_size` returns None, the paginator injects nothing, and the reduction is a dead end + # that only surfaces on the first failing response - after records have been emitted. + self._validate_page_size_is_reducible(strategy, model.page_size_reduction, name) + if isinstance(strategy, CursorPaginationModel): + self._validate_stop_condition_is_reduction_aware( + strategy, model.page_size_reduction, name + ) + + @staticmethod + def _validate_stop_condition_is_reduction_aware( + strategy: CursorPaginationModel, + page_size_reduction: Optional[PageSizeReductionModel], + name: str, + ) -> None: + """ + A `stop_condition` comparing `last_page_size` to a hardcoded page size reads a full reduced page as a + short page and ends the pagination early, dropping the rest of the partition without failing. The + strategy exposes the page size that was actually requested as `page_size`, so the condition can be + written correctly - but only if it is, which is what this checks. + + The condition is parsed as a Jinja expression rather than matched as a string: only the AST tells + `page_size`, which follows the reduction, apart from `config['page_size']`, which does not, and only the + AST tells an inequality, which a reduction can invalidate, apart from `last_page_size == 0`, which it + cannot. A condition that never names `last_page_size` is not waved through: a count read from the + response body - `{{ response['data'] | length < 100 }}` - truncates in exactly the same way, and which + response field counts the records of a page is not knowable here. A shape the analysis does not + understand is warned about rather than rejected - this runs at stream construction, so a false + rejection takes `check`, `discover` and `read` down with it. + """ + stop_condition = strategy.stop_condition + if not stop_condition: + return + + minimum_page_size = ( + page_size_reduction.minimum_page_size if page_size_reduction else None + ) or 1 + verdict, reason = classify_stop_condition(stop_condition, minimum_page_size) + if verdict is StopConditionSafety.TRUNCATES: + raise ValueError( + f"`page_size_reduction` on stream {name} cannot be used with the `stop_condition` " + f"{stop_condition!r}: {reason}. The pagination would then end early, silently dropping the " + f"rest of the partition. Compare against the `page_size` interpolation variable instead, " + f"which holds the page size that was actually requested (for example " + f"`{{{{ last_page_size < page_size }}}}`), or test the page for emptiness with " + f"`{{{{ last_page_size == 0 }}}}`." + ) + if verdict is StopConditionSafety.UNKNOWN: + LOGGER.warning( + f"Stream {name} uses `page_size_reduction` with the `stop_condition` {stop_condition!r}, " + f"which could not be checked against the reduction because {reason}. Make sure a page that is " + f"full at a reduced page size does not satisfy it, otherwise the pagination ends early and the " + f"rest of the partition is silently dropped. Comparing against the `page_size` interpolation " + f"variable, which holds the page size that was actually requested, is always safe." + ) + + @staticmethod + def _validate_page_size_is_reducible( + strategy: Union[CursorPaginationModel, OffsetIncrementModel], + page_size_reduction: Optional[PageSizeReductionModel], + name: str, + ) -> None: + page_size = strategy.page_size + if page_size is None: + raise ValueError( + f"`page_size_reduction` requires `page_size` on the pagination strategy of stream {name}: " + f"without it the paginator does not send a page size, so there is nothing to reduce." + ) + + # The schema allows a string so that the page size can be interpolated. `page_size` is + # `Optional[Union[int, str]]` and pydantic v1 tries `int` first, so both `100` and `"100"` arrive as + # an int and only a genuinely non-numeric template stays a str. Such a template is only known once + # the config is available, so it is left to the runtime check in `PageSizeReducer.reduce`. + try: + configured_page_size: Optional[int] = int(page_size) + except ValueError: + configured_page_size = None + + minimum_page_size = ( + page_size_reduction.minimum_page_size if page_size_reduction else None + ) or 1 + if configured_page_size is not None and configured_page_size <= minimum_page_size: + raise ValueError( + f"`page_size_reduction` on stream {name} can never reduce its page size: the pagination " + f"strategy's `page_size` is {configured_page_size} and `minimum_page_size` is " + f"{minimum_page_size}. Lower `minimum_page_size` or raise `page_size`." + ) + + # Each reduction divides the page size by `reduction_factor` and spends one attempt, so an unbroken + # run of failures bottoms out at `page_size / reduction_factor ** max_attempts` whatever + # `minimum_page_size` says. Only warned about, and not raised: pages that succeed in between restart + # the budget while `NEVER` keeps the page size, so the floor is reachable over a partition even when + # it is out of reach of a single run - and a manifest that deliberately gives up earlier than its + # floor is not wrong, only worth pointing out. + reduction_factor = ( + page_size_reduction.reduction_factor if page_size_reduction else None + ) or 2.0 + max_attempts = (page_size_reduction.max_attempts if page_size_reduction else None) or 5 + # Only when there is a floor worth reaching. The default of 1 is out of reach of the default budget on + # any page size above 32, so this would otherwise fire on nearly every stream that opts in - and + # `__fields_set__` would not help, since it records that a value was supplied and not that it differs + # from the default, so spelling `minimum_page_size: 1` out longhand would earn the warning. + if ( + configured_page_size is not None + and minimum_page_size > 1 + and reduction_factor**max_attempts < configured_page_size / minimum_page_size + ): + reachable_page_size = max( + minimum_page_size, int(configured_page_size // reduction_factor**max_attempts) + ) + LOGGER.warning( + f"Stream {name} sets `minimum_page_size` to {minimum_page_size}, which `page_size_reduction` " + f"cannot reach in one run of failing pages: `max_attempts` is {max_attempts} and " + f"`reduction_factor` is {reduction_factor}, so {max_attempts} reductions of a page size of " + f"{configured_page_size} stop at {reachable_page_size} records per page and the sync then " + f"fails with a transient error. Whichever of the two bounds is tighter wins; reaching " + f"{minimum_page_size} in one run needs `max_attempts` of at least " + f"{math.ceil(math.log(configured_page_size / minimum_page_size, reduction_factor))}." + ) + + def _reject_reduce_page_size_action(self, requester: Any, description: str) -> None: + """ + `error_handler` is defined on `HttpRequester`, which is referenced by requesters that have no page of + their own, so REDUCE_PAGE_SIZE is schema-legal in places where nothing can honor it. Only + `SimpleRetriever._read_pages` re-issues a page, so every other requester is rejected here rather than + surfacing the exception mid-sync as a generic failure. + """ + if requester is None: + return + if self._uses_reduce_page_size_action(getattr(requester, "error_handler", None)): + raise ValueError( + f"The REDUCE_PAGE_SIZE response action is not supported on the {description}: only the main " + f"requester of a SimpleRetriever can re-issue its page with a smaller page size. Use a " + f"different action on that error handler." + ) + + def _uses_reduce_page_size_action(self, error_handler: Any) -> bool: + if isinstance(error_handler, CompositeErrorHandlerModel): + return any( + self._uses_reduce_page_size_action(nested) + for nested in error_handler.error_handlers + ) + if isinstance(error_handler, DefaultErrorHandlerModel): + return any( + response_filter.action == HttpResponseFilterActionModel.REDUCE_PAGE_SIZE + for response_filter in error_handler.response_filters or [] + ) + # A CustomErrorHandler can return any action and we cannot inspect it, so we do not validate it. + return False + def _create_pagination_tracker_factory( self, model: Optional[PaginationResetModel], cursor: Cursor ) -> Callable[[], PaginationTracker]: @@ -3946,6 +4284,19 @@ def create_async_retriever( f"`download_target_extractor` required if using a `download_target_requester`" ) + for requester_field in ( + "creation_requester", + "polling_requester", + "download_requester", + "download_target_requester", + "abort_requester", + "delete_requester", + ): + self._reject_reduce_page_size_action( + getattr(model, requester_field, None), + f"`{requester_field}` of the AsyncRetriever of stream {name}", + ) + def _get_download_retriever( requester: Requester, extractor: RecordExtractor, _decoder: Decoder ) -> SimpleRetriever: @@ -4559,6 +4910,7 @@ def create_fixed_window_call_rate_policy( def create_file_uploader( self, model: FileUploaderModel, config: Config, **kwargs: Any ) -> FileUploader: + self._reject_reduce_page_size_action(model.requester, "requester of a `file_uploader`") name = "File Uploader" requester = self._create_component_from_model( model=model.requester, diff --git a/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py b/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py new file mode 100644 index 0000000000..9ec61ce2b1 --- /dev/null +++ b/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py @@ -0,0 +1,407 @@ +# +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. +# + +""" +Config-time analysis of a `CursorPagination` `stop_condition` against a page size reduction. + +A condition that infers "this was the last page" from a short page is only correct relative to the page size +that was actually requested. When `page_size_reduction` shrinks that size, a *full* page at the reduced size +reads as a short page against the configured size, the pagination ends, and the rest of the partition is +dropped without anything failing. + +Matching the expression as a string cannot tell the reduction-aware `page_size` variable apart from a +config-derived constant - `\\bpage_size\\b` matches inside `config['page_size']` just as well - and it cannot +tell an inequality, which a reduction can invalidate, apart from `last_page_size == 0`, which no reduction can. +So the expression is parsed with Jinja and the comparisons involving `last_page_size` are inspected on the AST, +where a bare `Name` node is distinguishable from a `Getitem` on `config`. + +The safety rule is a single question: can the condition be true for a page that is *full at the size that was +requested*? A full page holds exactly `requested_page_size` records, and a reduction only ever lowers that +number, so: + +- `last_page_size == 0` and `last_page_size != 0` cannot change verdict: a full page is never empty, because + `minimum_page_size` is at least 1. +- `last_page_size > x` and `last_page_size >= x` can only stop being true as the page shrinks, so a reduction + cannot introduce a stop that would not have happened anyway. +- `last_page_size < x` and `last_page_size <= x` are the dangerous shape. They are safe only when `x` is the + `page_size` variable, which follows the reduction, or when `x` is a literal at or below `minimum_page_size`, + which makes the comparison a rewrite of "the page is empty". An expression merely built from `page_size`, + such as `[page_size, 100] | max` or `page_size + 50`, does not count: it can hold the configured size again. + +Each verdict above reads the comparison on its own, which is only sound while the comparison decides the +condition in its own polarity. `{{ not (last_page_size >= 100) }}` means `last_page_size < 100`, and +`{{ last_page_size - 100 < 0 }}` means the same again, so the operator and the operands of the comparison no +longer say what the condition does. A comparison is therefore only classified when the truth of the condition follows its own: reached from the +root through `and`/`or`, or as the test of a `{% if %}` that renders truthy text and nothing else, and with +`last_page_size` compared bare rather than transformed first. Anything else is unclassifiable. + +A condition that never names `last_page_size` is not safe by default either: the response body carries the +same count, so `{{ response['data'] | length < 100 }}` truncates in exactly the same way. Which of a +connector's own response fields counts the records of a page is not knowable here, so any comparison that +bounds something from above is reported as unclassifiable. A condition with no such comparison, such as +`{{ not response.next }}`, cannot read a page length by comparison and is safe. + +Anything else is reported as unclassifiable rather than as a truncation: this analysis rejects a manifest at +stream construction, so a shape it does not understand must not be treated as a defect. +""" + +from enum import Enum +from typing import Iterator, List, Optional, Tuple + +from jinja2 import nodes +from jinja2.environment import Environment +from jinja2.exceptions import TemplateSyntaxError + +from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import FALSE_VALUES + +LAST_PAGE_SIZE_VARIABLE = "last_page_size" +REQUESTED_PAGE_SIZE_VARIABLE = "page_size" + +# Parsing is all this environment is used for: no filter, test or global is resolved, so the plain environment +# parses everything the interpolation environment accepts. +_PARSING_ENVIRONMENT = Environment() + +_MIRRORED_OPERATORS = { + "lt": "gt", + "lteq": "gteq", + "gt": "lt", + "gteq": "lteq", + "eq": "eq", + "ne": "ne", +} + + +class StopConditionSafety(Enum): + """Whether a page size reduction can change what a `stop_condition` decides.""" + + SAFE = "SAFE" + """No reduction can make the condition stop the pagination earlier than it already would.""" + + TRUNCATES = "TRUNCATES" + """A full page at a reduced size satisfies the condition, so the partition would be cut short.""" + + UNKNOWN = "UNKNOWN" + """The condition uses `last_page_size` in a shape this analysis cannot reason about.""" + + +def classify_stop_condition( + stop_condition: str, minimum_page_size: int +) -> Tuple[StopConditionSafety, str]: + """ + :param stop_condition: the raw `stop_condition` template from the manifest + :param minimum_page_size: the smallest page size the reduction is allowed to request + :return: the verdict and a human readable reason for it + """ + try: + template = _PARSING_ENVIRONMENT.parse(stop_condition) + except TemplateSyntaxError as exception: + return ( + StopConditionSafety.UNKNOWN, + f"it is not a valid Jinja expression ({exception.message})", + ) + + if not _references(template, LAST_PAGE_SIZE_VARIABLE): + return _classify_without_last_page_size(template) + + unclassifiable: List[str] = [] + understood = 0 + for left, operator, right, decides_condition in _comparisons(template): + if not _references(left, LAST_PAGE_SIZE_VARIABLE) and not _references( + right, LAST_PAGE_SIZE_VARIABLE + ): + continue + if not decides_condition: + # The comparison is negated or is an operand of a larger expression, so its own operator no longer + # says what the condition does and neither verdict below would be about the right question. + unclassifiable.append( + f"it uses `{LAST_PAGE_SIZE_VARIABLE}` in a comparison that does not decide the condition on " + f"its own, such as one under a `not` or inside a larger expression" + ) + continue + verdict, reason = _classify_comparison(left, operator, right, minimum_page_size) + if verdict is StopConditionSafety.TRUNCATES: + return verdict, reason + if verdict is StopConditionSafety.UNKNOWN: + unclassifiable.append(reason) + else: + understood += 1 + + if unclassifiable: + return StopConditionSafety.UNKNOWN, unclassifiable[0] + if understood: + return ( + StopConditionSafety.SAFE, + f"every comparison it makes against `{LAST_PAGE_SIZE_VARIABLE}` holds whatever page size was requested", + ) + return ( + StopConditionSafety.UNKNOWN, + f"it uses `{LAST_PAGE_SIZE_VARIABLE}` outside of a comparison", + ) + + +def _classify_without_last_page_size( + template: nodes.Template, +) -> Tuple[StopConditionSafety, str]: + """ + Classify a condition that never names `last_page_size`. + + `last_page_size` is not the only way to observe how many records a page held: the response body carries + the same number, and `{{ response['data'] | length < 100 }}` is the comparison `{{ last_page_size < 100 }}` + counted one layer out. The CDK cannot tell which of a connector's own response fields counts the records + of the page, so a condition that bounds *anything* from above is reported as unclassifiable and warned + about rather than assumed to be safe. A condition that makes no such comparison - the common + `{{ not response.next }}` shape - cannot read a page length by comparison at all, so it stays safe. + """ + for left, operator, right, decides_condition in _comparisons(template): + bounded = _bounded_from_above(left, operator, right, decides_condition) + if bounded is not None: + return ( + StopConditionSafety.UNKNOWN, + f"it stops when {_describe(bounded)} stays below a threshold. `{LAST_PAGE_SIZE_VARIABLE}` is " + f"not the only way to count the records of a page - a count read from the response body is " + f"another - and the connector cannot tell whether this one does, so a full page at a reduced " + f"size may read as a short page here", + ) + return ( + StopConditionSafety.SAFE, + f"it does not use `{LAST_PAGE_SIZE_VARIABLE}` and makes no comparison that could be bounding the " + f"number of records in a page from above", + ) + + +def _bounded_from_above( + left: nodes.Node, operator: str, right: nodes.Node, decides_condition: bool +) -> Optional[nodes.Node]: + """ + :return: the operand the comparison bounds from above, or None when it bounds nothing from above + + An upper bound is the shape a reduction can invalidate, for the same reason it can invalidate + `last_page_size < 100`: the reduction lowers what a full page holds, so the bound starts holding for pages + that are not short at all. A lower bound can only stop being satisfied as pages get smaller. When the + comparison does not decide the condition in its own polarity, which of the two it is cannot be read off + the operator, so it counts as an upper bound. + + Every comparison with one of these four operators bounds one of its operands from above - `a < b` bounds + `a`, `b > a` bounds `a` too - and a literal cannot be a page length, so the bounded side only escapes when + it is one. The two operator directions are therefore mirrored, the way `_MIRRORED_OPERATORS` mirrors them + on the `last_page_size` path: reading only `lt`/`lteq` in full left + `{{ config['page_size'] > response['data'] | length }}` classified as safe while + `{{ response['data'] | length < config['page_size'] }}` was warned about. + """ + if operator not in ("lt", "lteq", "gt", "gteq"): + return None + if not decides_condition: + return left if not isinstance(left, nodes.Const) else right + if operator in ("lt", "lteq"): + # `x < 100` bounds x from above; `100 < x` bounds the literal, which is not a page length. + return None if isinstance(left, nodes.Const) else left + # `100 > x` is the mirror of `x < 100`; `x > 100` bounds the literal again. + return None if isinstance(right, nodes.Const) else right + + +def _comparisons( + node: nodes.Node, decides_condition: bool = True +) -> Iterator[Tuple[nodes.Node, str, nodes.Node, bool]]: + """ + Flatten every comparison, including the chained ones, into (left, operator, right, decides_condition). + + `decides_condition` is true only for a comparison the truth of the whole condition follows directly: + reached from the root through `and`/`or` alone. Under a `not`, inside a conditional expression, piped + through a filter or used as an operand of another expression, the comparison's own operator says nothing + about what the condition decides, so it is flagged and left unclassified. + """ + if isinstance(node, nodes.Compare): + left = node.expr + for operand in node.ops: + yield left, operand.op, operand.expr, decides_condition + left = operand.expr + # A comparison nested inside an operand of this one is an ordinary sub-expression, not a decider. + for operand_node in [node.expr, *(operand.expr for operand in node.ops)]: + yield from _comparisons(operand_node, False) + return + + if isinstance(node, nodes.If): + # `{% if last_page_size < 100 %}true{% endif %}` renders truthy text exactly when its test holds, so + # the test decides the condition. That only follows while the branch it guards is the whole story: + # an `else`, an `elif`, or a body rendering something the CDK reads as false breaks the equivalence. + test_decides = decides_condition and _if_follows_its_test(node) + yield from _comparisons(node.test, test_decides) + for branch in [*node.body, *node.elif_, *node.else_]: + yield from _comparisons(branch, False) + return + + children_decide = decides_condition and _passes_truth_through(node) + for child in node.iter_child_nodes(): + yield from _comparisons(child, children_decide) + + +def _passes_truth_through(node: nodes.Node) -> bool: + """Whether the truth of the condition follows the truth of this node's children.""" + if isinstance(node, (nodes.And, nodes.Or)): + return True + if isinstance(node, nodes.Template): + # Anything rendered next to the comparison is text of its own, which makes the rendered condition + # truthy whatever the comparison decided. + return len(node.body) == 1 + if isinstance(node, nodes.Output): + return len(node.nodes) == 1 + return False + + +def _if_follows_its_test(node: nodes.If) -> bool: + if node.elif_ or node.else_: + return False + rendered = "" + for statement in node.body: + if not isinstance(statement, nodes.Output): + return False + for output in statement.nodes: + if isinstance(output, nodes.TemplateData): + rendered += output.data + elif isinstance(output, nodes.Const): + rendered += str(output.value) + else: + return False + return rendered not in FALSE_VALUES + + +def _classify_comparison( + left: nodes.Node, operator: str, right: nodes.Node, minimum_page_size: int +) -> Tuple[StopConditionSafety, str]: + if _references(right, LAST_PAGE_SIZE_VARIABLE) and not _references( + left, LAST_PAGE_SIZE_VARIABLE + ): + left, right, operator = right, left, _MIRRORED_OPERATORS.get(operator, operator) + elif _references(left, LAST_PAGE_SIZE_VARIABLE) and _references(right, LAST_PAGE_SIZE_VARIABLE): + return ( + StopConditionSafety.UNKNOWN, + f"it compares `{LAST_PAGE_SIZE_VARIABLE}` against itself", + ) + + is_bare = isinstance(left, nodes.Name) and left.name == LAST_PAGE_SIZE_VARIABLE + + if operator in ("eq", "ne"): + if not is_bare: + return ( + StopConditionSafety.UNKNOWN, + f"`{LAST_PAGE_SIZE_VARIABLE}` is transformed before being compared", + ) + if isinstance(right, nodes.Const) and right.value == 0: + # A page that is full at the requested size holds at least `minimum_page_size` records, which is + # at least 1, so no reduction can make an emptiness test fire. + return ( + StopConditionSafety.SAFE, + f"`{LAST_PAGE_SIZE_VARIABLE}` is only tested for emptiness", + ) + return ( + StopConditionSafety.UNKNOWN, + f"it tests `{LAST_PAGE_SIZE_VARIABLE}` for equality against {_describe(right)} rather than against 0", + ) + + if operator in ("gt", "gteq"): + if is_bare: + # A reduction only lowers the size of a full page, so a lower bound can only stop being satisfied. + return ( + StopConditionSafety.SAFE, + f"a smaller page can only make `{LAST_PAGE_SIZE_VARIABLE} {'>' if operator == 'gt' else '>='} " + f"{_describe(right)}` less true, never more", + ) + return ( + StopConditionSafety.UNKNOWN, + f"`{LAST_PAGE_SIZE_VARIABLE}` is transformed before being compared", + ) + + if operator in ("lt", "lteq"): + if isinstance(left, nodes.BinExpr): + # `last_page_size - 100 < 0` is `last_page_size < 100` in disguise: with arithmetic on the left the + # threshold on the right is no longer the page size the condition really stops at, so neither the + # `page_size` reading nor the `minimum_page_size` reading below applies. + return ( + StopConditionSafety.UNKNOWN, + f"`{LAST_PAGE_SIZE_VARIABLE}` takes part in arithmetic before being compared", + ) + if _references(right, REQUESTED_PAGE_SIZE_VARIABLE): + if not _is_requested_page_size(right): + # `[page_size, 100] | max` and `page_size + 50` both mention the variable while holding a + # value a reduction does not lower, so the threshold does not follow the reduction after all. + return ( + StopConditionSafety.UNKNOWN, + f"it compares `{LAST_PAGE_SIZE_VARIABLE}` against an expression built from " + f"`{REQUESTED_PAGE_SIZE_VARIABLE}` rather than against `{REQUESTED_PAGE_SIZE_VARIABLE}` " + f"itself, so the threshold may not follow the reduction", + ) + return ( + StopConditionSafety.SAFE, + f"it compares `{LAST_PAGE_SIZE_VARIABLE}` against `{REQUESTED_PAGE_SIZE_VARIABLE}`, " + f"which follows the reduction", + ) + if isinstance(right, nodes.Const) and _is_whole_number(right.value): + # `last_page_size < k` is false for every full page exactly when k is at or below the smallest page + # the connector is allowed to request, which makes it another way of writing "the page is empty". + threshold = right.value if operator == "lt" else right.value + 1 + if threshold <= minimum_page_size: + return ( + StopConditionSafety.SAFE, + f"no page the connector is allowed to request is smaller than {threshold}", + ) + return ( + StopConditionSafety.TRUNCATES, + f"it stops as soon as a page is smaller than {_describe(right)}, which does not follow the " + f"reduction, so a full page at a reduced size reads as a short page", + ) + + return ( + StopConditionSafety.UNKNOWN, + f"it uses `{LAST_PAGE_SIZE_VARIABLE}` with the `{operator}` operator", + ) + + +def _is_requested_page_size(node: nodes.Node) -> bool: + """Whether the node is the `page_size` variable itself, filters aside.""" + unfiltered: Optional[nodes.Node] = node + while isinstance(unfiltered, nodes.Filter): + unfiltered = unfiltered.node + return isinstance(unfiltered, nodes.Name) and unfiltered.name == REQUESTED_PAGE_SIZE_VARIABLE + + +def references_requested_page_size(template: str) -> bool: + """ + Whether an expression reads the `page_size` interpolation variable. + + The variable is only bound to a number when the strategy declares a `page_size`, and a comparison against + an unbound one does not fail: Jinja raises, the interpolation falls back to the raw template string, and a + non-empty string is truthy - so `{{ last_page_size < page_size }}` would stop the pagination after the + first page. Reading it off the AST is what tells the variable apart from `config['page_size']`, which is a + `Getitem` on `config` rather than a `Name` and is bound whatever the strategy declares. + """ + try: + parsed = _PARSING_ENVIRONMENT.parse(template) + except TemplateSyntaxError: + # An expression that does not parse is not this function's to report on. The interpolation layer + # renders it as the literal string it is, and it names no variable either way. + return False + return _references(parsed, REQUESTED_PAGE_SIZE_VARIABLE) + + +def _references(node: nodes.Node, name: str) -> bool: + if isinstance(node, nodes.Name): + return bool(node.name == name) + return any(referenced.name == name for referenced in node.find_all(nodes.Name)) + + +def _is_whole_number(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + + +def _describe(node: nodes.Node) -> str: + """Render the expression a `last_page_size` comparison is made against, for an error message.""" + if isinstance(node, nodes.Const): + return repr(node.value) + if isinstance(node, nodes.Name): + return f"`{node.name}`" + if isinstance(node, nodes.Getitem) and isinstance(node.arg, nodes.Const): + return f"`{_describe(node.node).strip('`')}[{node.arg.value!r}]`" + if isinstance(node, nodes.Getattr): + return f"`{_describe(node.node).strip('`')}.{node.attr}`" + return "a value the connector computes itself" diff --git a/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py b/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py index c09f187276..5cf9532ec8 100644 --- a/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +++ b/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py @@ -71,6 +71,7 @@ def interpret_response( ResponseAction.RETRY, ResponseAction.IGNORE, ResponseAction.RESET_PAGINATION, + ResponseAction.REDUCE_PAGE_SIZE, ]: return matched_error_resolution diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/__init__.py b/airbyte_cdk/sources/declarative/requesters/paginators/__init__.py index 3b077ec0c3..475d722e1f 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/__init__.py @@ -12,6 +12,8 @@ PaginationStrategy, ) +# `page_size_override_kwargs` is deliberately not re-exported here: it is a CDK-internal helper, every call +# site is inside the CDK, and it imports from `paginators.paginator` directly. __all__ = [ "DefaultPaginator", "NoPagination", diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py b/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py index cb1072a8c5..d00e75ad29 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py @@ -13,7 +13,10 @@ PaginationDecoderDecorator, ) from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString -from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator +from airbyte_cdk.sources.declarative.requesters.paginators.paginator import ( + Paginator, + page_size_override_kwargs, +) from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import ( PaginationStrategy, ) @@ -133,18 +136,23 @@ def get_initial_token(self) -> Optional[Any]: """ return self.pagination_strategy.initial_token + def get_page_size(self) -> Optional[int]: + return self.pagination_strategy.get_page_size() + def next_page_token( self, response: requests.Response, last_page_size: int, last_record: Optional[Record], last_page_token_value: Optional[Any] = None, + page_size_override: Optional[int] = None, ) -> Optional[Mapping[str, Any]]: next_page_token = self.pagination_strategy.next_page_token( response=response, last_page_size=last_page_size, last_record=last_record, last_page_token_value=last_page_token_value, + **page_size_override_kwargs(page_size_override), ) if next_page_token: return {"next_page_token": next_page_token} @@ -169,8 +177,11 @@ def get_request_params( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> MutableMapping[str, Any]: - return self._get_request_options(RequestOptionType.request_parameter, next_page_token) + return self._get_request_options( + RequestOptionType.request_parameter, next_page_token, page_size_override + ) def get_request_headers( self, @@ -178,8 +189,11 @@ def get_request_headers( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Mapping[str, str]: - return self._get_request_options(RequestOptionType.header, next_page_token) + return self._get_request_options( + RequestOptionType.header, next_page_token, page_size_override + ) def get_request_body_data( self, @@ -187,8 +201,11 @@ def get_request_body_data( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Mapping[str, Any]: - return self._get_request_options(RequestOptionType.body_data, next_page_token) + return self._get_request_options( + RequestOptionType.body_data, next_page_token, page_size_override + ) def get_request_body_json( self, @@ -196,11 +213,17 @@ def get_request_body_json( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Mapping[str, Any]: - return self._get_request_options(RequestOptionType.body_json, next_page_token) + return self._get_request_options( + RequestOptionType.body_json, next_page_token, page_size_override + ) def _get_request_options( - self, option_type: RequestOptionType, next_page_token: Optional[Mapping[str, Any]] + self, + option_type: RequestOptionType, + next_page_token: Optional[Mapping[str, Any]], + page_size_override: Optional[int] = None, ) -> MutableMapping[str, Any]: options: MutableMapping[str, Any] = {} @@ -213,13 +236,14 @@ def _get_request_options( ): self.page_token_option.inject_into_request(options, token, self.config) - if ( - self.page_size_option - and self.pagination_strategy.get_page_size() - and self.page_size_option.inject_into == option_type - ): - page_size = self.pagination_strategy.get_page_size() - self.page_size_option.inject_into_request(options, page_size, self.config) + if self.page_size_option and self.page_size_option.inject_into == option_type: + page_size = ( + page_size_override + if page_size_override is not None + else self.pagination_strategy.get_page_size() + ) + if page_size: + self.page_size_option.inject_into_request(options, page_size, self.config) return options @@ -231,6 +255,16 @@ class PaginatorTestReadDecorator(Paginator): WARNING: This decorator is not currently thread-safe like the rest of the low-code framework because it has an internal state to track the current number of pages counted so that it can exit early during a test read + + The count bounds *successful* pages: it is incremented from `next_page_token`, which a page that resolved + to `ResponseAction.REDUCE_PAGE_SIZE` never reaches because `SimpleRetriever._read_pages` re-issues that + page before asking for the next token. A stream with `page_size_reduction` can therefore issue up to + `max_attempts` extra requests per slice on top of this limit. + + The Connector Builder counts pages a second time, from the request/response logs rather than from this + class (`connector_builder/test_reader/reader.py::_has_reached_limit`). The two counts agree because + `HttpClient` logs a response resolving to `REDUCE_PAGE_SIZE` as an auxiliary request, which the Builder + does not turn into a page. """ _PAGE_COUNT_BEFORE_FIRST_NEXT_CALL = 1 @@ -248,19 +282,27 @@ def get_initial_token(self) -> Optional[Any]: self._page_count = self._PAGE_COUNT_BEFORE_FIRST_NEXT_CALL return self._decorated.get_initial_token() + def get_page_size(self) -> Optional[int]: + return self._decorated.get_page_size() + def next_page_token( self, response: requests.Response, last_page_size: int, last_record: Optional[Record], last_page_token_value: Optional[Any] = None, + page_size_override: Optional[int] = None, ) -> Optional[Mapping[str, Any]]: if self._page_count >= self._maximum_number_of_pages: return None self._page_count += 1 return self._decorated.next_page_token( - response, last_page_size, last_record, last_page_token_value + response, + last_page_size, + last_record, + last_page_token_value, + **page_size_override_kwargs(page_size_override), ) def path( @@ -281,9 +323,13 @@ def get_request_params( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Mapping[str, Any]: return self._decorated.get_request_params( - stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + **page_size_override_kwargs(page_size_override), ) def get_request_headers( @@ -292,9 +338,13 @@ def get_request_headers( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Mapping[str, str]: return self._decorated.get_request_headers( - stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + **page_size_override_kwargs(page_size_override), ) def get_request_body_data( @@ -303,9 +353,13 @@ def get_request_body_data( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Union[Mapping[str, Any], str]: return self._decorated.get_request_body_data( - stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + **page_size_override_kwargs(page_size_override), ) def get_request_body_json( @@ -314,7 +368,11 @@ def get_request_body_json( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Mapping[str, Any]: return self._decorated.get_request_body_json( - stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + **page_size_override_kwargs(page_size_override), ) diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py b/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py index b3b1d3b662..f207b1cce9 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py @@ -72,5 +72,6 @@ def next_page_token( last_page_size: int, last_record: Optional[Record], last_page_token_value: Optional[Any], + page_size_override: Optional[int] = None, ) -> Optional[Mapping[str, Any]]: return {} diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py b/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py index f8c31d4f5f..17f974416e 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py @@ -4,7 +4,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import Any, Mapping, Optional +from typing import Any, Dict, Mapping, Optional import requests @@ -14,6 +14,16 @@ from airbyte_cdk.sources.types import Record, StreamSlice +def page_size_override_kwargs(page_size_override: Optional[int]) -> Dict[str, Any]: + """ + Build the `page_size_override` keyword argument only when there is an override to pass. + + Paginators and pagination strategies defined outside of the CDK may not accept the argument, and they only + need to when the stream actually reduces its page size (see `ResponseAction.REDUCE_PAGE_SIZE`). + """ + return {"page_size_override": page_size_override} if page_size_override is not None else {} + + @dataclass class Paginator(ABC, RequestOptionsProvider): """ @@ -36,6 +46,7 @@ def next_page_token( last_page_size: int, last_record: Optional[Record], last_page_token_value: Optional[Any], + page_size_override: Optional[int] = None, ) -> Optional[Mapping[str, Any]]: """ Returns the next_page_token to use to fetch the next page of records. @@ -44,10 +55,22 @@ def next_page_token( :param last_page_size: the number of records read from the response :param last_record: the last record extracted from the response :param last_page_token_value: The current value of the page token made on the last request + :param page_size_override: the page size that was actually requested, when it differs from the configured + one because of a `REDUCE_PAGE_SIZE` response action :return: A mapping {"next_page_token": } for the next page from the input response object. Returning None means there are no more pages to read in this response. """ pass + def get_page_size(self) -> Optional[int]: + """ + Evaluated against the config alone. A pagination strategy whose `page_size` template references the + response evaluates it with the response in context inside `next_page_token`, so the two can disagree - + pre-existing, and only observable for a `page_size` that is not a constant. + + :return: the number of records this paginator asks for per page, or None if it does not define one + """ + return None + @abstractmethod def path( self, diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py index 73a644b5ba..a4c43aed45 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py @@ -14,6 +14,10 @@ ) from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.parsers.stop_condition_safety import ( + REQUESTED_PAGE_SIZE_VARIABLE, + references_requested_page_size, +) from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import ( PaginationStrategy, ) @@ -62,6 +66,39 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None: raise Exception(f"{page_size} is of type {type(page_size)}. Expected {int}") self._page_size = page_size + if self._page_size is None: + self._reject_unbound_page_size_variable() + + def _reject_unbound_page_size_variable(self) -> None: + """ + Fail construction when an expression reads `page_size` while the strategy declares none. + + The variable holds the page size that was actually requested, so it is the one safe thing to compare + `last_page_size` against while a `page_size_reduction` is shrinking the page. Without a declared + `page_size` there is nothing to bind it to, and an unbound comparison does not raise: Jinja fails, the + interpolation falls back to the raw template string, and a non-empty string is truthy. A + `stop_condition` written that way stops after the first page and drops the rest of the partition + without failing, which is why this is a construction error rather than a warning. + """ + for field_name, template in ( + ("stop_condition", self.stop_condition), + ("cursor_value", self.cursor_value), + ): + expression = template.string if isinstance(template, InterpolatedString) else template + if isinstance(template, InterpolatedBoolean): + expression = template.condition + if not isinstance(expression, str) or not references_requested_page_size(expression): + continue + raise ValueError( + f"The `{field_name}` {expression!r} reads the `{REQUESTED_PAGE_SIZE_VARIABLE}` interpolation " + f"variable, but the CursorPagination strategy it belongs to declares no `page_size`, so there " + f"is nothing to bind it to. The comparison would not fail either - it renders as the template " + f"string itself, which is truthy - so the pagination would end after the first page and the " + f"rest of the partition would be dropped silently. Declare `page_size` on the pagination " + f"strategy, or compare against a value the manifest defines, such as " + f"`config['page_size']`." + ) + @property def initial_token(self) -> Optional[Any]: """ @@ -77,7 +114,16 @@ def next_page_token( last_page_size: int, last_record: Optional[Record], last_page_token_value: Optional[Any] = None, + page_size_override: Optional[int] = None, ) -> Optional[Any]: + # The next page is a cursor read from the response, so `page_size_override` does not change how the token + # is computed. It is still exposed to the interpolation context as `page_size` because a `stop_condition` + # comparing `last_page_size` to a hardcoded page size would read a full reduced page as a short page and + # end the pagination early, silently dropping the rest of the partition. Writing the condition as + # `{{ last_page_size < page_size }}` keeps it correct while a reduction is in effect. + requested_page_size = ( + page_size_override if page_size_override is not None else self._page_size + ) decoded_response = next(self.decoder.decode(response)) # The default way that link is presented in requests.Response is a string of various links (last, next, etc). This # is not indexable or useful for parsing the cursor, so we replace it with the link dictionary from response.links @@ -90,6 +136,7 @@ def next_page_token( headers=headers, last_record=last_record, last_page_size=last_page_size, + page_size=requested_page_size, ) if should_stop: return None @@ -99,6 +146,7 @@ def next_page_token( headers=headers, last_record=last_record, last_page_size=last_page_size, + page_size=requested_page_size, ) return token if token else None diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py index 4370155dec..46939402c4 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py @@ -74,6 +74,7 @@ def next_page_token( last_page_size: int, last_record: Optional[Record], last_page_token_value: Optional[Any] = None, + page_size_override: Optional[int] = None, ) -> Optional[Any]: decoded_response = next(self.decoder.decode(response)) @@ -85,11 +86,20 @@ def next_page_token( page_size_from_response if page_size_from_response is not None else last_page_size ) - # Stop paginating when there are fewer records than the page size or the current page has no records - if ( - self._page_size - and last_page_size < self._page_size.eval(self.config, response=decoded_response) - ) or last_page_size == 0: + # Stop paginating when there are fewer records than the page size or the current page has no records. + # The comparison uses the page size that was actually requested: after a `REDUCE_PAGE_SIZE` reduction, a + # full page is smaller than the configured page size and comparing against the latter would end the + # pagination early and skip records. + requested_page_size = ( + page_size_override + if page_size_override is not None + else ( + self._page_size.eval(self.config, response=decoded_response) + if self._page_size + else None + ) + ) + if (requested_page_size and last_page_size < requested_page_size) or last_page_size == 0: return None elif last_page_token_value is None: # If the OffsetIncrement strategy does not inject on the first request, the incoming last_page_token_value diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py index f3bad7152b..026e0452d1 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py @@ -7,12 +7,14 @@ import requests +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import ( PaginationStrategy, ) from airbyte_cdk.sources.types import Config, Record +from airbyte_cdk.utils.traced_exception import AirbyteTracedException @dataclass @@ -53,7 +55,23 @@ def next_page_token( last_page_size: int, last_record: Optional[Record], last_page_token_value: Optional[Any], + page_size_override: Optional[int] = None, ) -> Optional[Any]: + if page_size_override is not None: + # Reachable only when the factory is bypassed: a manifest naming PageIncrement, and a + # CustomPaginationStrategy that subclasses it, are both rejected at config time. An unrecognized + # ValueError here would be reported as a generic system error even though the message describes a + # configuration mistake. `next_page_token` runs after the page's records have been emitted, so an + # out-of-tree caller gets this mid-stream rather than at startup - still better than silently + # ignoring the override and skipping records. + raise AirbyteTracedException( + internal_message="PageIncrement received a page_size_override", + message="PageIncrement does not support reducing the page size while paginating: pages are " + "addressed as page number * page size, so a smaller page size shifts every following page " + "boundary and would skip records. Use OffsetIncrement or CursorPagination instead.", + failure_type=FailureType.config_error, + ) + if self.extractor: # The record count is dependent on the records returned from the response which may not always # align with the size of pages emitted. For example, a record filter can reduce the number of diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py index dae02ba138..9be3af80ed 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py @@ -31,12 +31,16 @@ def next_page_token( last_page_size: int, last_record: Optional[Record], last_page_token_value: Optional[Any], + page_size_override: Optional[int] = None, ) -> Optional[Any]: """ :param response: response to process :param last_page_size: the number of records read from the response :param last_record: the last record extracted from the response :param last_page_token_value: The current value of the page token made on the last request + :param page_size_override: the page size that was actually requested, when it differs from the configured + one because of a `REDUCE_PAGE_SIZE` response action. Strategies that use their configured page size as + a stop condition must honor this value, else they end the pagination early and skip records. :return: next page token. Returns None if there are no more pages to fetch """ pass diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py index 068df72cb8..6df7a3a344 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py @@ -7,6 +7,9 @@ import requests +from airbyte_cdk.sources.declarative.requesters.paginators.paginator import ( + page_size_override_kwargs, +) from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import ( PaginationStrategy, ) @@ -47,13 +50,18 @@ def next_page_token( last_page_size: int, last_record: Optional[Record], last_page_token_value: Optional[Any] = None, + page_size_override: Optional[int] = None, ) -> Optional[Any]: # We evaluate in reverse order because the assumption is that most of the APIs using data feed structure # will return records in descending order. In terms of performance/memory, we return the records lazily if last_record and self._stop_condition.is_met(last_record): return None return self._delegate.next_page_token( - response, last_page_size, last_record, last_page_token_value + response, + last_page_size, + last_record, + last_page_token_value, + **page_size_override_kwargs(page_size_override), ) def get_page_size(self) -> Optional[int]: diff --git a/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py b/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py new file mode 100644 index 0000000000..496313db2f --- /dev/null +++ b/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py @@ -0,0 +1,281 @@ +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. + +import logging +import time +from dataclasses import dataclass +from enum import Enum +from typing import Callable, Optional + +from airbyte_cdk.models import FailureType +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + +LOGGER = logging.getLogger("airbyte") + + +class PageSizeResetPolicy(Enum): + NEVER = "NEVER" + AFTER_SUCCESSFUL_PAGE = "AFTER_SUCCESSFUL_PAGE" + + +@dataclass(frozen=True) +class PageSizeReduction: + """ + How much to shrink the page size when an error handler resolves to `ResponseAction.REDUCE_PAGE_SIZE`. + + This is immutable configuration: it is created once per stream and shared, while the page size in effect + lives in a `PageSizeReducer` created per partition read. + """ + + reduction_factor: float = 2.0 + minimum_page_size: int = 1 + max_attempts: int = 5 + reset_policy: PageSizeResetPolicy = PageSizeResetPolicy.NEVER + # Base wait before a page is re-issued, multiplied by the number of attempts made in a row. A + # `REDUCE_PAGE_SIZE` response never reaches the HTTP retry budget - the exception raised for it is + # deliberately not a backoff exception - so this is the only thing spacing these requests out, and an + # API whose 502 means "we are briefly unwell" rather than "your page is too big" needs it to be more + # than a token pause. + backoff_seconds: float = 0.5 + # How many times the same page is re-issued unchanged once the page size cannot be shrunk any further, + # before the read gives up. Zero keeps the strict behaviour: the first response that cannot be answered + # with a smaller page fails the stream. It is what gives an API whose error is transient a budget at the + # floor, where reducing is no longer an option but waiting still is. + retries_at_minimum_page_size: int = 0 + # Appended to the two messages raised once the page size cannot be reduced any further. Those are + # `transient_error`s the CDK has no remediation for - it only knows that the API rejected every page size + # asked for - while the connector knows what narrows a query down on this particular API. + failure_message: Optional[str] = None + + def __post_init__(self) -> None: + if self.reduction_factor <= 1: + raise ValueError( + f"The page size reduction factor needs to be greater than 1. Got {self.reduction_factor}" + ) + if self.minimum_page_size < 1: + raise ValueError( + f"The minimum page size needs to be strictly positive. Got {self.minimum_page_size}" + ) + if self.max_attempts < 1: + raise ValueError( + f"The maximum number of page size reductions needs to be strictly positive. Got {self.max_attempts}" + ) + if self.backoff_seconds < 0: + raise ValueError( + f"The wait between page size reductions cannot be negative. Got {self.backoff_seconds}" + ) + if self.retries_at_minimum_page_size < 0: + raise ValueError( + f"The number of retries at the minimum page size cannot be negative. Got {self.retries_at_minimum_page_size}" + ) + + +class PageSizeReducer: + """ + Tracks the page size to use while reading one partition when the API asks for smaller pages. + + One instance is created per `SimpleRetriever._read_pages` call. This is deliberate: a retriever and its + paginator are shared by every partition of a stream and partitions are read concurrently, so the reduced + page size must not be stored on the paginator or on the retriever. + """ + + def __init__( + self, + config: PageSizeReduction, + configured_page_size: Optional[int], + stream_name: str = "", + sleep: Optional[Callable[[float], None]] = None, + ) -> None: + self._config = config + self._configured_page_size = configured_page_size + self._stream_name = stream_name + # Resolved on each call rather than bound here: a default of `time.sleep` would capture + # the function object, and a test that patches `time.sleep` to keep a run of reductions + # from taking its two minutes for real would have no effect on an already-bound default. + self._sleep_override = sleep + self._current_page_size: Optional[int] = None + self._attempts = 0 + self._total_reductions = 0 + self._retries_at_minimum_page_size = 0 + + def _sleep(self, seconds: float) -> None: + if self._sleep_override is not None: + self._sleep_override(seconds) + return + time.sleep(seconds) + + @property + def page_size_override(self) -> Optional[int]: + """ + :return: the reduced page size to request, or None while the configured page size is in effect + """ + return self._current_page_size + + def reduce(self) -> None: + """ + Shrink the page size used for the next request. Raises once the page size cannot be shrunk any further + so that an endpoint that keeps failing does not loop forever. + """ + if self._configured_page_size is None: + raise AirbyteTracedException( + internal_message=f"Stream {self._stream_name} received a REDUCE_PAGE_SIZE response action but its paginator does not inject a page size", + message=f"Stream {self._stream_name} is set up to reduce its page size on error but does not define " + f"one. Set `page_size` on the pagination strategy and `page_size_option` on the paginator.", + failure_type=FailureType.config_error, + ) + + current_page_size = ( + self._current_page_size + if self._current_page_size is not None + else self._configured_page_size + ) + if not isinstance(current_page_size, int) or isinstance(current_page_size, bool): + # A custom pagination strategy can return anything from `get_page_size`. Reducing + # is arithmetic, so a non-integer would otherwise fail with a bare TypeError in + # the middle of a sync. + raise AirbyteTracedException( + internal_message=f"Stream {self._stream_name} has a page size of type {type(current_page_size).__name__}: {current_page_size!r}", + message=f"The page size of stream {self._stream_name} is not a whole number, so the connector cannot " + f"reduce it. Make sure the pagination strategy's `page_size` is a number.", + failure_type=FailureType.config_error, + ) + + reduced_page_size = max( + self._config.minimum_page_size, + int(current_page_size // self._config.reduction_factor), + ) + if reduced_page_size >= current_page_size: + # Nothing left to give up on the page size. Whether that is the end of the read is + # `retries_at_minimum_page_size`'s call, not this branch's: the response may still be transient. + self._retry_at_minimum_page_size(current_page_size) + return + + self._attempts += 1 + self._total_reductions += 1 + if self._attempts > self._config.max_attempts: + # The budget counts the reductions that did *not* get a page through, which is what separates a + # partition that is stuck from one that is merely expensive. A partition where pages keep + # succeeding restarts this counter on each of them, under either reset policy, and reads to the + # end however many pages it has; a partition where nothing gets through burns the budget here. + raise AirbyteTracedException( + internal_message=f"Stream {self._stream_name} reduced its page size {self._attempts - 1} times in a row without a single page succeeding, which is the configured maximum of {self._config.max_attempts} ({self._total_reductions - 1} reductions so far while reading this partition)", + # `transient_error`, so the only remediation is the connector's own, if it defined one. + message=self._with_failure_message( + f"The source keeps rejecting pages of stream {self._stream_name} at every page size the " + f"connector requested, down to {current_page_size} records per page." + ), + failure_type=FailureType.transient_error, + ) + + backoff = self._config.backoff_seconds * self._attempts + LOGGER.info( + f"Reducing the page size of stream {self._stream_name} from {current_page_size} to {reduced_page_size} " + f"and retrying the same page in {backoff}s." + ) + self._current_page_size = reduced_page_size + self._sleep(backoff) + + def _retry_at_minimum_page_size(self, current_page_size: int) -> None: + """ + Handle a `REDUCE_PAGE_SIZE` response that arrives when the page size is already as small as the + connector is allowed to request. + + Reducing is out of options here, but re-issuing the page is not: an API that answers 502 to a page it + considers too heavy answers the same 502 when it is briefly unwell, and the error handler cannot tell + the two apart. `REDUCE_PAGE_SIZE` bypasses the HTTP retry budget, so without this budget the second + kind of 502 ends the stream on the first response once the floor is reached - fewer attempts than the + same connector got before it adopted the reduction. + + The budget applies however the page size arrived at the floor, whether by reduction or because the + configured page size was already there. Only once it is spent does the failure depend on that: a page + size the connector's own floor blocks from ever being reduced is a configuration error, while a floor + the reduction walked down to means the API kept rejecting every size, which is transient. + """ + # The retries come first, including on a stream whose page size started at the floor. How the page size + # arrived there says nothing about the response, so letting the misconfiguration branch below decide it + # would give the same manifest a retry budget or none depending on the user's `page_size`. + if self._retries_at_minimum_page_size < self._config.retries_at_minimum_page_size: + self._retries_at_minimum_page_size += 1 + backoff = self._config.backoff_seconds * self._retries_at_minimum_page_size + LOGGER.info( + f"Stream {self._stream_name} cannot request a page smaller than {current_page_size} records, " + f"so the same page is retried unchanged in {backoff}s " + f"({self._retries_at_minimum_page_size} of {self._config.retries_at_minimum_page_size})." + ) + self._sleep(backoff) + return + + if self._current_page_size is None and self._config.minimum_page_size > 1: + # No reduction was ever applied and the connector's own floor is what blocks it, so the page size + # can never be reduced on this stream however the API behaves. That is a configuration error, and + # it is actionable: both numbers in the message are the connector's to change. It is reported once + # the retries above are spent, so a manifest that asked for them still gets them. + raise AirbyteTracedException( + internal_message=f"Stream {self._stream_name} has a configured page size of {current_page_size} which is not greater than the configured minimum page size of {self._config.minimum_page_size}, so it can never be reduced" + + ( + f" ({self._retries_at_minimum_page_size} retries at that size were spent first)" + if self._retries_at_minimum_page_size + else "" + ), + message=f"The page size of stream {self._stream_name} ({current_page_size}) is already at or below " + f"the configured minimum of {self._config.minimum_page_size}, so the connector cannot reduce it. " + f"Raise the page size of the stream, or lower `minimum_page_size`.", + failure_type=FailureType.config_error, + ) + + raise AirbyteTracedException( + internal_message=f"Stream {self._stream_name} still fails with a page size of {current_page_size}, which is the smallest page size allowed by the configured minimum of {self._config.minimum_page_size}" + + ( + f", after {self._retries_at_minimum_page_size} retries at that size" + if self._retries_at_minimum_page_size + else "" + ), + # `transient_error`, so the only remediation is the connector's own, if it defined one. + message=self._with_failure_message( + f"The source keeps rejecting pages of stream {self._stream_name} at the smallest page size " + f"the connector is allowed to request ({current_page_size} records per page)." + ), + failure_type=FailureType.transient_error, + ) + + def _with_failure_message(self, message: str) -> str: + """ + :return: the message followed by the connector's `failure_message`, when it defined one + """ + failure_message = (self._config.failure_message or "").strip() + if not failure_message: + return message + return f"{message} {failure_message}" + + def on_successful_page(self) -> None: + """ + Called after each page that did not require a reduction. + + The `max_attempts` budget restarts under both policies. It counts the reductions made *in a row* + without a single page succeeding, which is what separates a partition that is stuck from one that is + merely expensive: on a stream whose per-page cost varies - the GraphQL case this feature exists for - + a handful of heavy pages spread over a long partition is a healthy read, and a budget spanning the + whole partition would fail it at the `max_attempts + 1`-th heavy page while every reduction so far had + been followed by a successful page. The terminal message says the source rejected every page size the + connector asked for, so the budget has to mean exactly that. + + The budget still terminates the read, because only a page that succeeded restarts it and only + `_read_pages` calls this, once per page it consumed. So between any two restarts the partition made one + page of progress, and the reductions that make no progress are bounded by `max_attempts`. Under `NEVER` + the reduced page size is never restored either, so it strictly decreases and `minimum_page_size` bounds + the reductions of the whole partition on its own. + + Only `AFTER_SUCCESSFUL_PAGE` restores the page size. That policy is for an API that rejects the + configured page size on every page, so every page legitimately costs one reduction; `NEVER` keeps the + reduced size for the rest of the partition, which is the right behaviour when reductions are one-off. + """ + self._attempts = 0 + self._retries_at_minimum_page_size = 0 + + if self._config.reset_policy != PageSizeResetPolicy.AFTER_SUCCESSFUL_PAGE: + return + + if self._current_page_size is not None: + LOGGER.info( + f"Restoring the page size of stream {self._stream_name} from {self._current_page_size} to {self._configured_page_size}." + ) + self._current_page_size = None diff --git a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index 6f82b8ebd9..85871d0eb7 100644 --- a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -23,6 +23,7 @@ import requests from typing_extensions import deprecated +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector from airbyte_cdk.sources.declarative.extractors.record_filter import ( ClientSideIncrementalRecordFilterDecorator, @@ -32,23 +33,35 @@ SinglePartitionRouter, ) from airbyte_cdk.sources.declarative.requesters.paginators.no_pagination import NoPagination -from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator +from airbyte_cdk.sources.declarative.requesters.paginators.paginator import ( + Paginator, + page_size_override_kwargs, +) from airbyte_cdk.sources.declarative.requesters.query_properties import QueryProperties from airbyte_cdk.sources.declarative.requesters.request_options import ( DefaultRequestOptionsProvider, RequestOptionsProvider, ) from airbyte_cdk.sources.declarative.requesters.requester import Requester +from airbyte_cdk.sources.declarative.retrievers.page_size_reducer import ( + PageSizeReducer, + PageSizeReduction, +) from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer from airbyte_cdk.sources.source import ExperimentalClassWarning from airbyte_cdk.sources.streams.core import StreamData +from airbyte_cdk.sources.streams.http.page_size_reduction_exception import ( + PageSizeReductionNotSupportedException, + PageSizeReductionRequiredException, +) from airbyte_cdk.sources.streams.http.pagination_reset_exception import ( PaginationResetRequiredException, ) from airbyte_cdk.sources.types import Config, Record, StreamSlice from airbyte_cdk.utils.mapping_helpers import combine_mappings +from airbyte_cdk.utils.traced_exception import AirbyteTracedException FULL_REFRESH_SYNC_COMPLETE_KEY = "__ab_full_refresh_sync_complete" LOGGER = logging.getLogger("airbyte") @@ -77,6 +90,12 @@ class SimpleRetriever(Retriever): parameters (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation post_pagination_filter (Optional[ClientSideIncrementalRecordFilterDecorator]): Set for data feed streams only. Records the cursor considers already synced are dropped once pagination has observed them + page_size_reduction (Optional[PageSizeReduction]): How much to shrink the page size when an error handler + resolves to `ResponseAction.REDUCE_PAGE_SIZE`. `None` disables page size reduction entirely. + It is immutable configuration; the page size in effect lives in a `PageSizeReducer` that + `_read_pages` creates per call, so the retriever and its paginator - both shared by every + partition of the stream, read concurrently - stay stateless. When `page_size_reduction` is + `None` no reducer is created and `_read_pages` keeps its previous behaviour """ requester: Requester @@ -101,6 +120,7 @@ class SimpleRetriever(Retriever): default_factory=lambda: lambda: PaginationTracker() ) post_pagination_filter: Optional[ClientSideIncrementalRecordFilterDecorator] = None + page_size_reduction: Optional[PageSizeReduction] = None def __post_init__(self, parameters: Mapping[str, Any]) -> None: self._paginator = self.paginator or NoPagination(parameters=parameters) @@ -145,6 +165,7 @@ def _get_request_options( next_page_token: Optional[Mapping[str, Any]], paginator_method: Callable[..., Optional[Union[Mapping[str, Any], str]]], stream_slicer_method: Callable[..., Optional[Union[Mapping[str, Any], str]]], + page_size_override: Optional[int] = None, ) -> Union[Mapping[str, Any], str]: """ Get the request_option from the paginator and the stream slicer. @@ -157,6 +178,7 @@ def _get_request_options( paginator_method( stream_slice=stream_slice, next_page_token=next_page_token, + **page_size_override_kwargs(page_size_override), ), ] if not next_page_token or not self.ignore_stream_slicer_parameters_on_paginated_requests: @@ -172,6 +194,7 @@ def _request_headers( self, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Mapping[str, Any]: """ Specifies request headers. @@ -182,6 +205,7 @@ def _request_headers( next_page_token, self._paginator.get_request_headers, self.request_option_provider.get_request_headers, + **page_size_override_kwargs(page_size_override), ) if isinstance(headers, str): raise ValueError("Request headers cannot be a string") @@ -191,6 +215,7 @@ def _request_params( self, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Mapping[str, Any]: """ Specifies the query parameters that should be set on an outgoing HTTP request given the inputs. @@ -202,6 +227,7 @@ def _request_params( next_page_token, self._paginator.get_request_params, self.request_option_provider.get_request_params, + **page_size_override_kwargs(page_size_override), ) if isinstance(params, str): raise ValueError("Request params cannot be a string") @@ -211,6 +237,7 @@ def _request_body_data( self, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Union[Mapping[str, Any], str]: """ Specifies how to populate the body of the request with a non-JSON payload. @@ -226,12 +253,14 @@ def _request_body_data( next_page_token, self._paginator.get_request_body_data, self.request_option_provider.get_request_body_data, + **page_size_override_kwargs(page_size_override), ) def _request_body_json( self, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Optional[Mapping[str, Any]]: """ Specifies how to populate the body of the request with a JSON payload. @@ -243,6 +272,7 @@ def _request_body_json( next_page_token, self._paginator.get_request_body_json, self.request_option_provider.get_request_body_json, + **page_size_override_kwargs(page_size_override), ) if isinstance(body_json, str): raise ValueError("Request body json cannot be a string") @@ -298,6 +328,7 @@ def _next_page_token( last_page_size: int, last_record: Optional[Record], last_page_token_value: Optional[Any], + page_size_override: Optional[int] = None, ) -> Optional[Mapping[str, Any]]: """ Specifies a pagination strategy. @@ -311,12 +342,14 @@ def _next_page_token( last_page_size=last_page_size, last_record=last_record, last_page_token_value=last_page_token_value, + **page_size_override_kwargs(page_size_override), ) def _fetch_next_page( self, stream_slice: StreamSlice, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Optional[requests.Response]: return self.requester.send_request( path=self._paginator_path( @@ -329,18 +362,22 @@ def _fetch_next_page( request_headers=self._request_headers( stream_slice=stream_slice, next_page_token=next_page_token, + **page_size_override_kwargs(page_size_override), ), request_params=self._request_params( stream_slice=stream_slice, next_page_token=next_page_token, + **page_size_override_kwargs(page_size_override), ), request_body_data=self._request_body_data( stream_slice=stream_slice, next_page_token=next_page_token, + **page_size_override_kwargs(page_size_override), ), request_body_json=self._request_body_json( stream_slice=stream_slice, next_page_token=next_page_token, + **page_size_override_kwargs(page_size_override), ), log_formatter=self.log_formatter, ) @@ -353,9 +390,20 @@ def _read_pages( ) -> Iterable[Record]: original_stream_slice = stream_slice pagination_tracker = self.pagination_tracker_factory() + page_size_reducer = ( + PageSizeReducer( + self.page_size_reduction, + self._paginator.get_page_size(), + stream_name=self.name, + ) + if self.page_size_reduction + else None + ) reset_pagination = False + reduce_page_size = False next_page_token = self._get_initial_next_page_token() while True: + page_size_override = page_size_reducer.page_size_override if page_size_reducer else None merged_records: MutableMapping[str, Any] = defaultdict(dict) last_page_size = 0 last_record: Optional[Record] = None @@ -371,7 +419,11 @@ def _read_pages( cursor_slice=stream_slice.cursor_slice or {}, extra_fields={"query_properties": properties}, ) - response = self._fetch_next_page(stream_slice, next_page_token) + response = self._fetch_next_page( + stream_slice, + next_page_token, + **page_size_override_kwargs(page_size_override), + ) for current_record in records_generator_fn(response): if self.additional_query_properties.property_chunking: @@ -401,7 +453,11 @@ def _read_pages( last_record = record yield record else: - response = self._fetch_next_page(stream_slice, next_page_token) + response = self._fetch_next_page( + stream_slice, + next_page_token, + **page_size_override_kwargs(page_size_override), + ) for current_record in records_generator_fn(response): pagination_tracker.observe(current_record) last_page_size += 1 @@ -409,10 +465,39 @@ def _read_pages( yield current_record except PaginationResetRequiredException: reset_pagination = True + except PageSizeReductionRequiredException: + if page_size_reducer is None: + # The action can be attached to a requester we cannot validate at config time, such as one + # built by a custom error handler or a CustomRequester. Re-raise as the misconfiguration it + # is: the exception being handled is the neutral "the API asked for a smaller page" signal. + raise PageSizeReductionNotSupportedException(stream_name=self.name) + if last_page_size: + # The reduction is safe only because it re-issues a page whose records were not emitted. + # Every in-CDK way of reaching this raises from `_fetch_next_page`, before the record loop, + # and the factory rejects the manifest constructs that would not, but a custom extractor, + # filter or transformation can issue its own request from inside the record generator. + # Re-issuing the page then duplicates the records already yielded, so fail instead. + raise AirbyteTracedException( + internal_message=f"Stream {self.name} requested a page size reduction after {last_page_size} records of the page had already been emitted", + message=f"Stream {self.name} asked for a smaller page size in the middle of a page. The page cannot be requested again without duplicating the records already read from it. Move the REDUCE_PAGE_SIZE action to the error handler of the stream's main requester.", + failure_type=FailureType.config_error, + ) + # Raises once the page size cannot be reduced any further and the retries allowed at that + # floor are spent, which is what stops the loop when the API keeps failing. + page_size_reducer.reduce() + reduce_page_size = True else: + if page_size_reducer: + page_size_reducer.on_successful_page() if not response: break + if reduce_page_size: + # Retry the very same page: neither the token nor the slice change, only the page size does - + # and not even that once the reducer is at its floor and only waiting is left. + reduce_page_size = False + continue + if reset_pagination or pagination_tracker.has_reached_limit(): next_page_token = self._get_initial_next_page_token() previous_slice = stream_slice @@ -432,6 +517,7 @@ def _read_pages( last_page_size=last_page_size, last_record=last_record, last_page_token_value=last_page_token_value, + **page_size_override_kwargs(page_size_override), ) if not next_page_token: break diff --git a/airbyte_cdk/sources/streams/http/error_handlers/response_models.py b/airbyte_cdk/sources/streams/http/error_handlers/response_models.py index 082d580d53..10416c1a7e 100644 --- a/airbyte_cdk/sources/streams/http/error_handlers/response_models.py +++ b/airbyte_cdk/sources/streams/http/error_handlers/response_models.py @@ -19,6 +19,7 @@ class ResponseAction(Enum): RESET_PAGINATION = "RESET_PAGINATION" RATE_LIMITED = "RATE_LIMITED" REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" + REDUCE_PAGE_SIZE = "REDUCE_PAGE_SIZE" @dataclass diff --git a/airbyte_cdk/sources/streams/http/http_client.py b/airbyte_cdk/sources/streams/http/http_client.py index c9008d1e64..caacdd503a 100644 --- a/airbyte_cdk/sources/streams/http/http_client.py +++ b/airbyte_cdk/sources/streams/http/http_client.py @@ -42,6 +42,9 @@ RequestBodyException, UserDefinedBackoffException, ) +from airbyte_cdk.sources.streams.http.page_size_reduction_exception import ( + PageSizeReductionRequiredException, +) from airbyte_cdk.sources.streams.http.pagination_reset_exception import ( PaginationResetRequiredException, ) @@ -95,6 +98,30 @@ def monkey_patched_get_item(self, key): # type: ignore # this interface is a co requests_cache.SQLiteDict.__getitem__ = monkey_patched_get_item # type: ignore # see the method doc for more information +def _as_auxiliary_request_log( + log_message: Any, title: Optional[str] = None, description: Optional[str] = None +) -> Any: + """ + Flag an already-formatted request/response log as an auxiliary request. + + The Connector Builder builds one page per non-auxiliary HTTP log and bounds a slice by the number of those + pages, so a request that will not produce a page has to be marked here or it inflates that count. The + Builder also labels its side panel from the log's `title` and `description`, which the formatter filled + with the wording of an ordinary page, so a caller that knows why the request is auxiliary passes its own + and the panel does not read as a successful page fetch. The log formatter is connector-supplied and only + the CDK's own one is guaranteed to have an `http` object, hence the defensive check. + """ + if isinstance(log_message, dict): + http = log_message.get("http") + if isinstance(http, dict): + http["is_auxiliary"] = True + if title is not None: + http["title"] = title + if description is not None: + http["description"] = description + return log_message + + class HttpClient: _DEFAULT_MAX_RETRY: int = 5 _DEFAULT_MAX_TIME: int = 60 * 10 @@ -447,9 +474,24 @@ def _send( and self._message_repository is not None ): formatter = log_formatter + # A response resolving to REDUCE_PAGE_SIZE is not a page of the stream: the retriever discards it + # and re-issues the same page with a smaller page size. Logging it as an auxiliary request keeps it + # visible in the Connector Builder while keeping it out of the per-slice page count, which would + # otherwise report "limit reached" on a read that only retried. + log_as_auxiliary = error_resolution.response_action == ResponseAction.REDUCE_PAGE_SIZE self._message_repository.log_message( Level.DEBUG, - lambda: formatter(response), + lambda: _as_auxiliary_request_log( + formatter(response), + title=f"Stream '{self._name}' page rejected, retrying with a smaller page size", + description=( + f"Request for stream '{self._name}' whose response asked for a smaller page. The " + f"same page is requested again with a reduced page size, so this request produced " + f"no records." + ), + ) + if log_as_auxiliary + else formatter(response), ) self._handle_error_resolution( @@ -510,6 +552,11 @@ def _handle_error_resolution( if error_resolution.response_action == ResponseAction.RESET_PAGINATION: raise PaginationResetRequiredException() + if error_resolution.response_action == ResponseAction.REDUCE_PAGE_SIZE: + raise PageSizeReductionRequiredException( + stream_name=self._name, error_message=error_resolution.error_message + ) + # Emit stream status RUNNING with the reason RATE_LIMITED to log that the rate limit has been reached if error_resolution.response_action == ResponseAction.RATE_LIMITED: # TODO: Update to handle with message repository when concurrent message repository is ready diff --git a/airbyte_cdk/sources/streams/http/page_size_reduction_exception.py b/airbyte_cdk/sources/streams/http/page_size_reduction_exception.py new file mode 100644 index 0000000000..451149eecd --- /dev/null +++ b/airbyte_cdk/sources/streams/http/page_size_reduction_exception.py @@ -0,0 +1,56 @@ +# +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. +# + +from typing import Optional + +from airbyte_cdk.models import FailureType +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + + +class PageSizeReductionRequiredException(AirbyteTracedException): + """ + Raised when an error handler resolves to `ResponseAction.REDUCE_PAGE_SIZE`. + + This is control flow rather than a failure: `SimpleRetriever._read_pages` catches it and re-issues the same + page with a smaller page size. It is raised on every reduction, including the ones a correctly configured + connector is expected to make, so the message describes what happened and nothing else - it must not read + as a bug report when it surfaces as the `__context__` of a later failure. + + A reduction the connector cannot honor raises `PageSizeReductionNotSupportedException` instead, but only on + the `SimpleRetriever` path, which is the only one that knows a reduction was impossible. A `CustomRetriever`, + or a plain Python-CDK `HttpStream` whose error handler returns `ResponseAction.REDUCE_PAGE_SIZE`, never + catches this and lets it escape. That escape is kept deliberately - it is the only signal such a stream gets + - and it is typed `config_error` because nothing retries the page there, so a job-level retry would fail the + same way forever. On the `SimpleRetriever` path the type is inert, since the exception never leaves the loop. + """ + + def __init__( + self, stream_name: Optional[str] = None, error_message: Optional[str] = None + ) -> None: + stream = f" of stream {stream_name}" if stream_name else "" + # The error handler's own `error_message` has no other outlet: the raise precedes every site that + # logs it, so an author who writes one on the reduction filter would otherwise never see it. + detail = f": {error_message}" if error_message else "" + super().__init__( + internal_message=f"An error handler{stream} resolved to REDUCE_PAGE_SIZE{detail}", + message=f"The API rejected a page{stream} and asked the connector for a smaller one. If this message ends a sync, the stream is not set up to request a smaller page: add `page_size_reduction` to its retriever, or remove the REDUCE_PAGE_SIZE action from its error handler.", + failure_type=FailureType.config_error, + ) + + +class PageSizeReductionNotSupportedException(AirbyteTracedException): + """ + Raised when a reduction is requested on a retriever that cannot honor it. + + The factory rejects this at config time wherever it can see the error handler, so reaching this means the + action came from somewhere it cannot inspect: a custom error handler, or a custom requester or retriever. + """ + + def __init__(self, stream_name: Optional[str] = None) -> None: + stream = f"Stream {stream_name}" if stream_name else "The stream" + super().__init__( + internal_message=f"An error handler of stream {stream_name} resolved to REDUCE_PAGE_SIZE but the retriever it is attached to defines no page_size_reduction. The action is only supported on the main requester of a SimpleRetriever that defines page_size_reduction.", + message=f"{stream} resolves an API response to the REDUCE_PAGE_SIZE action but is not set up to send a smaller page. Add `page_size_reduction` to the stream's retriever, or remove the REDUCE_PAGE_SIZE action from its error handler.", + failure_type=FailureType.config_error, + ) diff --git a/bin/generate_component_manifest_files.py b/bin/generate_component_manifest_files.py index 51b3d8efbf..098c108d3f 100755 --- a/bin/generate_component_manifest_files.py +++ b/bin/generate_component_manifest_files.py @@ -138,6 +138,14 @@ async def main(): "--set-default-enum-member", "--use-double-quotes", "--remove-special-field-name-prefix", + # NOTE: without `--field-constraints`, a numeric `minimum`/`exclusiveMinimum` in the YAML + # becomes a `conint(...)`/`confloat(...)` annotation, which mypy rejects - so regenerating + # today produces a file that does not type check, on fields that predate this comment + # (DynamicStreamCheckConfig.stream_count, both backoff strategies, AsyncRetriever, and + # PageSizeReduction). Adding the flag rewrites those fields across the generated module + # and is its own change; until then `declarative_component_schema.py` is edited by hand + # when a bounded numeric field is added, with the bound expressed as `Field(ge=...)`. + # The YAML bounds are what manifests are validated against either way. # allow usage of the extra key such as `deprecated`, etc. "--field-extra-keys", # account the `deprecated` flag provided for the field. diff --git a/unit_tests/connector_builder/test_connector_builder_handler.py b/unit_tests/connector_builder/test_connector_builder_handler.py index 5842e86162..c28f0f1485 100644 --- a/unit_tests/connector_builder/test_connector_builder_handler.py +++ b/unit_tests/connector_builder/test_connector_builder_handler.py @@ -1919,3 +1919,131 @@ def test_full_resolve_manifest(valid_resolve_manifest_config_file): } assert resolved_manifest.record.data["manifest"] == expected_resolved_manifest assert resolved_manifest.record.stream == "full_resolve_manifest" + + +_PAGE_SIZE_REDUCTION_STREAM_NAME = "reducing_stream" +_PAGE_SIZE_REDUCTION_MANIFEST = { + "version": "0.30.3", + "type": "DeclarativeSource", + "check": {"type": "CheckStream", "stream_names": [_PAGE_SIZE_REDUCTION_STREAM_NAME]}, + "streams": [ + { + "type": "DeclarativeStream", + "name": _PAGE_SIZE_REDUCTION_STREAM_NAME, + "schema_loader": {"type": "InlineSchemaLoader", "schema": {"type": "object"}}, + "retriever": { + "type": "SimpleRetriever", + "page_size_reduction": {"type": "PageSizeReduction"}, + "requester": { + "type": "HttpRequester", + "url_base": "https://demonslayers.com/api/v1/", + "path": "hashiras", + "http_method": "GET", + "error_handler": { + "type": "DefaultErrorHandler", + "response_filters": [ + { + "type": "HttpResponseFilter", + "http_codes": [502], + "action": "REDUCE_PAGE_SIZE", + } + ], + }, + }, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": ["result"]}, + }, + "paginator": { + "type": "DefaultPaginator", + "page_size_option": { + "type": "RequestOption", + "inject_into": "request_parameter", + "field_name": "first", + }, + # `page_size_reduction` rejects a RequestPath page token: the next-page URL built by the + # API already carries the page size, so the reduced one would be sent next to it. + "page_token_option": { + "type": "RequestOption", + "inject_into": "request_parameter", + "field_name": "after", + }, + "pagination_strategy": { + "type": "CursorPagination", + "page_size": 100, + "cursor_value": "{{ response._metadata.next }}", + "stop_condition": "{{ not response._metadata.next }}", + }, + }, + }, + } + ], + "spec": { + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": [], + "properties": {}, + }, + "documentation_url": "https://example.org", + "type": "Spec", + }, +} + + +def _create_502_page_response(): + response = requests.Response() + response.status_code = 502 + response._content = b"{}" + response.headers["Content-Type"] = "application/json" + response.request = _create_request() + return response + + +@patch("airbyte_cdk.sources.declarative.retrievers.page_size_reducer.time.sleep", lambda _: None) +@patch.object( + requests.Session, + "send", + side_effect=( + _create_502_page_response(), + _create_page_response({"result": [{"id": 0}], "_metadata": {"next": "next"}}), + _create_page_response({"result": [{"id": 1}], "_metadata": {}}), + ), +) +def test_given_page_size_reduction_when_test_read_then_the_retry_does_not_count_as_a_page( + mock_http_stream, +): + """ + The Connector Builder bounds a slice by the number of request/response logs it sees, not by the paginator's + own counter. A response that only triggers a reduction is never a page of the stream, so counting it would + show empty error pages and report "limit reached" on a read that merely retried. + """ + # three responses are served but only two of them are pages, so the limit is not reached + limits = TestLimits(max_records=100, max_pages_per_slice=3, max_slices=2) + catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream( + name=_PAGE_SIZE_REDUCTION_STREAM_NAME, + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh], + ), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.append, + ) + ] + ) + config = {"__injected_declarative_manifest": _PAGE_SIZE_REDUCTION_MANIFEST} + source = create_source(config=config, limits=limits, catalog=catalog, state=None) + + output_data = read_stream(source, config, catalog, None, limits).record.data + + pages = output_data["slices"][0]["pages"] + assert [page["response"]["status"] for page in pages] == [200, 200] + assert [record["id"] for page in pages for record in page["records"]] == [0, 1] + assert output_data["test_read_limit_reached"] is False + # the failed attempt stays visible, just not as a page + assert [ + auxiliary_request["response"]["status"] + for auxiliary_request in output_data["auxiliary_requests"] + ] == [502] diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index 21c99adc71..0b03204513 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -79,6 +79,7 @@ from airbyte_cdk.sources.declarative.models import DeclarativeStream as DeclarativeStreamModel from airbyte_cdk.sources.declarative.models import DefaultPaginator as DefaultPaginatorModel from airbyte_cdk.sources.declarative.models import DpathExtractor as DpathExtractorModel +from airbyte_cdk.sources.declarative.models import FileUploader as FileUploaderModel from airbyte_cdk.sources.declarative.models import ( GroupingPartitionRouter as GroupingPartitionRouterModel, ) @@ -88,6 +89,9 @@ from airbyte_cdk.sources.declarative.models import OAuthAuthenticator as OAuthAuthenticatorModel from airbyte_cdk.sources.declarative.models import PropertyChunking as PropertyChunkingModel from airbyte_cdk.sources.declarative.models import RecordSelector as RecordSelectorModel +from airbyte_cdk.sources.declarative.models import ( + SessionTokenAuthenticator as SessionTokenAuthenticatorModel, +) from airbyte_cdk.sources.declarative.models import SimpleRetriever as SimpleRetrieverModel from airbyte_cdk.sources.declarative.models import Spec as SpecModel from airbyte_cdk.sources.declarative.models import ( @@ -163,6 +167,9 @@ PageIncrement, StopConditionPaginationStrategyDecorator, ) +from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import ( + PaginationStrategy, +) from airbyte_cdk.sources.declarative.requesters.query_properties import ( PropertiesFromEndpoint, PropertyChunking, @@ -191,6 +198,10 @@ LazySimpleRetriever, SimpleRetriever, ) +from airbyte_cdk.sources.declarative.retrievers.page_size_reducer import ( + PageSizeReduction, + PageSizeResetPolicy, +) from airbyte_cdk.sources.declarative.schema import InlineSchemaLoader, JsonFileSchemaLoader from airbyte_cdk.sources.declarative.schema.caching_schema_loader_decorator import ( CachingSchemaLoaderDecorator, @@ -6424,6 +6435,1004 @@ def get_schema_loader(stream: DefaultStream): return stream._stream_partition_generator._partition_factory._schema_loader._decorated +_PAGE_SIZE_REDUCTION_STREAM = """ +type: DeclarativeStream +name: Test +primary_key: id +schema_loader: + type: InlineSchemaLoader + schema: + type: object +retriever: + type: SimpleRetriever + {page_size_reduction} + requester: + type: HttpRequester + url_base: "https://airbyte.io" + path: "/graphql" + http_method: POST + error_handler: + type: DefaultErrorHandler + response_filters: + - type: HttpResponseFilter + http_codes: [502, 504] + action: {action} + paginator: + type: DefaultPaginator + pagination_strategy: + {pagination_strategy} + {page_size_option} + {page_token_option} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["items"] +""" + +_CURSOR_PAGINATION_STRATEGY = ( + 'type: CursorPagination\n page_size: 100\n cursor_value: "{{ response.next }}"' +) +_PAGE_SIZE_OPTION = """page_size_option: + type: RequestOption + inject_into: request_parameter + field_name: first""" + + +def _page_size_reduction_stream( + page_size_reduction="page_size_reduction:\n type: PageSizeReduction", + action="REDUCE_PAGE_SIZE", + pagination_strategy=_CURSOR_PAGINATION_STRATEGY, + page_size_option=_PAGE_SIZE_OPTION, + page_token_option="", +): + content = _PAGE_SIZE_REDUCTION_STREAM.format( + page_size_reduction=page_size_reduction, + action=action, + pagination_strategy=pagination_strategy, + page_size_option=page_size_option, + page_token_option=page_token_option, + ) + stream_manifest = transformer.propagate_types_and_parameters( + "", resolver.preprocess_manifest(YamlDeclarativeSource._parse(content)), {} + ) + return factory.create_component( + model_type=DeclarativeStreamModel, component_definition=stream_manifest, config={} + ) + + +def test_given_page_size_reduction_then_create_retriever_with_defaults(): + retriever = get_retriever(_page_size_reduction_stream()) + + assert retriever.page_size_reduction == PageSizeReduction( + reduction_factor=2, + minimum_page_size=1, + max_attempts=5, + reset_policy=PageSizeResetPolicy.NEVER, + ) + + +def test_given_page_size_reduction_values_then_create_retriever_with_those_values(): + retriever = get_retriever( + _page_size_reduction_stream( + page_size_reduction=( + "page_size_reduction:\n" + " type: PageSizeReduction\n" + " reduction_factor: 4\n" + " minimum_page_size: 10\n" + " max_attempts: 2\n" + " reset_policy: AFTER_SUCCESSFUL_PAGE" + ) + ) + ) + + assert retriever.page_size_reduction == PageSizeReduction( + reduction_factor=4, + minimum_page_size=10, + max_attempts=2, + reset_policy=PageSizeResetPolicy.AFTER_SUCCESSFUL_PAGE, + ) + + +def test_given_no_page_size_reduction_then_retriever_has_none(): + retriever = get_retriever(_page_size_reduction_stream(page_size_reduction="", action="RETRY")) + + assert retriever.page_size_reduction is None + + +def test_given_reduce_page_size_action_without_page_size_reduction_then_raise(): + with pytest.raises(ValueError) as exception: + _page_size_reduction_stream(page_size_reduction="") + + assert "REDUCE_PAGE_SIZE" in str(exception.value) + + +def test_given_page_increment_and_page_size_reduction_then_raise(): + with pytest.raises(ValueError) as exception: + _page_size_reduction_stream(pagination_strategy="type: PageIncrement\n page_size: 100") + + assert "PageIncrement" in str(exception.value) + + +def test_given_no_page_size_option_and_page_size_reduction_then_raise(): + with pytest.raises(ValueError) as exception: + _page_size_reduction_stream(page_size_option="") + + assert "page_size_option" in str(exception.value) + + +def test_given_request_path_page_token_option_and_page_size_reduction_then_raise(): + # The next page is then a URL built by the API which already carries the page size it echoed back, so the + # reduced page size would be sent next to the original one and the API picks which one it honors. + with pytest.raises(ValueError) as exception: + _page_size_reduction_stream(page_token_option="page_token_option:\n type: RequestPath") + + assert "RequestPath" in str(exception.value) + + +def test_given_request_option_page_token_option_and_page_size_reduction_then_create_retriever(): + retriever = get_retriever( + _page_size_reduction_stream( + page_token_option=( + "page_token_option:\n" + " type: RequestOption\n" + " inject_into: request_parameter\n" + " field_name: after" + ) + ) + ) + + assert retriever.page_size_reduction is not None + + +def test_given_failure_message_then_create_retriever_with_that_message(): + retriever = get_retriever( + _page_size_reduction_stream( + page_size_reduction=( + "page_size_reduction:\n" + " type: PageSizeReduction\n" + " failure_message: Select fewer fields on this stream." + ) + ) + ) + + assert retriever.page_size_reduction.failure_message == "Select fewer fields on this stream." + + +class _StrategyHonoringOverride(PaginationStrategy): + """A custom strategy that can be told the reduced page size.""" + + @property + def initial_token(self): + return None + + def next_page_token( + self, + response, + last_page_size, + last_record, + last_page_token_value=None, + page_size_override=None, + ): + return None + + def get_page_size(self): + return 100 + + +class _StrategyIgnoringOverride(PaginationStrategy): + """A custom strategy predating the feature: it would raise TypeError on the first reduction.""" + + @property + def initial_token(self): + return None + + def next_page_token(self, response, last_page_size, last_record, last_page_token_value=None): + return None + + def get_page_size(self): + return 100 + + +def test_given_custom_pagination_strategy_accepting_the_override_and_page_size_reduction_then_create_retriever(): + """A custom strategy is written by whoever enables the reduction, so it is allowed as long + as it can receive the reduced page size. Rejecting every custom strategy would exclude the + GraphQL streams this feature exists for.""" + retriever = get_retriever( + _page_size_reduction_stream( + pagination_strategy=( + "type: CustomPaginationStrategy\n" + " class_name: unit_tests.sources.declarative.parsers.test_model_to_component_factory._StrategyHonoringOverride" + ) + ) + ) + + assert retriever.page_size_reduction is not None + + +def test_given_custom_pagination_strategy_ignoring_the_override_and_page_size_reduction_then_raise(): + with pytest.raises(ValueError, match="page_size_override"): + _page_size_reduction_stream( + pagination_strategy=( + "type: CustomPaginationStrategy\n" + " class_name: unit_tests.sources.declarative.parsers.test_model_to_component_factory._StrategyIgnoringOverride" + ) + ) + + +class _StrategyAcceptingKwargs(PaginationStrategy): + """A custom strategy that swallows the override through `**kwargs` rather than naming it.""" + + @property + def initial_token(self): + return None + + def next_page_token( + self, response, last_page_size, last_record, last_page_token_value=None, **kwargs + ): + return None + + def get_page_size(self): + return 100 + + +class _StrategySubclassingPageIncrement(PageIncrement): + """A custom strategy that inherits `next_page_token` - and therefore its rejection - from PageIncrement.""" + + +class _NotAPaginationStrategy: + """A class_name that builds but is not a pagination strategy: it defines no `next_page_token`.""" + + def __init__(self, **kwargs): + pass + + @property + def initial_token(self): + return None + + def get_page_size(self): + return 100 + + +def test_given_custom_pagination_strategy_accepting_kwargs_and_page_size_reduction_then_create_retriever(): + retriever = get_retriever( + _page_size_reduction_stream( + pagination_strategy=( + "type: CustomPaginationStrategy\n" + " class_name: unit_tests.sources.declarative.parsers.test_model_to_component_factory._StrategyAcceptingKwargs" + ) + ) + ) + + assert retriever.page_size_reduction is not None + + +def test_given_custom_pagination_strategy_subclassing_page_increment_then_raise(): + """ + `PageIncrement.next_page_token` declares `page_size_override` only to reject it, so a subclass that does + not override the method satisfies the signature check while being unable to honor a reduction. + """ + with pytest.raises(ValueError, match="PageIncrement"): + _page_size_reduction_stream( + pagination_strategy=( + "type: CustomPaginationStrategy\n" + " page_size: 100\n" + " class_name: unit_tests.sources.declarative.parsers.test_model_to_component_factory._StrategySubclassingPageIncrement" + ) + ) + + +def test_given_custom_pagination_strategy_without_next_page_token_then_raise_value_error(): + """Every other config-time failure in the factory is a ValueError; this one used to escape as a bare + AttributeError, which is reported as a system error rather than a manifest problem.""" + with pytest.raises(ValueError, match="next_page_token"): + _page_size_reduction_stream( + pagination_strategy=( + "type: CustomPaginationStrategy\n" + " class_name: unit_tests.sources.declarative.parsers.test_model_to_component_factory._NotAPaginationStrategy" + ) + ) + + +_CURSOR_PAGINATION_WITH_STOP_CONDITION = ( + 'type: CursorPagination\n page_size: 100\n cursor_value: "{{{{ response.next }}}}"\n' + ' stop_condition: "{condition}"' +) + + +def test_given_stop_condition_compares_last_page_size_to_a_constant_then_raise(): + """ + A full page at the reduced size satisfies `last_page_size < 100`, so the pagination would end early and + silently drop the rest of the partition. + """ + with pytest.raises(ValueError, match="last_page_size"): + _page_size_reduction_stream( + pagination_strategy=_CURSOR_PAGINATION_WITH_STOP_CONDITION.format( + condition="{{ last_page_size < 100 }}" + ) + ) + + +def test_given_stop_condition_compares_last_page_size_to_the_requested_page_size_then_create_retriever(): + retriever = get_retriever( + _page_size_reduction_stream( + pagination_strategy=_CURSOR_PAGINATION_WITH_STOP_CONDITION.format( + condition="{{ last_page_size < page_size }}" + ) + ) + ) + + assert retriever.page_size_reduction is not None + + +def test_given_stop_condition_does_not_mention_last_page_size_then_create_retriever(): + retriever = get_retriever( + _page_size_reduction_stream( + pagination_strategy=_CURSOR_PAGINATION_WITH_STOP_CONDITION.format( + condition="{{ not response.next }}" + ) + ) + ) + + assert retriever.page_size_reduction is not None + + +@pytest.mark.parametrize( + "condition", + [ + # the three literal forms in the monorepo today: source-discord x3 + pytest.param("{{ last_page_size < 100 }}", id="literal_100"), + pytest.param("{{ last_page_size < 200 }}", id="literal_200"), + pytest.param("{{ last_page_size < 1000 }}", id="literal_1000"), + # the form the string-matching gate let through: `\bpage_size\b` matches inside `config['page_size']` + pytest.param("{{ last_page_size < config['page_size'] }}", id="config_reference"), + pytest.param("{{ last_page_size < config.page_size }}", id="config_attribute_reference"), + pytest.param("{{ last_page_size < parameters['page_size'] }}", id="parameters_reference"), + pytest.param("{{ last_page_size <= 99 }}", id="less_than_or_equal_to_a_literal"), + pytest.param("{{ 100 > last_page_size }}", id="reversed_operands"), + pytest.param( + "{{ last_page_size < 100 or not response.next }}", id="inside_a_larger_expression" + ), + pytest.param("{{ last_page_size | int < 100 }}", id="through_a_filter"), + ], +) +def test_given_stop_condition_compares_last_page_size_to_a_value_that_does_not_follow_the_reduction_then_raise( + condition, +): + """ + A full page at the reduced size satisfies each of these, so the pagination would end early and silently + drop the rest of the partition. `config['page_size']` is the one the previous string-matching gate + accepted: it contains the substring `page_size` but holds the configured size, not the requested one. + """ + with pytest.raises(ValueError, match="last_page_size"): + _page_size_reduction_stream( + pagination_strategy=_CURSOR_PAGINATION_WITH_STOP_CONDITION.format(condition=condition) + ) + + +@pytest.mark.parametrize( + "condition", + [ + # 11 of the 14 `last_page_size` stop conditions in the monorepo, including all 6 in + # source-zendesk-support and all 5 in source-trello. A reduction cannot make an empty page non-empty. + pytest.param("{{ last_page_size == 0 }}", id="emptiness_test"), + pytest.param("{{ 0 == last_page_size }}", id="emptiness_test_reversed"), + pytest.param( + "{{ last_page_size == 0 or not response.next }}", id="emptiness_test_or_no_cursor" + ), + # equivalent to emptiness, because `minimum_page_size` defaults to 1 + pytest.param("{{ last_page_size < 1 }}", id="less_than_the_minimum_page_size"), + pytest.param("{{ last_page_size <= 0 }}", id="at_most_zero"), + # the sanctioned form, and variations on it that still follow the reduction + pytest.param("{{ last_page_size < page_size }}", id="requested_page_size"), + pytest.param( + "{{ last_page_size < page_size | int }}", id="requested_page_size_through_a_filter" + ), + pytest.param("{{ page_size > last_page_size }}", id="requested_page_size_reversed"), + # a reduction only makes a page smaller, so a lower bound can only stop being satisfied + pytest.param("{{ last_page_size > 1000 }}", id="lower_bound"), + ], +) +def test_given_reduction_safe_stop_condition_then_create_retriever(condition): + retriever = get_retriever( + _page_size_reduction_stream( + pagination_strategy=_CURSOR_PAGINATION_WITH_STOP_CONDITION.format(condition=condition) + ) + ) + + assert retriever.page_size_reduction is not None + + +@pytest.mark.parametrize( + "condition", + [ + pytest.param( + "{{ last_page_size == config['page_size'] }}", id="equality_against_a_config_value" + ), + pytest.param("{{ last_page_size is lt(100) }}", id="jinja_test_rather_than_a_comparison"), + pytest.param("{{ last_page_size < }}", id="not_a_valid_jinja_expression"), + ], +) +def test_given_stop_condition_shape_cannot_be_classified_then_warn_and_create_retriever( + caplog, condition +): + """ + The gate runs at stream construction, so rejecting a manifest it merely does not understand would take + `check` and `discover` down with `read`. A shape outside the analysis warns instead. + """ + with caplog.at_level(logging.WARNING): + retriever = get_retriever( + _page_size_reduction_stream( + pagination_strategy=_CURSOR_PAGINATION_WITH_STOP_CONDITION.format( + condition=condition + ) + ) + ) + + assert retriever.page_size_reduction is not None + assert "could not be checked against the reduction" in caplog.text + + +def test_given_no_paginator_and_page_size_reduction_then_raise(): + content = """ +type: DeclarativeStream +name: Test +primary_key: id +schema_loader: + type: InlineSchemaLoader + schema: + type: object +retriever: + type: SimpleRetriever + page_size_reduction: + type: PageSizeReduction + requester: + type: HttpRequester + url_base: "https://airbyte.io" + path: "/items" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["items"] +""" + stream_manifest = transformer.propagate_types_and_parameters( + "", resolver.preprocess_manifest(YamlDeclarativeSource._parse(content)), {} + ) + + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=DeclarativeStreamModel, component_definition=stream_manifest, config={} + ) + + assert "DefaultPaginator" in str(exception.value) + + +_OFFSET_INCREMENT_STRATEGY = "type: OffsetIncrement\n page_size: 100" + + +def test_given_offset_increment_and_page_size_reduction_then_create_retriever(): + """ + `OffsetIncrement` is the only strategy whose stop condition depends on the page size, so a typo in the + validator's isinstance tuple would reject every manifest this feature is meant to support. + """ + retriever = get_retriever( + _page_size_reduction_stream(pagination_strategy=_OFFSET_INCREMENT_STRATEGY) + ) + + assert retriever.page_size_reduction == PageSizeReduction() + + +def test_given_no_page_size_on_the_strategy_and_page_size_reduction_then_raise(): + """ + Without a `page_size` the paginator injects nothing, so the reduction is a dead end that would only + surface on the first failing response, after records have been emitted. + """ + with pytest.raises(ValueError) as exception: + _page_size_reduction_stream(pagination_strategy="type: OffsetIncrement") + + assert "page_size" in str(exception.value) + + +def test_given_minimum_page_size_not_below_the_page_size_then_raise(): + with pytest.raises(ValueError) as exception: + _page_size_reduction_stream( + page_size_reduction=( + "page_size_reduction:\n type: PageSizeReduction\n minimum_page_size: 100" + ) + ) + + assert "minimum_page_size" in str(exception.value) + + +def test_given_composite_error_handler_with_reduce_page_size_action_then_require_page_size_reduction(): + """The action can be nested in a CompositeErrorHandler, which the guard has to recurse into.""" + stream_definition = { + "type": "DeclarativeStream", + "name": "Test", + "primary_key": "id", + "schema_loader": {"type": "InlineSchemaLoader", "schema": {"type": "object"}}, + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/graphql", + "http_method": "POST", + "error_handler": { + "type": "CompositeErrorHandler", + "error_handlers": [ + { + "type": "DefaultErrorHandler", + "response_filters": [ + { + "type": "HttpResponseFilter", + "http_codes": [429], + "action": "RATE_LIMITED", + } + ], + }, + { + "type": "DefaultErrorHandler", + "response_filters": [ + { + "type": "HttpResponseFilter", + "http_codes": [502], + "action": "REDUCE_PAGE_SIZE", + } + ], + }, + ], + }, + }, + "paginator": { + "type": "DefaultPaginator", + "page_size_option": { + "type": "RequestOption", + "inject_into": "request_parameter", + "field_name": "first", + }, + "pagination_strategy": { + "type": "CursorPagination", + "page_size": 100, + "cursor_value": "{{ response.next }}", + }, + }, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": ["items"]}, + }, + }, + } + + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=DeclarativeStreamModel, + component_definition=stream_definition, + config={}, + ) + + assert "REDUCE_PAGE_SIZE" in str(exception.value) + + # and the same manifest with the block present builds + stream_definition["retriever"]["page_size_reduction"] = {"type": "PageSizeReduction"} + retriever = get_retriever( + factory.create_component( + model_type=DeclarativeStreamModel, + component_definition=stream_definition, + config={}, + ) + ) + assert retriever.page_size_reduction == PageSizeReduction() + + +def test_given_page_size_reduction_without_reduce_page_size_action_then_warn(caplog): + """ + A `CustomErrorHandler` can resolve to the action without being inspectable, so this cannot raise. It must + not stay silent either: the feature would be dead on a stream that only exists because it would fail. + """ + with caplog.at_level(logging.WARNING, logger="airbyte.model_to_component_factory"): + retriever = get_retriever(_page_size_reduction_stream(action="RETRY")) + + assert retriever.page_size_reduction == PageSizeReduction() + assert "REDUCE_PAGE_SIZE" in caplog.text + + +def test_given_minimum_page_size_out_of_reach_of_max_attempts_then_warn(caplog): + """ + Each reduction divides the page size by `reduction_factor` and spends one attempt, so the two settings are + two bounds and the tighter one wins. A floor the budget cannot reach in one run of failing pages is inert + there, and the error branch written for hitting it never fires on that run. + """ + with caplog.at_level(logging.WARNING, logger="airbyte.model_to_component_factory"): + retriever = get_retriever( + _page_size_reduction_stream( + page_size_reduction=( + "page_size_reduction:\n" + " type: PageSizeReduction\n" + " minimum_page_size: 10\n" + " max_attempts: 2" + ) + ) + ) + + assert retriever.page_size_reduction.minimum_page_size == 10 + assert "minimum_page_size" in caplog.text + assert "25 records per page" in caplog.text + assert "`max_attempts` of at least 4" in caplog.text + + +def test_given_minimum_page_size_within_reach_of_max_attempts_then_do_not_warn(caplog): + with caplog.at_level(logging.WARNING, logger="airbyte.model_to_component_factory"): + get_retriever( + _page_size_reduction_stream( + page_size_reduction=( + "page_size_reduction:\n" + " type: PageSizeReduction\n" + " minimum_page_size: 10\n" + " max_attempts: 5" + ) + ) + ) + + assert "minimum_page_size" not in caplog.text + + +@pytest.mark.parametrize( + "page_size_reduction", + [ + pytest.param("page_size_reduction:\n type: PageSizeReduction", id="floor_left_default"), + pytest.param( + "page_size_reduction:\n type: PageSizeReduction\n minimum_page_size: 1", + id="floor_set_to_the_default_value", + ), + ], +) +def test_given_no_floor_worth_reaching_then_do_not_warn_about_its_reachability( + page_size_reduction, caplog +): + # A floor of 1 is out of reach of the default budget on any page size above 32, so warning about it would + # fire on nearly every stream that opts in - including one that only spells the default out longhand. + with caplog.at_level(logging.WARNING, logger="airbyte.model_to_component_factory"): + get_retriever(_page_size_reduction_stream(page_size_reduction=page_size_reduction)) + + assert "minimum_page_size" not in caplog.text + + +def test_given_query_properties_and_page_size_reduction_then_raise(): + """ + Records of the earlier property chunks were already emitted when a later chunk asks for a smaller page, so + re-issuing the page would emit them twice. + """ + content = _PAGE_SIZE_REDUCTION_STREAM.format( + page_size_reduction="page_size_reduction:\n type: PageSizeReduction", + action="REDUCE_PAGE_SIZE", + pagination_strategy=_CURSOR_PAGINATION_STRATEGY, + page_size_option=_PAGE_SIZE_OPTION, + page_token_option="", + ).replace( + " http_method: POST\n", + " http_method: POST\n" + " query_properties:\n" + " type: QueryProperties\n" + ' property_list: ["a", "b"]\n', + ) + stream_manifest = transformer.propagate_types_and_parameters( + "", resolver.preprocess_manifest(YamlDeclarativeSource._parse(content)), {} + ) + + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=DeclarativeStreamModel, component_definition=stream_manifest, config={} + ) + + assert "query properties" in str(exception.value) + + +def _lazy_read_stream_definition(page_size_reduction, action): + return { + "type": "DeclarativeStream", + "name": "items", + "primary_key": [], + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": {"type": "object", "properties": {}}, + }, + "retriever": { + "type": "SimpleRetriever", + **page_size_reduction, + "requester": { + "type": "HttpRequester", + "url_base": "https://api.test.com", + "path": "parent/{{ stream_partition.parent_id }}/items", + "http_method": "GET", + "error_handler": { + "type": "DefaultErrorHandler", + "response_filters": [ + {"type": "HttpResponseFilter", "http_codes": [502], "action": action} + ], + }, + }, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": ["data"]}, + }, + "paginator": { + "type": "DefaultPaginator", + "page_size_option": { + "type": "RequestOption", + "inject_into": "request_parameter", + "field_name": "first", + }, + "pagination_strategy": { + "type": "CursorPagination", + "page_size": 100, + "cursor_value": '{{ response["data"][-1]["id"] }}', + }, + }, + "partition_router": { + "type": "SubstreamPartitionRouter", + "parent_stream_configs": [ + { + "type": "ParentStreamConfig", + "parent_key": "id", + "partition_field": "parent_id", + "lazy_read_pointer": ["items"], + "stream": { + "type": "DeclarativeStream", + "name": "parent", + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": {"type": "object", "properties": {}}, + }, + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url_base": "https://api.test.com", + "path": "/parents", + "http_method": "GET", + }, + "record_selector": { + "type": "RecordSelector", + "extractor": { + "type": "DpathExtractor", + "field_path": ["data"], + }, + }, + }, + }, + } + ], + }, + }, + } + + +@pytest.mark.parametrize( + "page_size_reduction, action", + [ + pytest.param( + {"page_size_reduction": {"type": "PageSizeReduction"}}, + "RETRY", + id="page_size_reduction_without_the_action", + ), + pytest.param({}, "REDUCE_PAGE_SIZE", id="action_without_page_size_reduction"), + pytest.param( + {"page_size_reduction": {"type": "PageSizeReduction"}}, + "REDUCE_PAGE_SIZE", + id="both", + ), + ], +) +@pytest.mark.parametrize( + "has_state", + [pytest.param(False, id="first_sync"), pytest.param(True, id="resumed_sync")], +) +def test_given_lazy_read_pointer_and_page_size_reduction_then_raise( + page_size_reduction, action, has_state +): + """ + `LazySimpleRetriever` paginates the parent's embedded pages, so there is no page of its own to re-issue. + The rejection must not depend on the presence of state: gating it on the lazy branch, which only applies + while a stream has no state, would accept the same manifest from the second sync onwards. + """ + connector_state_manager = ConnectorStateManager( + state=[ + AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="items"), + stream_state=AirbyteStateBlob({"created": "2025-01-01T00:00:00+0000"}), + ), + ) + ] + if has_state + else [] + ) + + with pytest.raises(ValueError) as exception: + ModelToComponentFactory(connector_state_manager=connector_state_manager).create_component( + model_type=DeclarativeStreamModel, + component_definition=_lazy_read_stream_definition(page_size_reduction, action), + config=input_config, + ) + + assert "lazy_read_pointer" in str(exception.value) + + +def test_given_file_uploader_and_page_size_reduction_then_raise(): + """ + The file uploader sends one request per record from inside the page's record generator, so a reduction + asked for halfway through a page would re-emit the records the generator already yielded. + """ + content = ( + _PAGE_SIZE_REDUCTION_STREAM.format( + page_size_reduction="page_size_reduction:\n type: PageSizeReduction", + action="REDUCE_PAGE_SIZE", + pagination_strategy=_CURSOR_PAGINATION_STRATEGY, + page_size_option=_PAGE_SIZE_OPTION, + page_token_option="", + ) + + """file_uploader: + type: FileUploader + requester: + type: HttpRequester + url_base: "https://airbyte.io" + path: "/download" + download_target_extractor: + type: DpathExtractor + field_path: ["url"] +""" + ) + stream_manifest = transformer.propagate_types_and_parameters( + "", resolver.preprocess_manifest(YamlDeclarativeSource._parse(content)), {} + ) + + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=DeclarativeStreamModel, component_definition=stream_manifest, config={} + ) + + assert "file_uploader" in str(exception.value) + + +_REDUCE_PAGE_SIZE_ERROR_HANDLER = { + "type": "DefaultErrorHandler", + "response_filters": [ + {"type": "HttpResponseFilter", "http_codes": [502], "action": "REDUCE_PAGE_SIZE"} + ], +} + + +def test_given_reduce_page_size_action_on_the_file_uploader_requester_then_raise(): + """ + `error_handler` is defined on `HttpRequester`, so the action is manifest-legal on requesters that have no + page of their own. Nothing there can honor it, so it is rejected rather than raised mid-sync. + """ + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=FileUploaderModel, + component_definition={ + "type": "FileUploader", + "requester": { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/download", + "error_handler": _REDUCE_PAGE_SIZE_ERROR_HANDLER, + }, + "download_target_extractor": { + "type": "DpathExtractor", + "field_path": ["url"], + }, + }, + config={}, + ) + + assert "REDUCE_PAGE_SIZE" in str(exception.value) + + +def test_given_reduce_page_size_action_on_the_login_requester_then_raise(): + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=SessionTokenAuthenticatorModel, + component_definition={ + "type": "SessionTokenAuthenticator", + "login_requester": { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/login", + "http_method": "POST", + "error_handler": _REDUCE_PAGE_SIZE_ERROR_HANDLER, + }, + "session_token_path": ["token"], + "request_authentication": { + "type": "ApiKey", + "inject_into": { + "type": "RequestOption", + "inject_into": "header", + "field_name": "Authorization", + }, + }, + }, + config={}, + name="a_stream", + ) + + assert "REDUCE_PAGE_SIZE" in str(exception.value) + + +@pytest.mark.parametrize( + "requester_field", + [ + "creation_requester", + "polling_requester", + "download_requester", + "download_target_requester", + "abort_requester", + "delete_requester", + ], +) +def test_given_reduce_page_size_action_on_an_async_retriever_requester_then_raise(requester_field): + definition = { + "type": "AsyncRetriever", + "status_mapping": { + "type": "AsyncJobStatusMap", + "running": ["running"], + "completed": ["ready"], + "failed": ["failed"], + "timeout": ["timeout"], + }, + "status_extractor": {"type": "DpathExtractor", "field_path": ["status"]}, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": ["items"]}, + }, + "creation_requester": { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/jobs", + "http_method": "POST", + }, + "polling_requester": { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/jobs/{{ creation_response.id }}", + }, + "download_requester": { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/jobs/{{ creation_response.id }}/download", + }, + } + definition[requester_field] = { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/jobs", + "error_handler": _REDUCE_PAGE_SIZE_ERROR_HANDLER, + } + if requester_field == "download_target_requester": + # Without it the `download_target_extractor` guard fires first and the assertion below would pass for + # the wrong reason. + definition["download_target_extractor"] = { + "type": "DpathExtractor", + "field_path": ["url"], + } + + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=AsyncRetrieverModel, + component_definition=definition, + config={}, + name="a_stream", + primary_key=None, + stream_slicer=None, + transformations=[], + ) + + assert "REDUCE_PAGE_SIZE" in str(exception.value) + + def get_retriever(stream: Union[DeclarativeStream, DefaultStream]): return ( stream.retriever diff --git a/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py b/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py new file mode 100644 index 0000000000..6d159cd354 --- /dev/null +++ b/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py @@ -0,0 +1,389 @@ +# +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. +# + +import pytest + +from airbyte_cdk.sources.declarative.parsers.stop_condition_safety import ( + LAST_PAGE_SIZE_VARIABLE, + StopConditionSafety, + classify_stop_condition, + references_requested_page_size, +) + + +@pytest.mark.parametrize( + "stop_condition,minimum_page_size,expected", + [ + # The 11 monorepo occurrences of the emptiness test - source-zendesk-support x6, source-trello x5 - + # which the previous string-matching gate rejected. No reduction can make a full page empty. + pytest.param("{{ last_page_size == 0 }}", 1, StopConditionSafety.SAFE, id="emptiness_test"), + pytest.param( + "{{ 0 == last_page_size }}", 1, StopConditionSafety.SAFE, id="emptiness_test_reversed" + ), + pytest.param( + "{{ last_page_size != 0 }}", 1, StopConditionSafety.SAFE, id="non_emptiness_test" + ), + pytest.param( + "{{ last_page_size == 0 or not response.next }}", + 1, + StopConditionSafety.SAFE, + id="emptiness_test_in_a_larger_expression", + ), + # The 3 monorepo occurrences of the literal form - source-discord x3 - plus the equivalents. + pytest.param( + "{{ last_page_size < 100 }}", 1, StopConditionSafety.TRUNCATES, id="literal_100" + ), + pytest.param( + "{{ last_page_size < 200 }}", 1, StopConditionSafety.TRUNCATES, id="literal_200" + ), + pytest.param( + "{{ last_page_size < 1000 }}", 1, StopConditionSafety.TRUNCATES, id="literal_1000" + ), + # The false negative of the string-matching gate: `\bpage_size\b` matches inside `config['page_size']` + # because `'` is a non-word character, so the gate read a configured constant as reduction-aware. + pytest.param( + "{{ last_page_size < config['page_size'] }}", + 1, + StopConditionSafety.TRUNCATES, + id="config_getitem", + ), + pytest.param( + "{{ last_page_size < config.page_size }}", + 1, + StopConditionSafety.TRUNCATES, + id="config_getattr", + ), + pytest.param( + "{{ last_page_size < parameters['page_size'] }}", + 1, + StopConditionSafety.TRUNCATES, + id="parameters_getitem", + ), + # The sanctioned form and the shapes that still follow the reduction. + pytest.param( + "{{ last_page_size < page_size }}", + 1, + StopConditionSafety.SAFE, + id="requested_page_size", + ), + pytest.param( + "{{ last_page_size < page_size | int }}", + 1, + StopConditionSafety.SAFE, + id="requested_page_size_through_a_filter", + ), + pytest.param( + "{{ page_size > last_page_size }}", + 1, + StopConditionSafety.SAFE, + id="requested_page_size_reversed", + ), + # A threshold at or below the smallest page the connector may request is a rewrite of the emptiness + # test, so it is safe - and it stops being safe as soon as the minimum rises above it. + pytest.param("{{ last_page_size < 1 }}", 1, StopConditionSafety.SAFE, id="below_minimum"), + pytest.param("{{ last_page_size <= 0 }}", 1, StopConditionSafety.SAFE, id="at_most_zero"), + pytest.param( + "{{ last_page_size < 10 }}", 10, StopConditionSafety.SAFE, id="at_a_raised_minimum" + ), + pytest.param( + "{{ last_page_size < 11 }}", + 10, + StopConditionSafety.TRUNCATES, + id="above_a_raised_minimum", + ), + # A reduction only makes a full page smaller, so a lower bound can only stop being satisfied. + pytest.param("{{ last_page_size > 1000 }}", 1, StopConditionSafety.SAFE, id="lower_bound"), + pytest.param( + "{{ last_page_size >= 1000 }}", 1, StopConditionSafety.SAFE, id="inclusive_lower_bound" + ), + # A condition that cannot observe the length of a page is unaffected by the reduction, whatever else + # it reads from the response. + pytest.param( + "{{ not response.next }}", 1, StopConditionSafety.SAFE, id="no_last_page_size" + ), + pytest.param( + "{{ response.next is none }}", 1, StopConditionSafety.SAFE, id="a_test_on_the_response" + ), + pytest.param( + "{{ 100 < response.count }}", + 1, + StopConditionSafety.SAFE, + id="a_literal_bounded_from_above", + ), + pytest.param( + "{{ response.count > 100 }}", + 1, + StopConditionSafety.SAFE, + id="a_literal_bounded_from_above_reversed", + ), + # `last_page_size` is not the only way to count the records of a page: the response body carries the + # same number, and these are `{{ last_page_size < 100 }}` counted one layer out. Which response field + # holds a page length is not knowable here, so an upper bound on any value is warned about rather than + # assumed to be safe. All four shapes below are live in the fleet today. + pytest.param( + "{{ response['values']|length < 1000 }}", + 1, + StopConditionSafety.UNKNOWN, + id="a_page_length_read_from_the_response", + ), + pytest.param( + '{{ response.get("count", 0) < 1000 }}', + 1, + StopConditionSafety.UNKNOWN, + id="a_count_read_from_the_response", + ), + pytest.param( + "{{ response.data | length < config['page_size'] }}", + 1, + StopConditionSafety.UNKNOWN, + id="a_page_length_against_a_config_value", + ), + pytest.param( + "{{ not response.result.emailClick or response.result.emailClick|length < 200 }}", + 1, + StopConditionSafety.UNKNOWN, + id="a_page_length_in_a_larger_expression", + ), + pytest.param( + "{{ 100 > response.data | length }}", + 1, + StopConditionSafety.UNKNOWN, + id="a_page_length_read_from_the_response_reversed", + ), + pytest.param( + "{{ not (response.data | length >= 100) }}", + 1, + StopConditionSafety.UNKNOWN, + id="a_negated_lower_bound_on_a_response_value", + ), + # Both operator directions are mirrored: every one of the four ordering operators bounds one of its + # operands from above, and only a literal is certain not to be a page length. Reading `lt`/`lteq` in + # full while `gt`/`gteq` only counted a literal threshold left the first two of these classified safe. + pytest.param( + "{{ config['page_size'] > response['data'] | length }}", + 1, + StopConditionSafety.UNKNOWN, + id="a_page_length_under_a_config_threshold_on_the_left", + ), + pytest.param( + "{{ response.total > response['data'] | length }}", + 1, + StopConditionSafety.UNKNOWN, + id="a_page_length_under_a_response_threshold_on_the_left", + ), + pytest.param( + "{{ response.page >= response.total_pages }}", + 1, + StopConditionSafety.UNKNOWN, + id="two_response_values_compared", + ), + # Shapes the analysis cannot reason about are reported as unknown so the caller can warn rather than + # reject: this runs at stream construction, where a false rejection also breaks `check` and `discover`. + pytest.param( + "{{ last_page_size == config['page_size'] }}", + 1, + StopConditionSafety.UNKNOWN, + id="equality_against_a_config_value", + ), + pytest.param( + "{{ last_page_size is lt(100) }}", + 1, + StopConditionSafety.UNKNOWN, + id="jinja_test_rather_than_a_comparison", + ), + pytest.param( + "{{ last_page_size }}", 1, StopConditionSafety.UNKNOWN, id="no_comparison_at_all" + ), + pytest.param( + "{{ last_page_size < }}", + 1, + StopConditionSafety.UNKNOWN, + id="not_a_valid_jinja_expression", + ), + pytest.param( + "{{ last_page_size > last_page_size }}", + 1, + StopConditionSafety.UNKNOWN, + id="compared_against_itself", + ), + # A transform makes the equality and the lower bound unreadable, but a shrinking upper bound is still + # a truncation whatever `last_page_size` was piped through. + pytest.param( + "{{ last_page_size | int > 100 }}", + 1, + StopConditionSafety.UNKNOWN, + id="lower_bound_through_a_filter", + ), + pytest.param( + "{{ last_page_size | int < 100 }}", + 1, + StopConditionSafety.TRUNCATES, + id="upper_bound_through_a_filter", + ), + # A negation flips what the comparison decides, so a lower bound that is safe on its own becomes the + # dangerous upper bound: `not (last_page_size >= 100)` is `last_page_size < 100`. The comparison alone + # no longer says what the condition does, so it cannot be classified. + pytest.param( + "{{ not (last_page_size >= 100) }}", + 1, + StopConditionSafety.UNKNOWN, + id="negated_lower_bound", + ), + pytest.param( + "{{ not last_page_size >= 100 }}", + 1, + StopConditionSafety.UNKNOWN, + id="negated_lower_bound_without_parentheses", + ), + pytest.param( + "{{ not (last_page_size == 0) }}", + 1, + StopConditionSafety.UNKNOWN, + id="negated_emptiness_test", + ), + pytest.param( + "{{ 1 if last_page_size == 0 else 0 }}", + 1, + StopConditionSafety.UNKNOWN, + id="comparison_inside_a_conditional_expression", + ), + # Arithmetic moves the threshold out of the comparison: `last_page_size - 100 < 0` is another way to + # write `last_page_size < 100`, so neither the literal on the right nor `minimum_page_size` bounds it. + pytest.param( + "{{ last_page_size - 100 < 0 }}", + 1, + StopConditionSafety.UNKNOWN, + id="upper_bound_with_arithmetic", + ), + pytest.param( + "{{ 0 > last_page_size - 100 }}", + 1, + StopConditionSafety.UNKNOWN, + id="upper_bound_with_arithmetic_reversed", + ), + pytest.param( + "{{ last_page_size * 2 < page_size }}", + 1, + StopConditionSafety.UNKNOWN, + id="requested_page_size_with_arithmetic", + ), + # A threshold merely built from `page_size` can hold the configured size again - `max(50, 100)` is 100 - + # so mentioning the variable is not enough for the threshold to follow the reduction. + pytest.param( + "{{ last_page_size < [page_size, 100] | max }}", + 1, + StopConditionSafety.UNKNOWN, + id="requested_page_size_inside_a_list", + ), + pytest.param( + "{{ last_page_size < page_size + 50 }}", + 1, + StopConditionSafety.UNKNOWN, + id="requested_page_size_plus_a_literal", + ), + # A `{% if %}` renders truthy text exactly when its test holds, so the test decides the condition. + pytest.param( + "{% if last_page_size < 100 %}true{% endif %}", + 1, + StopConditionSafety.TRUNCATES, + id="literal_threshold_in_an_if_block", + ), + pytest.param( + "{% if last_page_size == 0 %}true{% endif %}", + 1, + StopConditionSafety.SAFE, + id="emptiness_test_in_an_if_block", + ), + # ... but only while the guarded branch is the whole story: an `else` branch, or a body the CDK reads + # as false, breaks the equivalence between the test and what the condition renders. + pytest.param( + "{% if last_page_size < 100 %}true{% else %}also true{% endif %}", + 1, + StopConditionSafety.UNKNOWN, + id="literal_threshold_in_an_if_block_with_an_else", + ), + pytest.param( + "{% if last_page_size < 100 %}false{% endif %}", + 1, + StopConditionSafety.UNKNOWN, + id="if_block_rendering_a_false_value", + ), + pytest.param( + "{% if last_page_size < 100 %}{% endif %}", + 1, + StopConditionSafety.UNKNOWN, + id="if_block_rendering_nothing", + ), + # Text rendered next to the comparison makes the condition truthy whatever the comparison decided. + pytest.param( + "{{ last_page_size < 100 }} records", + 1, + StopConditionSafety.UNKNOWN, + id="comparison_rendered_next_to_text", + ), + ], +) +def test_classify_stop_condition(stop_condition, minimum_page_size, expected): + verdict, reason = classify_stop_condition(stop_condition, minimum_page_size) + + assert verdict is expected + assert reason + + +def test_given_one_truncating_comparison_among_several_then_truncates(): + verdict, _ = classify_stop_condition( + "{{ last_page_size == 0 or last_page_size < config['page_size'] }}", 1 + ) + + assert verdict is StopConditionSafety.TRUNCATES + + +def test_reason_names_the_value_the_page_size_is_compared_against(): + _, reason = classify_stop_condition("{{ last_page_size < config['page_size'] }}", 1) + + assert "config['page_size']" in reason + + +def test_given_negated_comparison_then_reason_points_at_the_shape(): + verdict, reason = classify_stop_condition("{{ not (last_page_size >= 100) }}", 1) + + assert verdict is StopConditionSafety.UNKNOWN + assert "does not decide the condition on its own" in reason + + +def test_given_truncating_comparison_next_to_a_negated_one_then_truncates(): + # The negated comparison is only warned about, so it must not mask a sibling that does truncate. + verdict, _ = classify_stop_condition( + "{{ not (last_page_size >= 100) or last_page_size < 500 }}", 1 + ) + + assert verdict is StopConditionSafety.TRUNCATES + + +def test_given_page_length_read_from_the_response_then_reason_names_the_expression(): + verdict, reason = classify_stop_condition("{{ response['values']|length < 1000 }}", 1) + + assert verdict is StopConditionSafety.UNKNOWN + assert LAST_PAGE_SIZE_VARIABLE in reason + + +@pytest.mark.parametrize( + "template,expected", + [ + pytest.param("{{ last_page_size < page_size }}", True, id="bare_variable"), + pytest.param("{{ page_size }}", True, id="rendered_on_its_own"), + pytest.param("{{ [page_size, 100] | max }}", True, id="passed_through_a_filter"), + pytest.param("{{ last_page_size < config['page_size'] }}", False, id="config_lookup"), + pytest.param("{{ response['page_size'] }}", False, id="response_lookup"), + pytest.param("{{ last_page_size == 0 }}", False, id="unrelated_expression"), + pytest.param("page_size", False, id="plain_text_naming_it"), + pytest.param("{{ last_page_size < page_size", False, id="does_not_parse"), + ], +) +def test_references_requested_page_size(template, expected): + """ + The distinction that matters is the bare `page_size` variable, which only exists when the strategy declares + a page size, against `config['page_size']`, which is bound whatever the strategy declares. + """ + assert references_requested_page_size(template) is expected diff --git a/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py b/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py index 765b879de3..afc767e5ea 100644 --- a/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +++ b/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py @@ -349,3 +349,26 @@ def test_composite_error_handler_always_uses_first_strategy(): assert len(composite_handler.backoff_strategies) == 2 assert isinstance(composite_handler.backoff_strategies[0], ConstantBackoffStrategy) assert composite_handler.backoff_strategies[1], ConstantBackoffStrategy + + +def test_given_reduce_page_size_when_interpret_response_then_stop_at_the_matching_handler(): + """ + Without the short circuit, a later handler resolving to FAIL would win over the page size reduction. + """ + reducing_handler = MagicMock() + reducing_handler.interpret_response.return_value = ErrorResolution( + response_action=ResponseAction.REDUCE_PAGE_SIZE, + failure_type=FailureType.transient_error, + ) + failing_handler = MagicMock() + failing_handler.interpret_response.return_value = ErrorResolution( + response_action=ResponseAction.FAIL, failure_type=FailureType.system_error + ) + error_handler = CompositeErrorHandler( + error_handlers=[reducing_handler, failing_handler], parameters={} + ) + + error_resolution = error_handler.interpret_response(MagicMock()) + + assert error_resolution.response_action == ResponseAction.REDUCE_PAGE_SIZE + assert failing_handler.interpret_response.call_count == 0 diff --git a/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py b/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py index da21e1074c..4b25086fa1 100644 --- a/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +++ b/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py @@ -152,3 +152,156 @@ def test_interpolated_page_size_raises_on_non_integer(): config={"page_size": "invalid"}, parameters={}, ) + + +def test_given_page_size_override_then_token_is_unchanged(): + strategy = CursorPaginationStrategy( + page_size=100, cursor_value="{{ response.next }}", config={}, parameters={} + ) + response = requests.Response() + response._content = json.dumps({"next": "a token"}).encode("utf-8") + + assert ( + strategy.next_page_token(response, 50, None, None, page_size_override=50) + == strategy.next_page_token(response, 100, None, None) + == "a token" + ) + + +def test_given_stop_condition_uses_page_size_and_page_is_full_at_the_reduced_size_then_keep_paginating(): + """ + Regression test for the silent truncation a page size reduction used to cause: a full page at the reduced + size satisfies `last_page_size < 100` and would end the pagination, dropping the rest of the partition. + `page_size` holds the size that was actually requested, so the same condition keeps paginating. + """ + strategy = CursorPaginationStrategy( + page_size=100, + cursor_value="{{ response.next }}", + stop_condition="{{ last_page_size < page_size }}", + config={}, + parameters={}, + ) + response = requests.Response() + response._content = json.dumps({"next": "a token"}).encode("utf-8") + + assert strategy.next_page_token(response, 50, None, None, page_size_override=50) == "a token" + assert strategy.next_page_token(response, 100, None, None) == "a token" + + +def test_given_stop_condition_uses_page_size_and_page_is_short_then_stop(): + strategy = CursorPaginationStrategy( + page_size=100, + cursor_value="{{ response.next }}", + stop_condition="{{ last_page_size < page_size }}", + config={}, + parameters={}, + ) + response = requests.Response() + response._content = json.dumps({"next": "a token"}).encode("utf-8") + + assert strategy.next_page_token(response, 49, None, None, page_size_override=50) is None + assert strategy.next_page_token(response, 99, None, None) is None + + +@pytest.mark.parametrize( + "page_size,page_size_override,expected_token", + [ + pytest.param(100, None, 100, id="test_configured_page_size_is_bound"), + pytest.param(100, 50, 50, id="test_reduced_page_size_is_bound"), + ], +) +def test_page_size_is_bound_in_the_cursor_value_interpolation_context( + page_size, page_size_override, expected_token +): + """ + `page_size` has to resolve to the size that was actually requested. The None case alone would also pass if + the variable were never bound at all, so the two positive cases are what pin it. + """ + strategy = CursorPaginationStrategy( + page_size=page_size, cursor_value="{{ page_size }}", config={}, parameters={} + ) + response = requests.Response() + response._content = json.dumps({}).encode("utf-8") + + assert ( + strategy.next_page_token(response, 10, None, None, page_size_override=page_size_override) + == expected_token + ) + + +@pytest.mark.parametrize( + "field_name, kwargs", + [ + pytest.param( + "stop_condition", + { + "cursor_value": "{{ response.next }}", + "stop_condition": "{{ last_page_size < page_size }}", + }, + id="test_stop_condition_reads_page_size", + ), + pytest.param( + "cursor_value", + {"cursor_value": "{{ page_size }}"}, + id="test_cursor_value_reads_page_size", + ), + pytest.param( + "stop_condition", + { + "cursor_value": "{{ response.next }}", + "stop_condition": InterpolatedBoolean( + condition="{{ last_page_size < page_size }}", parameters={} + ), + }, + id="test_stop_condition_given_as_a_component", + ), + ], +) +def test_given_no_page_size_when_an_expression_reads_page_size_then_raise(field_name, kwargs): + """ + `page_size` is unbound without a declared page size, and an unbound comparison does not fail loudly: Jinja + raises, the interpolation falls back to the raw template string, and a non-empty string is truthy. A + `stop_condition` written that way would stop after the first page, so it has to be rejected here. + """ + with pytest.raises(ValueError) as error: + CursorPaginationStrategy(config={}, parameters={}, **kwargs) + + assert f"`{field_name}`" in str(error.value) + assert "declares no `page_size`" in str(error.value) + + +def test_given_no_page_size_when_the_expression_reads_config_page_size_then_accept(): + """`config['page_size']` is a lookup on the config, not the reduction-aware variable, and is always bound.""" + strategy = CursorPaginationStrategy( + cursor_value="{{ response.next }}", + stop_condition="{{ last_page_size < config['page_size'] }}", + config={"page_size": 100}, + parameters={}, + ) + response = requests.Response() + response._content = json.dumps({"next": "a token"}).encode("utf-8") + + assert strategy.next_page_token(response, 100, None, None) == "a token" + + +@pytest.mark.parametrize( + "page_size", + [ + pytest.param(100, id="test_literal_page_size"), + pytest.param("{{ config['page_size'] }}", id="test_interpolated_page_size"), + ], +) +def test_given_a_page_size_when_an_expression_reads_page_size_then_accept(page_size): + strategy = CursorPaginationStrategy( + cursor_value="{{ response.next }}", + stop_condition="{{ last_page_size < page_size }}", + page_size=page_size, + config={"page_size": 100}, + parameters={}, + ) + response = requests.Response() + response._content = json.dumps({"next": "a token"}).encode("utf-8") + + assert strategy.next_page_token(response, 100, None, None) == "a token" + assert strategy.next_page_token(response, 50, None, None, page_size_override=50) == "a token" + assert strategy.next_page_token(response, 49, None, None, page_size_override=50) is None diff --git a/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py b/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py index ee9f12b123..5254e69d3d 100644 --- a/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +++ b/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py @@ -539,3 +539,138 @@ def test_path_returns_none_when_option_not_request_path() -> None: ) result = paginator.path(next_page_token) assert result is None + + +def _paginator_with_page_size(page_size=100, inject_into=RequestOptionType.request_parameter): + return DefaultPaginator( + page_size_option=RequestOption( + inject_into=inject_into, field_name="page_size", parameters={} + ), + page_token_option=RequestOption( + inject_into=RequestOptionType.request_parameter, field_name="after", parameters={} + ), + pagination_strategy=CursorPaginationStrategy( + page_size=page_size, cursor_value="{{ response.next }}", config={}, parameters={} + ), + config={}, + url_base="https://airbyte.io", + parameters={}, + ) + + +def test_get_page_size_returns_the_strategy_page_size(): + assert _paginator_with_page_size().get_page_size() == 100 + + +def test_given_page_size_override_then_injected_instead_of_the_configured_page_size(): + paginator = _paginator_with_page_size() + + assert paginator.get_request_params(page_size_override=25) == {"page_size": 25} + assert paginator.get_request_params() == {"page_size": 100} + + +def test_given_page_size_override_when_injected_into_body_json_then_use_override(): + paginator = _paginator_with_page_size(inject_into=RequestOptionType.body_json) + + assert paginator.get_request_body_json(page_size_override=25) == {"page_size": 25} + assert paginator.get_request_headers(page_size_override=25) == {} + + +def test_given_page_size_override_when_next_page_token_then_forward_to_strategy(): + strategy = Mock() + strategy.next_page_token.return_value = "a token" + paginator = DefaultPaginator( + pagination_strategy=strategy, config={}, url_base="https://airbyte.io", parameters={} + ) + response = requests.Response() + + paginator.next_page_token(response, 25, None, None, page_size_override=25) + + assert strategy.next_page_token.call_args.kwargs["page_size_override"] == 25 + + +def test_given_no_page_size_override_when_next_page_token_then_strategy_called_without_the_argument(): + strategy = Mock() + strategy.next_page_token.return_value = "a token" + paginator = DefaultPaginator( + pagination_strategy=strategy, config={}, url_base="https://airbyte.io", parameters={} + ) + response = requests.Response() + + paginator.next_page_token(response, 25, None, None) + + assert "page_size_override" not in strategy.next_page_token.call_args.kwargs + + +def test_test_read_decorator_delegates_page_size_override(): + decorated = _paginator_with_page_size() + paginator = PaginatorTestReadDecorator(decorated, 5) + + assert paginator.get_page_size() == 100 + assert paginator.get_request_params(page_size_override=25) == {"page_size": 25} + assert paginator.get_request_params() == {"page_size": 100} + + +_TEST_READ_DECORATOR_REQUEST_OPTION_METHODS = [ + "get_request_params", + "get_request_headers", + "get_request_body_data", + "get_request_body_json", +] + + +@pytest.mark.parametrize("method_name", _TEST_READ_DECORATOR_REQUEST_OPTION_METHODS) +def test_given_page_size_override_when_test_read_decorator_request_options_then_forward_it( + method_name, +): + """ + This is the Connector Builder path: a method that dropped the override would make a Builder test read of a + reduced stream request the configured page size again, or stop after the first reduced page. + """ + decorated = Mock() + paginator = PaginatorTestReadDecorator(decorated, 5) + + getattr(paginator, method_name)(page_size_override=25) + + assert getattr(decorated, method_name).call_args.kwargs["page_size_override"] == 25 + + +@pytest.mark.parametrize("method_name", _TEST_READ_DECORATOR_REQUEST_OPTION_METHODS) +def test_given_no_page_size_override_when_test_read_decorator_request_options_then_omit_it( + method_name, +): + decorated = Mock() + paginator = PaginatorTestReadDecorator(decorated, 5) + + getattr(paginator, method_name)() + + assert "page_size_override" not in getattr(decorated, method_name).call_args.kwargs + + +def test_given_page_size_override_when_test_read_decorator_next_page_token_then_forward_it(): + decorated = Mock() + decorated.next_page_token.return_value = "a token" + paginator = PaginatorTestReadDecorator(decorated, 5) + response = requests.Response() + + assert paginator.next_page_token(response, 25, None, None, page_size_override=25) == "a token" + assert decorated.next_page_token.call_args.kwargs["page_size_override"] == 25 + + +def test_given_no_page_size_override_when_test_read_decorator_next_page_token_then_omit_it(): + decorated = Mock() + decorated.next_page_token.return_value = "a token" + paginator = PaginatorTestReadDecorator(decorated, 5) + response = requests.Response() + + paginator.next_page_token(response, 25, None, None) + + assert "page_size_override" not in decorated.next_page_token.call_args.kwargs + + +def test_test_read_decorator_delegates_get_page_size(): + decorated = Mock() + decorated.get_page_size.return_value = 100 + paginator = PaginatorTestReadDecorator(decorated, 5) + + assert paginator.get_page_size() == 100 diff --git a/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py b/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py index 28f6717f54..a12bd1e5a1 100644 --- a/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +++ b/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py @@ -146,3 +146,98 @@ def test_offset_increment_paginator_strategy_initial_token( ) assert paginator_strategy.initial_token == expected_initial_token + + +def _response(records): + response = requests.Response() + response._content = json.dumps({"results": records}).encode("utf-8") + return response + + +def test_given_page_size_override_when_page_is_full_for_the_override_then_keep_paginating(): + """ + A page that is full for the reduced page size is not the last page, even though it is smaller than the + configured page size. + """ + strategy = OffsetIncrement(page_size=100, extractor=None, config={}, parameters={}) + + next_page_token = strategy.next_page_token( + response=_response([{"id": index} for index in range(50)]), + last_page_size=50, + last_record=None, + last_page_token_value=0, + page_size_override=50, + ) + + assert next_page_token == 50 + + +def test_given_page_size_override_when_page_is_not_full_for_the_override_then_stop_paginating(): + strategy = OffsetIncrement(page_size=100, extractor=None, config={}, parameters={}) + + next_page_token = strategy.next_page_token( + response=_response([{"id": index} for index in range(30)]), + last_page_size=30, + last_record=None, + last_page_token_value=0, + page_size_override=50, + ) + + assert next_page_token is None + + +def test_given_page_size_override_then_offset_follows_the_records_actually_returned(): + strategy = OffsetIncrement(page_size=100, extractor=None, config={}, parameters={}) + + assert ( + strategy.next_page_token( + response=_response([{"id": index} for index in range(100)]), + last_page_size=100, + last_record=None, + last_page_token_value=0, + ) + == 100 + ) + assert ( + strategy.next_page_token( + response=_response([{"id": index} for index in range(50)]), + last_page_size=50, + last_record=None, + last_page_token_value=100, + page_size_override=50, + ) + == 150 + ) + + +def test_given_page_size_interpolates_to_empty_string_then_paginate_until_an_empty_page(): + """ + `page_size` interpolating to an empty string is the one behaviour change on the no-override path: the stop + condition used to compare the rendered value with `<`, which raised a TypeError for a string. It is now + treated as "no page size known", so pagination runs until a page comes back empty. + + This is only reachable when the paginator has no `page_size_option`, since `get_page_size` raises for a + non-integer page size before the request is built. + """ + strategy = OffsetIncrement( + page_size="{{ config['page_size'] }}", extractor=None, config={}, parameters={} + ) + + assert ( + strategy.next_page_token( + response=_response([{"id": index} for index in range(30)]), + last_page_size=30, + last_record=None, + last_page_token_value=0, + ) + == 30 + ) + assert ( + strategy.next_page_token( + response=_response([]), + last_page_size=0, + last_record=None, + last_page_token_value=30, + ) + is None + ) diff --git a/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py b/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py index ecb458b951..ef8339fe55 100644 --- a/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +++ b/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py @@ -8,10 +8,12 @@ import pytest import requests +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.extractors import DpathExtractor from airbyte_cdk.sources.declarative.requesters.paginators.strategies.page_increment import ( PageIncrement, ) +from airbyte_cdk.utils.traced_exception import AirbyteTracedException @pytest.mark.parametrize( @@ -160,3 +162,24 @@ def test_page_increment_paginator_strategy_initial_token( ) assert paginator_strategy.initial_token == expected_initial_token + + +def test_given_page_size_override_then_raise_config_error(): + """ + Reducing the page size would move every following page boundary, so PageIncrement refuses it. This is only + reachable when the factory is bypassed, so it has to report itself as a configuration error rather than + surfacing as a generic system error. + """ + strategy = PageIncrement(page_size=100, config={}, parameters={}, start_from_page=1) + response = requests.Response() + + with pytest.raises(AirbyteTracedException) as exception: + strategy.next_page_token( + response=response, + last_page_size=50, + last_record=None, + last_page_token_value=1, + page_size_override=50, + ) + + assert exception.value.failure_type == FailureType.config_error diff --git a/unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py b/unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py index b89baf4430..cb2ccc5987 100644 --- a/unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py +++ b/unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py @@ -116,3 +116,18 @@ def test_when_get_page_size_then_delegate(mocked_pagination_strategy, mocked_sto assert page_size == mocked_pagination_strategy.get_page_size.return_value mocked_pagination_strategy.get_page_size.assert_called_once_with() + + +def test_given_page_size_override_when_next_page_token_then_forward_to_delegate( + mocked_pagination_strategy, mocked_stop_condition +): + mocked_stop_condition.is_met.return_value = False + decorator = StopConditionPaginationStrategyDecorator( + mocked_pagination_strategy, mocked_stop_condition + ) + + decorator.next_page_token(ANY_RESPONSE, 25, NO_RECORD, None, page_size_override=25) + + mocked_pagination_strategy.next_page_token.assert_called_once_with( + ANY_RESPONSE, 25, NO_RECORD, None, page_size_override=25 + ) diff --git a/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py b/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py new file mode 100644 index 0000000000..012ece3b9b --- /dev/null +++ b/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py @@ -0,0 +1,500 @@ +# +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. +# + +import pytest + +from airbyte_cdk.models import FailureType +from airbyte_cdk.sources.declarative.retrievers.page_size_reducer import ( + PageSizeReducer, + PageSizeReduction, + PageSizeResetPolicy, +) +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + +A_STREAM_NAME = "stream_name" + + +def _reducer(configured_page_size=100, sleeps=None, **kwargs): + return PageSizeReducer( + PageSizeReduction(**kwargs), + configured_page_size, + stream_name=A_STREAM_NAME, + # The reducer waits before each retry; tests record the waits instead of taking them. + sleep=sleeps.append if sleeps is not None else lambda _: None, + ) + + +def test_given_no_reduction_when_page_size_override_then_return_none(): + assert _reducer().page_size_override is None + + +def test_when_reduce_then_halve_page_size(): + reducer = _reducer() + + reducer.reduce() + assert reducer.page_size_override == 50 + + reducer.reduce() + assert reducer.page_size_override == 25 + + reducer.reduce() + assert reducer.page_size_override == 12 + + +def test_given_reduction_factor_when_reduce_then_use_factor(): + reducer = _reducer(reduction_factor=4) + + reducer.reduce() + + assert reducer.page_size_override == 25 + + +def test_given_minimum_page_size_when_reduce_then_clamp_to_minimum(): + reducer = _reducer(configured_page_size=30, minimum_page_size=20) + + reducer.reduce() + + assert reducer.page_size_override == 20 + + +def test_given_minimum_page_size_above_configured_page_size_when_reduce_then_raise_config_error(): + """ + No reduction can ever be applied here, so nothing about the response can fix it and the platform must not + retry the whole job for it. + """ + reducer = _reducer(configured_page_size=50, minimum_page_size=100) + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.config_error + assert "already at or below the configured minimum" in exception.value.message + + +def test_given_page_size_of_one_when_reduce_then_raise_transient_error(): + """ + Unlike a `minimum_page_size` above the configured page size, there is nothing to configure differently + here: one record per page is as small as a page gets. The API rejecting it is about the API, not about + the connector's setup, so the platform gets a transient error rather than a config error. + """ + reducer = _reducer(configured_page_size=1) + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.transient_error + + +def test_given_already_at_minimum_when_reduce_then_raise_transient_error(): + reducer = _reducer(configured_page_size=4, minimum_page_size=2) + reducer.reduce() + assert reducer.page_size_override == 2 + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.transient_error + + +def test_given_more_reductions_than_max_attempts_when_reduce_then_raise_transient_error(): + reducer = _reducer(configured_page_size=1000, max_attempts=2) + reducer.reduce() + reducer.reduce() + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.transient_error + assert reducer.page_size_override == 250 + + +def test_given_paginator_has_no_page_size_when_reduce_then_raise_config_error(): + reducer = _reducer(configured_page_size=None) + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.config_error + + +def test_given_reset_policy_never_when_on_successful_page_then_keep_reduced_page_size(): + reducer = _reducer() + reducer.reduce() + + reducer.on_successful_page() + + assert reducer.page_size_override == 50 + + +def test_given_reset_policy_after_successful_page_when_on_successful_page_then_restore_page_size(): + reducer = _reducer(reset_policy=PageSizeResetPolicy.AFTER_SUCCESSFUL_PAGE) + reducer.reduce() + + reducer.on_successful_page() + + assert reducer.page_size_override is None + + +def test_given_reset_policy_after_successful_page_when_on_successful_page_then_attempts_are_reset(): + """ + This policy exists for an API that rejects the configured page size on every page, so every page costs one + reduction. A budget spanning the whole partition would fail the sync at page `max_attempts + 1` however + healthy the reads are, which is the one workload the policy is for. + """ + reducer = _reducer(max_attempts=2, reset_policy=PageSizeResetPolicy.AFTER_SUCCESSFUL_PAGE) + + for _ in range(10): + reducer.reduce() + assert reducer.page_size_override == 50 + reducer.on_successful_page() + assert reducer.page_size_override is None + + +def test_given_reset_policy_after_successful_page_when_no_page_succeeds_then_max_attempts_still_applies(): + """The budget restarts on a successful page, not on a reduction, so an endpoint that fails whatever we ask + for still terminates. This is the genuinely-stuck case: nothing got through, so the read has to end.""" + reducer = _reducer(max_attempts=2, reset_policy=PageSizeResetPolicy.AFTER_SUCCESSFUL_PAGE) + reducer.reduce() + reducer.reduce() + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.transient_error + # the size named is the one that was just requested and failed, not the configured one + assert "down to 25 records per page" in exception.value.message + assert "2 times in a row without a single page succeeding" in exception.value.internal_message + + +def test_given_reset_policy_never_when_pages_succeed_then_attempts_are_reset(): + """ + `max_attempts` counts the reductions made in a row without a page getting through, which is what the + terminal error claims happened, so a page that succeeded has to restart it under this policy too. A stream + whose per-page cost varies - the GraphQL case this feature exists for - would otherwise fail at the + `max_attempts + 1`-th heavy page of a long partition in which every reduction was followed by a page. + """ + reducer = _reducer(configured_page_size=1000, max_attempts=2) + + for expected_page_size in [500, 250, 125, 62, 31, 15, 7, 3, 1]: + reducer.reduce() + assert reducer.page_size_override == expected_page_size + # the page size is not restored under NEVER, only the budget is + reducer.on_successful_page() + assert reducer.page_size_override == expected_page_size + + +def test_given_reset_policy_never_when_pages_succeed_then_minimum_page_size_still_ends_the_read(): + """ + With the budget restarting, `minimum_page_size` is what bounds a NEVER partition: the page size strictly + decreases, so the read cannot go on forever. + """ + reducer = _reducer(configured_page_size=100, minimum_page_size=10, max_attempts=2) + + for expected_page_size in [50, 25, 12, 10]: + reducer.reduce() + assert reducer.page_size_override == expected_page_size + reducer.on_successful_page() + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.transient_error + assert "smallest page size" in exception.value.message + + +def test_given_no_page_succeeds_then_attempts_are_not_reset(): + reducer = _reducer(max_attempts=2) + reducer.reduce() + reducer.reduce() + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert "2 times in a row without a single page succeeding" in exception.value.internal_message + + +def test_given_reset_policy_after_successful_page_when_every_page_succeeds_then_never_fail(): + """ + A partition where every page gets through after one reduction is healthy, however long it is: this policy + exists for an API that rejects the configured page size on every page. There is no partition-wide cap on + the number of reductions, because any such cap would fail this stream at the page it happens to sit on. + """ + reducer = _reducer( + configured_page_size=1000, + max_attempts=2, + reset_policy=PageSizeResetPolicy.AFTER_SUCCESSFUL_PAGE, + ) + + for _ in range(5_000): + reducer.reduce() + assert reducer.page_size_override == 500 + reducer.on_successful_page() + assert reducer.page_size_override is None + + +@pytest.mark.parametrize( + "kwargs", + [ + pytest.param({"reduction_factor": 1}, id="reduction_factor_does_not_reduce"), + pytest.param({"minimum_page_size": 0}, id="minimum_page_size_is_not_positive"), + pytest.param({"max_attempts": 0}, id="max_attempts_is_not_positive"), + pytest.param({"backoff_seconds": -1}, id="backoff_seconds_is_negative"), + pytest.param( + {"retries_at_minimum_page_size": -1}, id="retries_at_minimum_page_size_is_negative" + ), + ], +) +def test_given_invalid_configuration_then_raise_value_error(kwargs): + with pytest.raises(ValueError): + PageSizeReduction(**kwargs) + + +def test_given_non_integer_page_size_when_reduce_then_raise_config_error(): + """A custom pagination strategy can return anything from `get_page_size`; reducing is + arithmetic, so a non-integer has to be reported rather than raising a bare TypeError.""" + reducer = PageSizeReducer( + PageSizeReduction(), + "{{ config['page_size'] }}", # type: ignore[arg-type] + stream_name="a_stream", + ) + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.config_error + assert "not a whole number" in exception.value.message + + +def test_when_reduce_then_wait_before_the_retry(): + """ + `PageSizeReductionRequiredException` bypasses the HTTP retry budget on purpose, so this wait is the only + thing keeping an endpoint that fails at every page size from being hit in a burst. + """ + sleeps: list = [] + reducer = _reducer(configured_page_size=1000, sleeps=sleeps) + + reducer.reduce() + reducer.reduce() + + assert sleeps == [ + PageSizeReduction().backoff_seconds, + PageSizeReduction().backoff_seconds * 2, + ] + assert all(wait > 0 for wait in sleeps) + + +def test_given_backoff_seconds_when_reduce_then_wait_that_long(): + sleeps: list = [] + reducer = _reducer(configured_page_size=1000, sleeps=sleeps, backoff_seconds=10) + + reducer.reduce() + reducer.reduce() + + assert sleeps == [10, 20] + + +def test_given_retries_at_minimum_page_size_when_at_the_floor_then_retry_the_same_page(): + """ + At the floor the page size cannot answer the error any more, but an API that returns the same status for + "your page is too big" and for a passing hiccup still can. Without this budget the first such response + ends the stream, which is fewer attempts than the same error handler gave before the reduction existed. + """ + sleeps: list = [] + reducer = _reducer( + configured_page_size=2, + sleeps=sleeps, + minimum_page_size=1, + backoff_seconds=1, + retries_at_minimum_page_size=2, + ) + + reducer.reduce() + assert reducer.page_size_override == 1 + + reducer.reduce() + reducer.reduce() + assert reducer.page_size_override == 1, "the page size is already at the minimum" + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.transient_error + assert sleeps == [1, 1, 2], "the reduction, then the two retries at the floor" + + +def test_given_page_size_configured_at_a_floor_above_one_then_retry_before_reporting_the_misconfiguration(): + """ + The retry budget cannot depend on how the page size arrived at the floor: the same manifest would then give + a partition three retries when the reduction walked down to the floor and none when the user's `page_size` + was already there. The misconfiguration is still reported, once the retries the manifest asked for are + spent. + """ + sleeps: list = [] + reducer = _reducer( + configured_page_size=10, + sleeps=sleeps, + minimum_page_size=10, + backoff_seconds=1, + retries_at_minimum_page_size=2, + ) + + reducer.reduce() + reducer.reduce() + assert reducer.page_size_override is None, "there was never a reduction to apply" + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert sleeps == [1, 2] + assert exception.value.failure_type == FailureType.config_error + assert "already at or below the configured minimum" in exception.value.message + assert "2 retries at that size were spent first" in exception.value.internal_message + + +def test_given_page_size_configured_at_a_floor_above_one_and_no_retries_then_report_the_misconfiguration(): + reducer = _reducer(configured_page_size=10, minimum_page_size=10) + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.config_error + + +def test_given_no_sleep_override_when_reduce_then_wait_through_time_sleep(monkeypatch): + """ + The reducer resolves `time.sleep` per call rather than binding it as a default argument. Bound, a suite + that patches `time.sleep` to keep a run of reductions from taking its minutes for real has no effect - and + every other test here passes its own `sleep`, so this is the only one that exercises the default path. + """ + waits: list = [] + monkeypatch.setattr( + "airbyte_cdk.sources.declarative.retrievers.page_size_reducer.time.sleep", waits.append + ) + reducer = PageSizeReducer(PageSizeReduction(backoff_seconds=7), 100, stream_name=A_STREAM_NAME) + + reducer.reduce() + + assert waits == [7] + + +def test_given_retries_at_minimum_page_size_do_not_count_against_max_attempts(): + """ + The two budgets measure different things: `max_attempts` bounds the reductions that got no page through, + while this one bounds the waiting done once there is nothing left to reduce. + """ + reducer = _reducer(configured_page_size=2, max_attempts=1, retries_at_minimum_page_size=3) + + reducer.reduce() + reducer.reduce() + reducer.reduce() + reducer.reduce() + + with pytest.raises(AirbyteTracedException): + reducer.reduce() + + +def test_given_successful_page_then_restart_the_retries_at_minimum_page_size(): + reducer = _reducer(configured_page_size=1, retries_at_minimum_page_size=1) + reducer.reduce() + + reducer.on_successful_page() + + reducer.reduce() # would raise if the budget had not restarted + + with pytest.raises(AirbyteTracedException): + reducer.reduce() + + +@pytest.mark.parametrize( + "reducer_kwargs,reductions", + [ + pytest.param({"configured_page_size": 4, "minimum_page_size": 2}, 1, id="minimum_reached"), + pytest.param( + {"configured_page_size": 1000, "max_attempts": 2}, 2, id="max_attempts_exhausted" + ), + ], +) +def test_given_reduction_fails_then_message_names_the_stream_and_leaves_out_remediation( + reducer_kwargs, reductions +): + # The user cannot act on a `transient_error`, so the message states the failure alone. Naming the stream + # is what makes it actionable for whoever reads the sync, since a sync reduces per stream. + reducer = _reducer(**reducer_kwargs) + for _ in range(reductions): + reducer.reduce() + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.transient_error + assert A_STREAM_NAME in exception.value.message + assert "contact the API provider" not in exception.value.message + assert "Try syncing fewer streams" not in exception.value.message + + +@pytest.mark.parametrize( + "reducer_kwargs", + [ + pytest.param({"configured_page_size": 5, "minimum_page_size": 10}, id="never_reducible"), + pytest.param({"configured_page_size": None}, id="no_page_size_at_all"), + pytest.param({"configured_page_size": "100"}, id="page_size_is_not_a_number"), + ], +) +def test_given_misconfiguration_then_message_names_the_stream_and_keeps_remediation(reducer_kwargs): + # A `config_error` is the user's to fix, so the remediation stays in the message. + reducer = _reducer(**reducer_kwargs) + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.config_error + assert A_STREAM_NAME in exception.value.message + assert exception.value.message.rstrip().endswith(".") + + +@pytest.mark.parametrize( + "reducer_kwargs,reductions", + [ + pytest.param({"configured_page_size": 4, "minimum_page_size": 2}, 1, id="minimum_reached"), + pytest.param( + {"configured_page_size": 1000, "max_attempts": 2}, 2, id="max_attempts_exhausted" + ), + ], +) +def test_given_failure_message_when_reduction_fails_then_append_it(reducer_kwargs, reductions): + # The CDK only knows that the API rejected every page size it asked for; what narrows a query down is + # API-specific, so the connector supplies that sentence. + reducer = _reducer(failure_message="Select fewer fields on this stream.", **reducer_kwargs) + for _ in range(reductions): + reducer.reduce() + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.transient_error + assert exception.value.message.endswith(" Select fewer fields on this stream.") + + +def test_given_no_failure_message_when_reduction_fails_then_message_ends_with_the_cdk_sentence(): + reducer = _reducer(configured_page_size=4, minimum_page_size=2) + reducer.reduce() + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.message.endswith("records per page).") + + +def test_given_failure_message_when_misconfigured_then_do_not_append_it(): + # A `config_error` is about the manifest, not about the API rejecting a page size, so the connector's + # sentence about narrowing the query down would be misleading there. + reducer = _reducer( + configured_page_size=None, failure_message="Select fewer fields on this stream." + ) + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.config_error + assert "Select fewer fields" not in exception.value.message diff --git a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index e05c427acb..1afa3b5fc9 100644 --- a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -3,6 +3,9 @@ # import json +import threading +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor from functools import partial from typing import Any, Iterable, Mapping, Optional from unittest.mock import MagicMock, Mock, patch @@ -13,6 +16,7 @@ from airbyte_cdk.models import ( AirbyteLogMessage, AirbyteMessage, + FailureType, Level, SyncMode, Type, @@ -37,15 +41,27 @@ GroupByKey, PropertyLimitType, ) -from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType +from airbyte_cdk.sources.declarative.requesters.request_option import ( + RequestOption, + RequestOptionType, +) from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod, Requester +from airbyte_cdk.sources.declarative.retrievers.page_size_reducer import ( + PageSizeReducer, + PageSizeReduction, + PageSizeResetPolicy, +) from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever +from airbyte_cdk.sources.streams.http.page_size_reduction_exception import ( + PageSizeReductionRequiredException, +) from airbyte_cdk.sources.streams.http.pagination_reset_exception import ( PaginationResetRequiredException, ) from airbyte_cdk.sources.types import Record, StreamSlice from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from airbyte_cdk.utils.traced_exception import AirbyteTracedException A_RECORD_SCHEMA = {} A_SLICE_STATE = {"slice_state": "slice state value"} @@ -1426,6 +1442,475 @@ def test_given_reach_pagination_limit_after_two_pages_when_read_records_than_red } +@pytest.fixture(autouse=True) +def _no_page_size_reduction_backoff(monkeypatch): + """The reducer waits between reduction retries; taking those waits for real adds seconds to every CI run.""" + monkeypatch.setattr( + "airbyte_cdk.sources.declarative.retrievers.page_size_reducer.time.sleep", lambda _: None + ) + + +def _page_size_reduction_retriever( + requester: Requester, + paginator: Paginator, + record_selector: HttpSelector, + page_size_reduction: PageSizeReduction, +) -> SimpleRetriever: + return SimpleRetriever( + name=A_STREAM_NAME, + primary_key=primary_key, + requester=requester, + record_selector=record_selector, + paginator=paginator, + page_size_reduction=page_size_reduction, + parameters={}, + config={}, + ) + + +def test_given_page_size_reduction_when_read_records_then_retry_same_page_with_reduced_page_size(): + requester = Mock(spec=Requester) + requester.send_request.side_effect = [ + PageSizeReductionRequiredException(), + [{"id": 1}], + ] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.return_value = [{"id": 1}] + paginator = _mock_paginator() + paginator.get_page_size.return_value = 100 + paginator.get_initial_token.return_value = "a token" + paginator.next_page_token.return_value = None + + retriever = _page_size_reduction_retriever( + requester, paginator, record_selector, PageSizeReduction() + ) + + records = list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + assert records == [{"id": 1}] + assert requester.send_request.call_count == 2 + # the same page is requested again: only the page size changes + assert [call.kwargs["next_page_token"] for call in requester.send_request.call_args_list] == [ + {"next_page_token": "a token"}, + {"next_page_token": "a token"}, + ] + assert [call.kwargs["stream_slice"] for call in requester.send_request.call_args_list] == [ + A_STREAM_SLICE, + A_STREAM_SLICE, + ] + assert [ + call.kwargs.get("page_size_override") + for call in paginator.get_request_params.call_args_list + ] == [None, 50] + + +def test_given_retries_at_minimum_page_size_when_at_the_floor_then_re_issue_the_same_page(): + """The page size stays at the floor: what the retry buys is the wait, not a smaller request.""" + requester = Mock(spec=Requester) + requester.send_request.side_effect = [ + PageSizeReductionRequiredException(), + [{"id": 1}], + ] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.return_value = [{"id": 1}] + paginator = _mock_paginator() + paginator.get_page_size.return_value = 1 + paginator.get_initial_token.return_value = None + paginator.next_page_token.return_value = None + + retriever = _page_size_reduction_retriever( + requester, + paginator, + record_selector, + PageSizeReduction(retries_at_minimum_page_size=1), + ) + + records = list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + assert records == [{"id": 1}] + assert requester.send_request.call_count == 2, ( + "the page that could not be reduced was issued twice" + ) + assert [ + call.kwargs.get("page_size_override") + for call in paginator.get_request_params.call_args_list + ] == [None, None] + + +def test_given_retries_at_minimum_page_size_are_spent_then_raise_transient_error(): + requester = Mock(spec=Requester) + requester.send_request.side_effect = PageSizeReductionRequiredException() + record_selector = Mock(spec=HttpSelector) + paginator = _mock_paginator() + paginator.get_page_size.return_value = 1 + paginator.get_initial_token.return_value = None + + retriever = _page_size_reduction_retriever( + requester, + paginator, + record_selector, + PageSizeReduction(retries_at_minimum_page_size=1), + ) + + with pytest.raises(AirbyteTracedException) as exception: + list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + assert exception.value.failure_type == FailureType.transient_error + assert requester.send_request.call_count == 2 + + +def test_given_page_size_reduction_when_read_records_then_next_page_token_not_computed_for_failed_page(): + requester = Mock(spec=Requester) + requester.send_request.side_effect = [ + PageSizeReductionRequiredException(), + [{"id": 1}], + ] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.return_value = [{"id": 1}] + paginator = _mock_paginator() + paginator.get_page_size.return_value = 100 + paginator.get_initial_token.return_value = None + paginator.next_page_token.return_value = None + + retriever = _page_size_reduction_retriever( + requester, paginator, record_selector, PageSizeReduction() + ) + + list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + assert paginator.next_page_token.call_count == 1 + assert paginator.next_page_token.call_args.kwargs["page_size_override"] == 50 + + +def test_given_reset_policy_never_when_page_succeeds_then_following_pages_stay_reduced(): + requester = Mock(spec=Requester) + requester.send_request.side_effect = [ + PageSizeReductionRequiredException(), + [{"id": 1}], + [{"id": 2}], + ] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.side_effect = [[{"id": 1}], [{"id": 2}]] + paginator = _mock_paginator() + paginator.get_page_size.return_value = 100 + paginator.get_initial_token.return_value = None + paginator.next_page_token.side_effect = [{"next_page_token": 2}, None] + + retriever = _page_size_reduction_retriever( + requester, paginator, record_selector, PageSizeReduction() + ) + + list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + assert [ + call.kwargs.get("page_size_override") + for call in paginator.get_request_params.call_args_list + ] == [None, 50, 50] + + +def test_given_reset_policy_after_successful_page_when_page_succeeds_then_page_size_restored(): + requester = Mock(spec=Requester) + requester.send_request.side_effect = [ + PageSizeReductionRequiredException(), + [{"id": 1}], + [{"id": 2}], + ] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.side_effect = [[{"id": 1}], [{"id": 2}]] + paginator = _mock_paginator() + paginator.get_page_size.return_value = 100 + paginator.get_initial_token.return_value = None + paginator.next_page_token.side_effect = [{"next_page_token": 2}, None] + + retriever = _page_size_reduction_retriever( + requester, + paginator, + record_selector, + PageSizeReduction(reset_policy=PageSizeResetPolicy.AFTER_SUCCESSFUL_PAGE), + ) + + list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + # the reduced page size applies to the retry only, the following page is back to the configured one + assert [ + call.kwargs.get("page_size_override") + for call in paginator.get_request_params.call_args_list + ] == [None, 50, None] + + +def test_given_reductions_exhausted_when_read_records_then_raise_transient_error(): + requester = Mock(spec=Requester) + requester.send_request.side_effect = PageSizeReductionRequiredException() + record_selector = Mock(spec=HttpSelector) + paginator = _mock_paginator() + paginator.get_page_size.return_value = 100 + paginator.get_initial_token.return_value = None + + retriever = _page_size_reduction_retriever( + requester, paginator, record_selector, PageSizeReduction(max_attempts=2) + ) + + with pytest.raises(AirbyteTracedException) as exception: + list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + assert exception.value.failure_type == FailureType.transient_error + assert requester.send_request.call_count == 3 + + +def test_given_no_page_size_reduction_when_reduce_page_size_required_then_raise_config_error(): + requester = Mock(spec=Requester) + requester.send_request.side_effect = PageSizeReductionRequiredException() + record_selector = Mock(spec=HttpSelector) + paginator = _mock_paginator() + paginator.get_initial_token.return_value = None + + retriever = SimpleRetriever( + name=A_STREAM_NAME, + primary_key=primary_key, + requester=requester, + record_selector=record_selector, + paginator=paginator, + parameters={}, + config={}, + ) + + with pytest.raises(AirbyteTracedException) as exception: + list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + assert exception.value.failure_type == FailureType.config_error + # the neutral "the API asked for a smaller page" message is replaced by the one describing the + # misconfiguration, which is what this branch actually means + assert "not set up to send a smaller page" in exception.value.message + + +def test_given_records_already_emitted_when_reduce_page_size_required_then_raise_instead_of_retrying(): + """ + Re-issuing a page is only safe while none of its records have been emitted. Every in-CDK path raises from + the fetch, but a custom extractor, filter or transformation can issue its own request from inside the + record generator, and retrying then would emit those records twice. + """ + requester = Mock(spec=Requester) + requester.send_request.return_value = [{"id": 1}] + paginator = _mock_paginator() + paginator.get_page_size.return_value = 100 + paginator.get_initial_token.return_value = None + + def select_records(**kwargs): + yield Record(data={"id": 1}, stream_name=A_STREAM_NAME) + raise PageSizeReductionRequiredException() + + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.side_effect = select_records + + retriever = _page_size_reduction_retriever( + requester, paginator, record_selector, PageSizeReduction() + ) + + with pytest.raises(AirbyteTracedException) as exception: + list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + assert exception.value.failure_type == FailureType.config_error + assert "middle of a page" in exception.value.message + assert requester.send_request.call_count == 1 + + +def test_given_page_size_reduction_and_pagination_limit_reached_when_read_records_then_reduce_before_resetting(): + """ + The reduction retry runs before the pagination limit check, so a page that failed defers the reset by one + iteration. That is correct - the failed page observed no record, so the limit it reports is stale - but the + two features never met in a test. + """ + requester = Mock(spec=Requester) + requester.send_request.side_effect = [ + PageSizeReductionRequiredException(), + [{"id": 1}], + [{"id": 2}], + ] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.side_effect = [[{"id": 1}], [{"id": 2}]] + pagination_tracker = Mock(spec=PaginationTracker) + pagination_tracker.has_reached_limit.side_effect = [True, False] + paginator = _mock_paginator() + paginator.get_page_size.return_value = 100 + paginator.get_initial_token.return_value = 1 + paginator.next_page_token.return_value = None + + retriever = SimpleRetriever( + name=A_STREAM_NAME, + primary_key=primary_key, + requester=requester, + record_selector=record_selector, + paginator=paginator, + pagination_tracker_factory=lambda: pagination_tracker, + page_size_reduction=PageSizeReduction(), + parameters={}, + config={}, + ) + + records = list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + assert len(records) == 2 + # the failed page is retried on the same slice with a smaller page, and only the page after it resets + assert pagination_tracker.has_reached_limit.call_count == 2 + assert pagination_tracker.reduce_slice_range_if_possible.call_count == 1 + assert [ + call.kwargs.get("page_size_override") + for call in paginator.get_request_params.call_args_list + ] == [None, 50, 50] + assert requester.send_request.call_args_list[1].kwargs["stream_slice"] == A_STREAM_SLICE + assert ( + requester.send_request.call_args_list[2].kwargs["stream_slice"] + == pagination_tracker.reduce_slice_range_if_possible.return_value + ) + + +def test_given_partitions_read_concurrently_when_one_reduces_then_others_keep_configured_page_size(): + """ + One retriever instance is shared by every partition of a stream, so the page size in effect must not leak + from one partition to another. + + The handshake has to make partition b read the page size in effect *after* partition a has reduced, which + means blocking b's first response until a is reduced and giving b a second page: the page size is read at + the top of the page loop, before the request is built, so a barrier inside the request building would come + too late and the assertion would hold even with a single shared reducer. + """ + slice_a = StreamSlice(cursor_slice={}, partition={"id": "a"}) + slice_b = StreamSlice(cursor_slice={}, partition={"id": "b"}) + partition_a_reduced = threading.Event() + requested_page_sizes = defaultdict(list) + + paginator = _mock_paginator() + paginator.get_page_size.return_value = 100 + paginator.get_initial_token.return_value = None + + def get_request_params(*, stream_slice, next_page_token, page_size_override=None): + requested_page_sizes[stream_slice.partition["id"]].append(page_size_override) + return {} + + paginator.get_request_params.side_effect = get_request_params + + def next_page_token(*, response, last_page_token_value, **kwargs): + # only partition b has a second page, which it requests once partition a is known to be reduced + if response[0]["id"] == "b" and last_page_token_value is None: + return {"next_page_token": "b page 2"} + return None + + paginator.next_page_token.side_effect = next_page_token + + def send_request(*args, **kwargs): + partition = kwargs["stream_slice"].partition["id"] + if partition == "a": + if len(requested_page_sizes["a"]) == 1: + raise PageSizeReductionRequiredException() + # the retry is in flight with the reduced page size + partition_a_reduced.set() + elif len(requested_page_sizes["b"]) == 1: + # hold partition b's first page open until partition a has reduced, so that b reads the page + # size in effect strictly after the reduction happened + assert partition_a_reduced.wait(timeout=10) + return [{"id": partition}] + + requester = Mock(spec=Requester) + requester.send_request.side_effect = send_request + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.side_effect = lambda **kwargs: [{"id": 1}] + + retriever = _page_size_reduction_retriever( + requester, paginator, record_selector, PageSizeReduction() + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [ + executor.submit(lambda s=s: list(retriever.read_records(A_RECORD_SCHEMA, s))) + for s in (slice_a, slice_b) + ] + for future in futures: + future.result() + + assert requested_page_sizes["a"] == [None, 50] + assert requested_page_sizes["b"] == [None, None] + + +def test_given_partitions_read_concurrently_then_each_read_owns_its_page_size_reducer(): + """ + Cheap and fully deterministic counterpart to the test above: two concurrent reads of the same retriever + must not share the object that holds the page size in effect. + """ + reducers = [] + original_init = PageSizeReducer.__init__ + + def record_reducer(self, *args, **kwargs): + original_init(self, *args, **kwargs) + reducers.append(self) + + requester = Mock(spec=Requester) + requester.send_request.return_value = [{"id": 1}] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.return_value = [{"id": 1}] + paginator = _mock_paginator() + paginator.get_page_size.return_value = 100 + paginator.get_initial_token.return_value = None + paginator.next_page_token.return_value = None + + retriever = _page_size_reduction_retriever( + requester, paginator, record_selector, PageSizeReduction() + ) + + with patch.object(PageSizeReducer, "__init__", record_reducer): + for partition in ("a", "b"): + list( + retriever.read_records( + A_RECORD_SCHEMA, StreamSlice(cursor_slice={}, partition={"id": partition}) + ) + ) + + assert len(reducers) == 2 + assert reducers[0] is not reducers[1] + + +def test_given_page_size_reduction_when_read_records_then_outgoing_request_carries_reduced_page_size(): + """ + The tests above assert that the retriever hands the reduced page size to the paginator. This one uses a + real DefaultPaginator so that a break anywhere between the retriever and `inject_into_request` is caught. + """ + response = requests.Response() + response.status_code = 200 + response._content = b"{}" + requester = Mock(spec=Requester) + requester.send_request.side_effect = [ + PageSizeReductionRequiredException(), + response, + ] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.return_value = [{"id": 1}] + paginator = DefaultPaginator( + page_size_option=RequestOption( + field_name="limit", inject_into=RequestOptionType.request_parameter, parameters={} + ), + page_token_option=RequestOption( + field_name="cursor", inject_into=RequestOptionType.request_parameter, parameters={} + ), + pagination_strategy=CursorPaginationStrategy( + page_size=100, cursor_value="{{ None }}", config={}, parameters={} + ), + config={}, + url_base="https://airbyte.io", + parameters={}, + ) + + retriever = _page_size_reduction_retriever( + requester, paginator, record_selector, PageSizeReduction() + ) + + records = list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + assert records == [{"id": 1}] + assert [call.kwargs["request_params"] for call in requester.send_request.call_args_list] == [ + {"limit": 100}, + {"limit": 50}, + ] + + def _mock_paginator(): paginator = Mock(spec=Paginator) paginator.get_request_params.__name__ = "get_request_params" diff --git a/unit_tests/sources/declarative/test_concurrent_declarative_source.py b/unit_tests/sources/declarative/test_concurrent_declarative_source.py index e2fef671c9..c8aaca355b 100644 --- a/unit_tests/sources/declarative/test_concurrent_declarative_source.py +++ b/unit_tests/sources/declarative/test_concurrent_declarative_source.py @@ -4981,6 +4981,192 @@ def test_given_response_action_is_pagination_reset_when_read_then_reset_paginati assert len(list(filter(lambda message: message.type == Type.RECORD, messages))) +def _page_size_reduction_manifest(pagination_strategy, page_token_option=None): + paginator = { + "type": "DefaultPaginator", + "pagination_strategy": pagination_strategy, + "page_size_option": { + "type": "RequestOption", + "inject_into": "request_parameter", + "field_name": "first", + }, + } + if page_token_option: + paginator["page_token_option"] = page_token_option + return { + "version": "0.34.2", + "type": "DeclarativeSource", + "check": {"type": "CheckStream", "stream_names": ["Test"]}, + "streams": [ + { + "type": "DeclarativeStream", + "name": "Test", + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": {"type": "object"}, + }, + "retriever": { + "type": "SimpleRetriever", + "page_size_reduction": {"type": "PageSizeReduction"}, + "requester": { + "type": "HttpRequester", + "url_base": "https://example.org", + "path": "/test", + "authenticator": {"type": "NoAuth"}, + "error_handler": { + "type": "DefaultErrorHandler", + "response_filters": [ + { + "type": "HttpResponseFilter", + "http_codes": [502], + # no `failure_type`: HttpResponseFilter only applies it to the FAIL + # action, and the failure the user sees comes from PageSizeReducer + "action": "REDUCE_PAGE_SIZE", + }, + ], + }, + }, + "paginator": paginator, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": ["items"]}, + }, + }, + } + ], + "spec": { + "type": "Spec", + "documentation_url": "https://example.org", + "connection_specification": {}, + }, + } + + +def _read_page_size_reduction_source(manifest): + catalog = create_catalog("Test") + source = ConcurrentDeclarativeSource( + source_config=manifest, + config={}, + catalog=catalog, + state=None, + ) + # the reducer waits before each reduction retry; taking those waits for real adds seconds to every CI run + with patch("airbyte_cdk.sources.declarative.retrievers.page_size_reducer.time.sleep"): + yield from source.read(logger=source.logger, config={}, catalog=catalog, state=[]) + + +def test_given_reduce_page_size_action_when_read_then_retry_page_with_smaller_page_size(): + """ + The call counts are asserted explicitly: the context-manager form of `HttpMocker` does not validate that + every matcher was called, so without them the test would also pass if the connector had started at the + reduced page size and never requested the configured one. + """ + manifest = _page_size_reduction_manifest( + { + "type": "CursorPagination", + "page_size": 100, + "cursor_value": "{{ response.next }}", + "stop_condition": "{{ not response.next }}", + } + ) + full_page_request = HttpRequest("https://example.org/test", query_params={"first": "100"}) + reduced_page_request = HttpRequest("https://example.org/test", query_params={"first": "50"}) + + with HttpMocker() as http_mocker: + http_mocker.get(full_page_request, HttpResponse("", 502)) + http_mocker.get(reduced_page_request, HttpResponse(json.dumps({"items": [{"id": 1}]}), 200)) + + messages = list(_read_page_size_reduction_source(manifest)) + + http_mocker.assert_number_of_calls(full_page_request, 1) + http_mocker.assert_number_of_calls(reduced_page_request, 1) + + assert [message.record.data["id"] for message in messages if message.type == Type.RECORD] == [1] + + +def test_given_offset_increment_and_reduce_page_size_action_when_read_then_keep_paginating(): + """ + `OffsetIncrement` is the only strategy whose stop condition depends on the page size. A page that is full + for the reduced size is smaller than the configured size, so comparing against the configured size would + end the pagination there and silently drop the tail of the partition. + """ + manifest = _page_size_reduction_manifest( + {"type": "OffsetIncrement", "page_size": 100}, + page_token_option={ + "type": "RequestOption", + "inject_into": "request_parameter", + "field_name": "offset", + }, + ) + full_page_request = HttpRequest("https://example.org/test", query_params={"first": "100"}) + first_reduced_page_request = HttpRequest( + "https://example.org/test", query_params={"first": "50"} + ) + second_reduced_page_request = HttpRequest( + "https://example.org/test", query_params={"first": "50", "offset": "50"} + ) + with HttpMocker() as http_mocker: + http_mocker.get(full_page_request, HttpResponse("", 502)) + http_mocker.get( + first_reduced_page_request, + HttpResponse(json.dumps({"items": [{"id": index} for index in range(50)]}), 200), + ) + http_mocker.get( + second_reduced_page_request, + HttpResponse(json.dumps({"items": [{"id": 50 + index} for index in range(20)]}), 200), + ) + + messages = list(_read_page_size_reduction_source(manifest)) + + http_mocker.assert_number_of_calls(full_page_request, 1) + http_mocker.assert_number_of_calls(first_reduced_page_request, 1) + http_mocker.assert_number_of_calls(second_reduced_page_request, 1) + + assert [ + message.record.data["id"] for message in messages if message.type == Type.RECORD + ] == list(range(70)) + + +def test_given_reductions_exhausted_when_read_then_emit_a_transient_error(): + """ + The failure type decides whether the platform retries the whole job, and an endpoint that refuses every + page size is the case the reduction budget exists for. + """ + manifest = _page_size_reduction_manifest( + { + "type": "CursorPagination", + "page_size": 100, + "cursor_value": "{{ response.next }}", + "stop_condition": "{{ not response.next }}", + } + ) + manifest["streams"][0]["retriever"]["page_size_reduction"]["max_attempts"] = 2 + + messages = [] + with HttpMocker() as http_mocker: + for page_size in ("100", "50", "25"): + http_mocker.get( + HttpRequest("https://example.org/test", query_params={"first": page_size}), + HttpResponse("", 502), + ) + + # the read fails, which is the point: the messages emitted before it are what the platform sees + with pytest.raises(AirbyteTracedException): + messages.extend(_read_page_size_reduction_source(manifest)) + + errors = [ + message.trace.error + for message in messages + if message.type == Type.TRACE and message.trace.type == TraceType.ERROR + ] + assert errors + assert all(error.failure_type == FailureType.transient_error for error in errors) + assert any( + "keeps rejecting pages of stream" in error.message and "records per page" in error.message + for error in errors + ) + + def test_given_pagination_limit_reached_when_read_then_reset_pagination(): input_config = {} manifest = { diff --git a/unit_tests/sources/streams/http/test_http_client.py b/unit_tests/sources/streams/http/test_http_client.py index 2c642ab9b1..14ddcaf7a6 100644 --- a/unit_tests/sources/streams/http/test_http_client.py +++ b/unit_tests/sources/streams/http/test_http_client.py @@ -1,5 +1,6 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. +import json import logging import os import time @@ -11,7 +12,9 @@ from pympler import asizeof from requests_cache import CachedRequest -from airbyte_cdk.models import FailureType +from airbyte_cdk.models import FailureType, Level +from airbyte_cdk.sources.http_logger import format_http_message +from airbyte_cdk.sources.message import InMemoryMessageRepository from airbyte_cdk.sources.streams.call_rate import CachedLimiterSession, LimiterSession from airbyte_cdk.sources.streams.http import HttpClient from airbyte_cdk.sources.streams.http.error_handlers import ( @@ -27,6 +30,9 @@ UserDefinedBackoffException, ) from airbyte_cdk.sources.streams.http.http_client import MessageRepresentationAirbyteTracedErrors +from airbyte_cdk.sources.streams.http.page_size_reduction_exception import ( + PageSizeReductionRequiredException, +) from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from airbyte_cdk.utils.traced_exception import AirbyteTracedException @@ -1441,3 +1447,121 @@ def test_deprecated_alias_is_catchable_as_airbyte_traced_exception(): internal_message="test", message="test user message", ) + + +def test_send_raises_page_size_reduction_required_exception_with_reduce_page_size_response_action(): + mocked_session = MagicMock(spec=requests.Session) + http_client = HttpClient( + name="test", + logger=MagicMock(), + error_handler=HttpStatusErrorHandler( + logger=MagicMock(), + error_mapping={ + 502: ErrorResolution( + ResponseAction.REDUCE_PAGE_SIZE, + FailureType.transient_error, + "test reduce page size message", + ) + }, + ), + session=mocked_session, + ) + mocked_response = requests.Response() + mocked_response.status_code = 502 + mocked_session.send.return_value = mocked_response + + # the retriever is responsible for retrying with a smaller page, so the backoff handlers must not retry + with pytest.raises(PageSizeReductionRequiredException) as exception: + http_client.send_request(http_method="get", url="https://airbyte.io", request_kwargs={}) + + assert http_client._session.send.call_count == 1 + # the error handler's own error_message has no other outlet, so it must reach the internal message. The + # stream is also called "test", so this asserts the mapping's text rather than any occurrence of "test". + assert "test reduce page size message" in exception.value.internal_message + # the exception is raised on every reduction, including the ones a correctly configured connector makes, + # so its message must describe the event rather than accuse the connector of a bug + assert "should be reported" not in exception.value.message + assert exception.value.message == ( + "The API rejected a page of stream test and asked the connector for a smaller one. If this message " + "ends a sync, the stream is not set up to request a smaller page: add `page_size_reduction` to its " + "retriever, or remove the REDUCE_PAGE_SIZE action from its error handler." + ) + # a retriever that cannot re-issue the page never retries it, so a job-level retry cannot help + assert exception.value.failure_type == FailureType.config_error + + +def test_given_reduce_page_size_action_then_log_the_response_as_an_auxiliary_request(): + """ + The Connector Builder builds one page per non-auxiliary HTTP log and bounds a slice by the number of those + pages. A response resolving to REDUCE_PAGE_SIZE never becomes a page - the retriever re-issues it - so + counting it would report "limit reached" on a read that only retried. + """ + message_repository = InMemoryMessageRepository(Level.DEBUG) + mocked_session = MagicMock(spec=requests.Session) + http_client = HttpClient( + name="test", + logger=MagicMock(), + error_handler=HttpStatusErrorHandler( + logger=MagicMock(), + error_mapping={ + 502: ErrorResolution( + ResponseAction.REDUCE_PAGE_SIZE, + FailureType.transient_error, + "test reduce page size message", + ) + }, + ), + session=mocked_session, + message_repository=message_repository, + ) + mocked_response = requests.Response() + mocked_response.status_code = 502 + mocked_response.request = requests.Request(method="GET", url="https://airbyte.io").prepare() + mocked_session.send.return_value = mocked_response + + with pytest.raises(PageSizeReductionRequiredException): + http_client.send_request( + http_method="get", + url="https://airbyte.io", + request_kwargs={}, + log_formatter=lambda response: format_http_message( + response, "a title", "a description", "test" + ), + ) + + logged = [json.loads(message.log.message) for message in message_repository.consume_queue()] + assert [entry["http"]["is_auxiliary"] for entry in logged] == [True] + # The Builder labels its auxiliary panel from these two, and the formatter filled them with the wording of + # an ordinary page, so a rejected request would otherwise be indistinguishable from a successful fetch. + assert logged[0]["http"]["title"] == ( + "Stream 'test' page rejected, retrying with a smaller page size" + ) + assert "no records" in logged[0]["http"]["description"] + + +def test_given_no_reduce_page_size_action_then_log_the_response_as_a_page(): + message_repository = InMemoryMessageRepository(Level.DEBUG) + mocked_session = MagicMock(spec=requests.Session) + http_client = HttpClient( + name="test", + logger=MagicMock(), + error_handler=HttpStatusErrorHandler(logger=MagicMock()), + session=mocked_session, + message_repository=message_repository, + ) + mocked_response = requests.Response() + mocked_response.status_code = 200 + mocked_response.request = requests.Request(method="GET", url="https://airbyte.io").prepare() + mocked_session.send.return_value = mocked_response + + http_client.send_request( + http_method="get", + url="https://airbyte.io", + request_kwargs={}, + log_formatter=lambda response: format_http_message( + response, "a title", "a description", "test" + ), + ) + + logged = [json.loads(message.log.message) for message in message_repository.consume_queue()] + assert [entry["http"].get("is_auxiliary") for entry in logged] == [None]