From 1c5424d8fff5e7934645fa0fc54bf4b835c3d973 Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Tue, 15 Sep 2026 20:02:06 +0300 Subject: [PATCH 01/13] feat: add REDUCE_PAGE_SIZE response action for dynamic page-size reduction An HttpClient retry re-sends the same PreparedRequest, so an error handler that mutates the stream's page size can never affect the request being retried: the oversized page size is already serialized into the URL or the body. This adds a `REDUCE_PAGE_SIZE` response action and a `page_size_reduction` block on `SimpleRetriever` so a manifest can say "re-issue this page smaller" instead. On a matching response, `SimpleRetriever._read_pages` shrinks the page size and re-issues the same page. All reduction state lives in a `PageSizeReducer` created per `_read_pages` call, mirroring `PaginationTracker`, because one retriever and one paginator are shared by every partition of a stream and partitions are read concurrently. The reduced page size reaches the paginator and the pagination strategy as an appended `page_size_override: Optional[int] = None` keyword argument, built by `page_size_override_kwargs` so it is omitted when there is no reduction. A paginator or strategy defined outside the CDK keeps working unchanged. Stop conditions see the size that was actually requested: - `OffsetIncrement` compares `last_page_size` against the requested size. - `CursorPagination` exposes it to `stop_condition` and `cursor_value` as `page_size`, and the factory rejects a `stop_condition` comparing `last_page_size` against anything else, because a full page at the reduced size would otherwise read as a short page and end the pagination silently. - `PageIncrement` is rejected, including through a subclass: pages are addressed as page number * page size, so a smaller page shifts every following boundary. Termination is bounded by `max_attempts` reductions. That budget restarts after every successful page under `reset_policy: AFTER_SUCCESSFUL_PAGE`, which is the policy for an API that rejects the configured page size on every page - without the restart such a stream would fail at page `max_attempts + 1` however healthy the reads are. `PageSizeReducer.MAX_TOTAL_REDUCTIONS` bounds the partition either way, and each reduction waits a short, growing amount of time before re-issuing. The action is rejected at config time everywhere it cannot be honored: query properties, `file_uploader`, `lazy_read_pointer`, all six `AsyncRetriever` sub-requesters and `login_requester`. A `CustomPaginationStrategy` is accepted only when its `next_page_token` can receive the override. When the action reaches a retriever the factory could not inspect, the reduction fails with a config error naming the stream rather than a bare crash. `HttpClient` logs a response resolving to `REDUCE_PAGE_SIZE` as an auxiliary request, so a Connector Builder test read still shows the failed attempt without counting it against `max_pages_per_slice`. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 3 + .../declarative_component_schema.yaml | 85 +- .../models/declarative_component_schema.py | 44 +- .../parsers/model_to_component_factory.py | 282 +++++- .../error_handlers/composite_error_handler.py | 1 + .../requesters/paginators/__init__.py | 2 + .../paginators/default_paginator.py | 94 +- .../requesters/paginators/no_pagination.py | 1 + .../requesters/paginators/paginator.py | 25 +- .../strategies/cursor_pagination_strategy.py | 11 + .../paginators/strategies/offset_increment.py | 20 +- .../paginators/strategies/page_increment.py | 18 + .../strategies/pagination_strategy.py | 4 + .../paginators/strategies/stop_condition.py | 10 +- .../retrievers/page_size_reducer.py | 182 ++++ .../retrievers/simple_retriever.py | 87 +- .../http/error_handlers/response_models.py | 1 + .../sources/streams/http/http_client.py | 33 +- .../http/page_size_reduction_exception.py | 51 ++ .../test_connector_builder_handler.py | 125 +++ .../test_model_to_component_factory.py | 809 ++++++++++++++++++ .../test_composite_error_handler.py | 23 + .../test_cursor_pagination_strategy.py | 57 ++ .../paginators/test_default_paginator.py | 135 +++ .../paginators/test_offset_increment.py | 95 ++ .../paginators/test_page_increment.py | 23 + .../paginators/test_stop_condition.py | 15 + .../retrievers/test_page_size_reducer.py | 237 +++++ .../retrievers/test_simple_retriever.py | 430 +++++++++- .../test_concurrent_declarative_source.py | 184 ++++ .../sources/streams/http/test_http_client.py | 115 ++- 31 files changed, 3167 insertions(+), 35 deletions(-) create mode 100644 airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py create mode 100644 airbyte_cdk/sources/streams/http/page_size_reduction_exception.py create mode 100644 unit_tests/sources/declarative/retrievers/test_page_size_reducer.py diff --git a/.gitignore b/.gitignore index 6835e1e631..6644e574e2 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,9 @@ node_modules # TODO: these are tmp files generated by unit tests. They should go to the /tmp directory. - + Template string evaluating when to stop paginating. Compare last_page_size against page_size rather than + against a hardcoded number: page_size is the page size that was actually requested, so the condition stays + correct when page_size_reduction shrinks it. type: string interpolation_context: - config - headers + - last_page_size - last_record + - page_size - response examples: - "{{ response.data.has_more is false }}" - "{{ 'next' not in headers['link'] }}" + - "{{ last_page_size < page_size }}" $parameters: type: object additionalProperties: true @@ -2658,6 +2665,7 @@ definitions: - RESET_PAGINATION - RATE_LIMITED - REFRESH_TOKEN_THEN_RETRY + - REDUCE_PAGE_SIZE examples: - SUCCESS - FAIL @@ -2666,6 +2674,7 @@ definitions: - RESET_PAGINATION - RATE_LIMITED - REFRESH_TOKEN_THEN_RETRY + - REDUCE_PAGE_SIZE failure_type: title: Failure Type description: Failure type of traced exception if a response matches the filter. @@ -4223,6 +4232,14 @@ definitions: pagination_reset: description: Describes what triggers pagination reset and how to handle it. "$ref": "#/definitions/PaginationReset" + page_size_reduction: + description: >- + Describes how the page size is reduced when an error handler resolves to the REDUCE_PAGE_SIZE action. + Requires a DefaultPaginator that defines both page_size_option and a pagination strategy with a page_size. + Cannot be combined with query properties, a file uploader, or a parent stream read lazily through + lazy_read_pointer, because in those cases records of the failing page have already been emitted and + re-issuing the page would emit them twice. + "$ref": "#/definitions/PageSizeReduction" ignore_stream_slicer_parameters_on_paginated_requests: description: If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored. type: boolean @@ -4278,6 +4295,72 @@ definitions: enum: [PaginationResetLimits] number_of_records: type: integer + PageSizeReduction: + title: Page Size Reduction + description: >- + Describes how the page size is reduced when an error handler resolves to the REDUCE_PAGE_SIZE action. On such + a response, the connector re-issues the same page with a smaller page size instead of retrying the identical + request. Only supported on a DefaultPaginator that has a page_size_option and whose pagination_strategy + defines a page_size and is CursorPagination, OffsetIncrement, or a CustomPaginationStrategy whose + next_page_token accepts a page_size_override keyword argument. PageIncrement is not supported because a + smaller page size moves every following page boundary and would skip records. It is also rejected when the + stream uses query properties, a file uploader, or reads its parent stream lazily through lazy_read_pointer, + since re-issuing a page whose records were already emitted would duplicate them. Each reduction waits a + short, growing amount of time before re-issuing the page so that an endpoint failing at every page size + is not hit in a burst. + type: object + required: + - type + properties: + type: + type: string + enum: [PageSizeReduction] + reduction_factor: + title: Reduction Factor + description: Divisor applied to the page size on each reduction. The new page size is floor(current page size / reduction factor). + type: number + default: 2 + exclusiveMinimum: 1 + examples: + - 2 + - 4 + minimum_page_size: + title: Minimum Page Size + description: >- + Page size below which the connector stops reducing and fails the sync. It must be smaller than the page + size configured on the pagination strategy, otherwise no reduction could ever be applied. + type: integer + default: 1 + minimum: 1 + examples: + - 1 + - 10 + max_attempts: + title: Maximum Reduction Attempts + description: >- + Maximum number of consecutive page size reductions allowed before the sync fails with a transient error. + Every reduction follows a request that failed, so at most max_attempts + 1 failing requests are issued + before giving up. With reset_policy NEVER this bounds the reductions for the whole partition; with + AFTER_SUCCESSFUL_PAGE the budget restarts after every page that succeeds, so it bounds the reductions + needed to get a single page through. + type: integer + default: 5 + minimum: 1 + examples: + - 5 + - 10 + reset_policy: + title: Reset Policy + description: >- + When to restore the page size configured on the pagination strategy. NEVER keeps the reduced page size for + the rest of the partition. AFTER_SUCCESSFUL_PAGE restores it as soon as one page succeeds, which means + hitting the same error again on every page - use it only when the reduction is worth one extra request per + page, for instance because the configured page size usually works and only some pages are too heavy. + type: string + enum: + - NEVER + - AFTER_SUCCESSFUL_PAGE + default: NEVER GzipDecoder: title: gzip description: Select 'gzip' for response data that is compressed with gzip. Requires specifying an inner data type/decoder to parse the decompressed data. diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 21a92cc3be..2f6b52defb 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -140,10 +140,11 @@ class CursorPagination(BaseModel): ) stop_condition: Optional[str] = Field( None, - description="Template string evaluating when to stop paginating.", + description="Template string evaluating when to stop paginating. Compare last_page_size against page_size rather than against a hardcoded number: page_size is the page size that was actually requested, so the condition stays correct when page_size_reduction shrinks it.", examples=[ "{{ response.data.has_more is false }}", "{{ 'next' not in headers['link'] }}", + "{{ last_page_size < page_size }}", ], title="Stop Condition", ) @@ -714,6 +715,7 @@ class Action(Enum): RESET_PAGINATION = "RESET_PAGINATION" RATE_LIMITED = "RATE_LIMITED" REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" + REDUCE_PAGE_SIZE = "REDUCE_PAGE_SIZE" class FailureType(Enum): @@ -735,6 +737,7 @@ class HttpResponseFilter(BaseModel): "RESET_PAGINATION", "RATE_LIMITED", "REFRESH_TOKEN_THEN_RETRY", + "REDUCE_PAGE_SIZE", ], title="Action", ) @@ -1410,6 +1413,41 @@ class PaginationResetLimits(BaseModel): number_of_records: Optional[int] = None +class ResetPolicy(Enum): + NEVER = "NEVER" + AFTER_SUCCESSFUL_PAGE = "AFTER_SUCCESSFUL_PAGE" + + +class PageSizeReduction(BaseModel): + type: Literal["PageSizeReduction"] + reduction_factor: Optional[float] = Field( + 2, + description="Divisor applied to the page size on each reduction. The new page size is floor(current page size / reduction factor).", + examples=[2, 4], + gt=1.0, + title="Reduction Factor", + ) + minimum_page_size: Optional[int] = Field( + 1, + description="Page size below which the connector stops reducing and fails the sync. It must be smaller than the page size configured on the pagination strategy, otherwise no reduction could ever be applied.", + examples=[1, 10], + ge=1, + title="Minimum Page Size", + ) + max_attempts: Optional[int] = Field( + 5, + description="Maximum number of consecutive page size reductions allowed before the sync fails with a transient error. Every reduction follows a request that failed, so at most max_attempts + 1 failing requests are issued before giving up. With reset_policy NEVER this bounds the reductions for the whole partition; with AFTER_SUCCESSFUL_PAGE the budget restarts after every page that succeeds, so it bounds the reductions needed to get a single page through.", + examples=[5, 10], + ge=1, + title="Maximum Reduction Attempts", + ) + reset_policy: Optional[ResetPolicy] = Field( + ResetPolicy.NEVER, + description="When to restore the page size configured on the pagination strategy. NEVER keeps the reduced page size for the rest of the partition. AFTER_SUCCESSFUL_PAGE restores it as soon as one page succeeds, which means hitting the same error again on every page - use it only when the reduction is worth one extra request per page, for instance because the configured page size usually works and only some pages are too heavy.", + title="Reset Policy", + ) + + class CsvDecoder(BaseModel): type: Literal["CsvDecoder"] encoding: Optional[str] = "utf-8" @@ -3222,6 +3260,10 @@ class SimpleRetriever(BaseModel): None, description="Describes what triggers pagination reset and how to handle it.", ) + page_size_reduction: Optional[PageSizeReduction] = Field( + None, + description="Describes how the page size is reduced when an error handler resolves to the REDUCE_PAGE_SIZE action. Requires a DefaultPaginator that defines both page_size_option and a pagination strategy with a page_size. Cannot be combined with query properties, a file uploader, or a parent stream read lazily through lazy_read_pointer, because in those cases records of the failing page have already been emitted and re-issuing the page would emit them twice.", + ) ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( False, description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index aa546c73ab..b8d7ddf87b 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -138,6 +138,9 @@ DEPRECATION_LOGS_TAG, BaseModelWithDeprecations, ) +from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( + Action as HttpResponseFilterActionModel, +) from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( Action1 as PaginationResetActionModel, ) @@ -384,6 +387,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( PageIncrement as PageIncrementModel, ) +from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( + PageSizeReduction as PageSizeReductionModel, +) from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( PaginationReset as PaginationResetModel, ) @@ -579,6 +585,10 @@ LocalFileSystemFileWriter, NoopFileWriter, ) +from airbyte_cdk.sources.declarative.retrievers.page_size_reducer import ( + PageSizeReduction, + PageSizeResetPolicy, +) from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker from airbyte_cdk.sources.declarative.schema import ( ComplexFieldType, @@ -1197,6 +1207,9 @@ def create_legacy_to_per_partition_state_migration( def create_session_token_authenticator( self, model: SessionTokenAuthenticatorModel, config: Config, name: str, **kwargs: Any ) -> Union[ApiKeyAuthenticator, BearerAuthenticator]: + self._reject_reduce_page_size_action( + model.login_requester, f"`login_requester` of the SessionTokenAuthenticator of {name}" + ) decoder = ( self._create_component_from_model(model=model.decoder, config=config) if model.decoder @@ -3614,14 +3627,30 @@ def _get_url(req: Requester) -> str: model.ignore_stream_slicer_parameters_on_paginated_requests or False ) - if ( + reads_parent_stream_lazily = bool( model.partition_router and isinstance(model.partition_router, SubstreamPartitionRouterModel) - and not bool(self._connector_state_manager.get_stream_state(name, None)) and any( parent_stream_config.lazy_read_pointer for parent_stream_config in model.partition_router.parent_stream_configs ) + ) + if reads_parent_stream_lazily and ( + model.page_size_reduction + or self._uses_reduce_page_size_action(getattr(model.requester, "error_handler", None)) + ): + # Checked outside of the LazySimpleRetriever branch below, which only applies on the first + # sync of a stream: gating it on the absence of state would accept the same manifest from the + # second sync onwards. LazySimpleRetriever paginates the parent's embedded pages, so there is + # no page of its own to re-issue with a smaller page size. + raise ValueError( + f"`page_size_reduction` and the REDUCE_PAGE_SIZE response action are not supported when " + f"reading a parent stream lazily. Remove either the page size reduction or the parent " + f"stream's `lazy_read_pointer` for stream {name}." + ) + + if reads_parent_stream_lazily and not bool( + self._connector_state_manager.get_stream_state(name, None) ): if incremental_sync: if incremental_sync.type != "DatetimeBasedCursor": @@ -3675,10 +3704,245 @@ def _get_url(req: Requester) -> str: pagination_tracker_factory=self._create_pagination_tracker_factory( model.pagination_reset, cursor ), + page_size_reduction=self._create_page_size_reduction( + model, name, query_properties, file_uploader + ), post_pagination_filter=post_pagination_filter, parameters=model.parameters or {}, ) + def _create_page_size_reduction( + self, + model: SimpleRetrieverModel, + name: str, + query_properties: Optional[QueryProperties], + file_uploader: Optional[DefaultFileUploader] = None, + ) -> Optional[PageSizeReduction]: + # A CustomRequester does not necessarily define an error handler. A CustomRequester that does define + # one keeps it as a raw dict rather than a typed model, so this returns False for it as well. + error_handler = getattr(model.requester, "error_handler", None) + uses_action = self._uses_reduce_page_size_action(error_handler) + if uses_action and not model.page_size_reduction: + raise ValueError( + f"Stream {name} has a response filter with the REDUCE_PAGE_SIZE action but the retriever does not " + f"define `page_size_reduction`. Add a `page_size_reduction` block to the retriever." + ) + + if not model.page_size_reduction: + return None + + if not uses_action: + # Not raised: a CustomErrorHandler can resolve to REDUCE_PAGE_SIZE without us being able to see + # it, so the only safe reaction to a block we cannot tie to an action is a warning. Without it a + # misspelled action leaves the feature silently dead on a stream that only exists because it + # would otherwise fail. + LOGGER.warning( + f"Stream {name} defines `page_size_reduction` but no response filter with the " + f"REDUCE_PAGE_SIZE action was found on its requester. The page size will never be reduced " + f"unless a custom error handler resolves to that action." + ) + + self._validate_page_size_reduction_is_supported( + model, name, query_properties, file_uploader + ) + + reset_policy = model.page_size_reduction.reset_policy + return PageSizeReduction( + reduction_factor=model.page_size_reduction.reduction_factor, # type: ignore[arg-type] # the schema defines a default + minimum_page_size=model.page_size_reduction.minimum_page_size, # type: ignore[arg-type] # the schema defines a default + max_attempts=model.page_size_reduction.max_attempts, # type: ignore[arg-type] # the schema defines a default + reset_policy=PageSizeResetPolicy(reset_policy.value) + if reset_policy is not None + else PageSizeResetPolicy.NEVER, + ) + + def _validate_page_size_reduction_is_supported( + self, + model: SimpleRetrieverModel, + name: str, + query_properties: Optional[QueryProperties], + file_uploader: Optional[DefaultFileUploader] = None, + ) -> None: + """ + Page size reduction re-issues the same page with a smaller page size. That is only correct when the next + page does not depend on the page size, and it only has an effect when the paginator injects the page size + in the request. A custom pagination strategy is accepted when it can receive the reduced page size, which + is checked by inspecting its signature rather than by recognizing its type. + """ + if query_properties: + raise ValueError( + f"`page_size_reduction` cannot be used together with query properties on stream {name}. Records " + f"from the earlier property chunks have already been emitted when a chunk asks for a smaller page, " + f"so retrying the page would emit them twice." + ) + + if file_uploader: + raise ValueError( + f"`page_size_reduction` cannot be used together with a `file_uploader` on stream {name}. The " + f"file uploader sends one request per record from inside the page's record generator, so a " + f"reduction asked for halfway through a page would re-emit the records already yielded by it." + ) + + if not isinstance(model.paginator, DefaultPaginatorModel): + raise ValueError( + f"`page_size_reduction` requires a DefaultPaginator on stream {name} so that the connector can " + f"send a smaller page size." + ) + + if not model.paginator.page_size_option: + raise ValueError( + f"`page_size_reduction` requires `page_size_option` on the paginator of stream {name}: without it " + f"the connector cannot tell the API to send a smaller page." + ) + + strategy = model.paginator.pagination_strategy + if isinstance(strategy, PageIncrementModel): + raise ValueError( + f"`page_size_reduction` does not support the PageIncrement pagination strategy used by stream " + f"{name}. Pages are addressed as page number * page size, so a smaller page size shifts every " + f"following page boundary and would skip records. Use OffsetIncrement or CursorPagination." + ) + if isinstance(strategy, CustomPaginationStrategyModel): + # A custom strategy is written by the same person enabling the reduction, so the + # question is not whether we recognize it but whether it can be told the reduced + # page size. Checking the signature keeps a strategy that would raise TypeError + # mid-sync from being accepted at config time. + custom_class = self._get_class_from_fully_qualified_class_name(strategy.class_name) + if isinstance(custom_class, type) and issubclass(custom_class, PageIncrement): + # `PageIncrement.next_page_token` declares `page_size_override` only to reject it, so a + # subclass that does not override the method would pass the signature check below and only + # fail once the first reduction is requested, mid-sync. + raise ValueError( + f"`page_size_reduction` does not support the PageIncrement pagination strategy that the " + f"custom pagination strategy {strategy.class_name} used by stream {name} inherits from. " + f"Pages are addressed as page number * page size, so a smaller page size shifts every " + f"following page boundary and would skip records. Use OffsetIncrement or CursorPagination." + ) + try: + parameters = inspect.signature(custom_class.next_page_token).parameters + except (AttributeError, TypeError, ValueError) as exception: + raise ValueError( + f"`page_size_reduction` could not check the signature of `next_page_token` on the custom " + f"pagination strategy {strategy.class_name} used by stream {name}: {exception}. Make sure " + f"`class_name` points at a PaginationStrategy subclass whose `next_page_token` accepts a " + f"`page_size_override` keyword argument." + ) + accepts_override = "page_size_override" in parameters or any( + parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in parameters.values() + ) + if not accepts_override: + raise ValueError( + f"`page_size_reduction` requires the custom pagination strategy " + f"{strategy.class_name} used by stream {name} to accept a `page_size_override` keyword " + f"argument in `next_page_token`, so that it can honor the reduced page size. Add " + f"`page_size_override: Optional[int] = None` to its signature; a strategy that does not " + f"use its page size as a stop condition can ignore the value." + ) + elif not isinstance(strategy, (CursorPaginationModel, OffsetIncrementModel)): + raise ValueError( + f"`page_size_reduction` only supports the CursorPagination, OffsetIncrement and " + f"CustomPaginationStrategy pagination strategies. Stream {name} uses " + f"{type(strategy).__name__}." + ) + else: + # A CustomPaginationStrategy carries its page size in its own code, so this is only checkable + # for the strategies the CDK defines. `page_size` is optional on both of them, and without it + # `get_page_size` returns None, the paginator injects nothing, and the reduction is a dead end + # that only surfaces on the first failing response - after records have been emitted. + self._validate_page_size_is_reducible(strategy, model.page_size_reduction, name) + if isinstance(strategy, CursorPaginationModel): + self._validate_stop_condition_is_reduction_aware(strategy, name) + + @staticmethod + def _validate_stop_condition_is_reduction_aware( + strategy: CursorPaginationModel, name: str + ) -> None: + """ + A `stop_condition` comparing `last_page_size` to a hardcoded page size reads a full reduced page as a + short page and ends the pagination early, dropping the rest of the partition without failing. The + strategy exposes the page size that was actually requested as `page_size`, so the condition can be + written correctly - but only if it is, which is what this checks. + """ + stop_condition = strategy.stop_condition + if not stop_condition: + return + + # `\b` does not match between the `_` and the `p` of `last_page_size`, so the second pattern only + # matches a standalone `page_size` reference. + uses_last_page_size = re.search(r"\blast_page_size\b", stop_condition) + uses_requested_page_size = re.search(r"\bpage_size\b", stop_condition) + if uses_last_page_size and not uses_requested_page_size: + raise ValueError( + f"`page_size_reduction` on stream {name} cannot be used with the `stop_condition` " + f"{stop_condition!r}: it compares `last_page_size` to a value that does not follow the " + f"reduction, so a full page at the reduced size would read as a short page and end the " + f"pagination early, silently dropping the rest of the partition. Compare against the " + f"`page_size` interpolation variable instead, which holds the page size that was actually " + f"requested (for example `{{{{ last_page_size < page_size }}}}`)." + ) + + @staticmethod + def _validate_page_size_is_reducible( + strategy: Union[CursorPaginationModel, OffsetIncrementModel], + page_size_reduction: Optional[PageSizeReductionModel], + name: str, + ) -> None: + page_size = strategy.page_size + if page_size is None: + raise ValueError( + f"`page_size_reduction` requires `page_size` on the pagination strategy of stream {name}: " + f"without it the paginator does not send a page size, so there is nothing to reduce." + ) + + # The schema allows a string so that the page size can be interpolated. `page_size` is + # `Optional[Union[int, str]]` and pydantic v1 tries `int` first, so both `100` and `"100"` arrive as + # an int and only a genuinely non-numeric template stays a str. Such a template is only known once + # the config is available, so it is left to the runtime check in `PageSizeReducer.reduce`. + try: + configured_page_size: Optional[int] = int(page_size) + except ValueError: + configured_page_size = None + + minimum_page_size = ( + page_size_reduction.minimum_page_size if page_size_reduction else None + ) or 1 + if configured_page_size is not None and configured_page_size <= minimum_page_size: + raise ValueError( + f"`page_size_reduction` on stream {name} can never reduce its page size: the pagination " + f"strategy's `page_size` is {configured_page_size} and `minimum_page_size` is " + f"{minimum_page_size}. Lower `minimum_page_size` or raise `page_size`." + ) + + def _reject_reduce_page_size_action(self, requester: Any, description: str) -> None: + """ + `error_handler` is defined on `HttpRequester`, which is referenced by requesters that have no page of + their own, so REDUCE_PAGE_SIZE is schema-legal in places where nothing can honor it. Only + `SimpleRetriever._read_pages` re-issues a page, so every other requester is rejected here rather than + surfacing the exception mid-sync as a generic failure. + """ + if requester is None: + return + if self._uses_reduce_page_size_action(getattr(requester, "error_handler", None)): + raise ValueError( + f"The REDUCE_PAGE_SIZE response action is not supported on the {description}: only the main " + f"requester of a SimpleRetriever can re-issue its page with a smaller page size. Use a " + f"different action on that error handler." + ) + + def _uses_reduce_page_size_action(self, error_handler: Any) -> bool: + if isinstance(error_handler, CompositeErrorHandlerModel): + return any( + self._uses_reduce_page_size_action(nested) + for nested in error_handler.error_handlers + ) + if isinstance(error_handler, DefaultErrorHandlerModel): + return any( + response_filter.action == HttpResponseFilterActionModel.REDUCE_PAGE_SIZE + for response_filter in error_handler.response_filters or [] + ) + # A CustomErrorHandler can return any action and we cannot inspect it, so we do not validate it. + return False + def _create_pagination_tracker_factory( self, model: Optional[PaginationResetModel], cursor: Cursor ) -> Callable[[], PaginationTracker]: @@ -3946,6 +4210,19 @@ def create_async_retriever( f"`download_target_extractor` required if using a `download_target_requester`" ) + for requester_field in ( + "creation_requester", + "polling_requester", + "download_requester", + "download_target_requester", + "abort_requester", + "delete_requester", + ): + self._reject_reduce_page_size_action( + getattr(model, requester_field, None), + f"`{requester_field}` of the AsyncRetriever of stream {name}", + ) + def _get_download_retriever( requester: Requester, extractor: RecordExtractor, _decoder: Decoder ) -> SimpleRetriever: @@ -4559,6 +4836,7 @@ def create_fixed_window_call_rate_policy( def create_file_uploader( self, model: FileUploaderModel, config: Config, **kwargs: Any ) -> FileUploader: + self._reject_reduce_page_size_action(model.requester, "requester of a `file_uploader`") name = "File Uploader" requester = self._create_component_from_model( model=model.requester, diff --git a/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py b/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py index c09f187276..5cf9532ec8 100644 --- a/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +++ b/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py @@ -71,6 +71,7 @@ def interpret_response( ResponseAction.RETRY, ResponseAction.IGNORE, ResponseAction.RESET_PAGINATION, + ResponseAction.REDUCE_PAGE_SIZE, ]: return matched_error_resolution diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/__init__.py b/airbyte_cdk/sources/declarative/requesters/paginators/__init__.py index 3b077ec0c3..475d722e1f 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/__init__.py @@ -12,6 +12,8 @@ PaginationStrategy, ) +# `page_size_override_kwargs` is deliberately not re-exported here: it is a CDK-internal helper, every call +# site is inside the CDK, and it imports from `paginators.paginator` directly. __all__ = [ "DefaultPaginator", "NoPagination", diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py b/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py index cb1072a8c5..d00e75ad29 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py @@ -13,7 +13,10 @@ PaginationDecoderDecorator, ) from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString -from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator +from airbyte_cdk.sources.declarative.requesters.paginators.paginator import ( + Paginator, + page_size_override_kwargs, +) from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import ( PaginationStrategy, ) @@ -133,18 +136,23 @@ def get_initial_token(self) -> Optional[Any]: """ return self.pagination_strategy.initial_token + def get_page_size(self) -> Optional[int]: + return self.pagination_strategy.get_page_size() + def next_page_token( self, response: requests.Response, last_page_size: int, last_record: Optional[Record], last_page_token_value: Optional[Any] = None, + page_size_override: Optional[int] = None, ) -> Optional[Mapping[str, Any]]: next_page_token = self.pagination_strategy.next_page_token( response=response, last_page_size=last_page_size, last_record=last_record, last_page_token_value=last_page_token_value, + **page_size_override_kwargs(page_size_override), ) if next_page_token: return {"next_page_token": next_page_token} @@ -169,8 +177,11 @@ def get_request_params( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> MutableMapping[str, Any]: - return self._get_request_options(RequestOptionType.request_parameter, next_page_token) + return self._get_request_options( + RequestOptionType.request_parameter, next_page_token, page_size_override + ) def get_request_headers( self, @@ -178,8 +189,11 @@ def get_request_headers( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Mapping[str, str]: - return self._get_request_options(RequestOptionType.header, next_page_token) + return self._get_request_options( + RequestOptionType.header, next_page_token, page_size_override + ) def get_request_body_data( self, @@ -187,8 +201,11 @@ def get_request_body_data( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Mapping[str, Any]: - return self._get_request_options(RequestOptionType.body_data, next_page_token) + return self._get_request_options( + RequestOptionType.body_data, next_page_token, page_size_override + ) def get_request_body_json( self, @@ -196,11 +213,17 @@ def get_request_body_json( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Mapping[str, Any]: - return self._get_request_options(RequestOptionType.body_json, next_page_token) + return self._get_request_options( + RequestOptionType.body_json, next_page_token, page_size_override + ) def _get_request_options( - self, option_type: RequestOptionType, next_page_token: Optional[Mapping[str, Any]] + self, + option_type: RequestOptionType, + next_page_token: Optional[Mapping[str, Any]], + page_size_override: Optional[int] = None, ) -> MutableMapping[str, Any]: options: MutableMapping[str, Any] = {} @@ -213,13 +236,14 @@ def _get_request_options( ): self.page_token_option.inject_into_request(options, token, self.config) - if ( - self.page_size_option - and self.pagination_strategy.get_page_size() - and self.page_size_option.inject_into == option_type - ): - page_size = self.pagination_strategy.get_page_size() - self.page_size_option.inject_into_request(options, page_size, self.config) + if self.page_size_option and self.page_size_option.inject_into == option_type: + page_size = ( + page_size_override + if page_size_override is not None + else self.pagination_strategy.get_page_size() + ) + if page_size: + self.page_size_option.inject_into_request(options, page_size, self.config) return options @@ -231,6 +255,16 @@ class PaginatorTestReadDecorator(Paginator): WARNING: This decorator is not currently thread-safe like the rest of the low-code framework because it has an internal state to track the current number of pages counted so that it can exit early during a test read + + The count bounds *successful* pages: it is incremented from `next_page_token`, which a page that resolved + to `ResponseAction.REDUCE_PAGE_SIZE` never reaches because `SimpleRetriever._read_pages` re-issues that + page before asking for the next token. A stream with `page_size_reduction` can therefore issue up to + `max_attempts` extra requests per slice on top of this limit. + + The Connector Builder counts pages a second time, from the request/response logs rather than from this + class (`connector_builder/test_reader/reader.py::_has_reached_limit`). The two counts agree because + `HttpClient` logs a response resolving to `REDUCE_PAGE_SIZE` as an auxiliary request, which the Builder + does not turn into a page. """ _PAGE_COUNT_BEFORE_FIRST_NEXT_CALL = 1 @@ -248,19 +282,27 @@ def get_initial_token(self) -> Optional[Any]: self._page_count = self._PAGE_COUNT_BEFORE_FIRST_NEXT_CALL return self._decorated.get_initial_token() + def get_page_size(self) -> Optional[int]: + return self._decorated.get_page_size() + def next_page_token( self, response: requests.Response, last_page_size: int, last_record: Optional[Record], last_page_token_value: Optional[Any] = None, + page_size_override: Optional[int] = None, ) -> Optional[Mapping[str, Any]]: if self._page_count >= self._maximum_number_of_pages: return None self._page_count += 1 return self._decorated.next_page_token( - response, last_page_size, last_record, last_page_token_value + response, + last_page_size, + last_record, + last_page_token_value, + **page_size_override_kwargs(page_size_override), ) def path( @@ -281,9 +323,13 @@ def get_request_params( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Mapping[str, Any]: return self._decorated.get_request_params( - stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + **page_size_override_kwargs(page_size_override), ) def get_request_headers( @@ -292,9 +338,13 @@ def get_request_headers( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Mapping[str, str]: return self._decorated.get_request_headers( - stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + **page_size_override_kwargs(page_size_override), ) def get_request_body_data( @@ -303,9 +353,13 @@ def get_request_body_data( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Union[Mapping[str, Any], str]: return self._decorated.get_request_body_data( - stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + **page_size_override_kwargs(page_size_override), ) def get_request_body_json( @@ -314,7 +368,11 @@ def get_request_body_json( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Mapping[str, Any]: return self._decorated.get_request_body_json( - stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + **page_size_override_kwargs(page_size_override), ) diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py b/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py index b3b1d3b662..f207b1cce9 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py @@ -72,5 +72,6 @@ def next_page_token( last_page_size: int, last_record: Optional[Record], last_page_token_value: Optional[Any], + page_size_override: Optional[int] = None, ) -> Optional[Mapping[str, Any]]: return {} diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py b/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py index f8c31d4f5f..17f974416e 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py @@ -4,7 +4,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import Any, Mapping, Optional +from typing import Any, Dict, Mapping, Optional import requests @@ -14,6 +14,16 @@ from airbyte_cdk.sources.types import Record, StreamSlice +def page_size_override_kwargs(page_size_override: Optional[int]) -> Dict[str, Any]: + """ + Build the `page_size_override` keyword argument only when there is an override to pass. + + Paginators and pagination strategies defined outside of the CDK may not accept the argument, and they only + need to when the stream actually reduces its page size (see `ResponseAction.REDUCE_PAGE_SIZE`). + """ + return {"page_size_override": page_size_override} if page_size_override is not None else {} + + @dataclass class Paginator(ABC, RequestOptionsProvider): """ @@ -36,6 +46,7 @@ def next_page_token( last_page_size: int, last_record: Optional[Record], last_page_token_value: Optional[Any], + page_size_override: Optional[int] = None, ) -> Optional[Mapping[str, Any]]: """ Returns the next_page_token to use to fetch the next page of records. @@ -44,10 +55,22 @@ def next_page_token( :param last_page_size: the number of records read from the response :param last_record: the last record extracted from the response :param last_page_token_value: The current value of the page token made on the last request + :param page_size_override: the page size that was actually requested, when it differs from the configured + one because of a `REDUCE_PAGE_SIZE` response action :return: A mapping {"next_page_token": } for the next page from the input response object. Returning None means there are no more pages to read in this response. """ pass + def get_page_size(self) -> Optional[int]: + """ + Evaluated against the config alone. A pagination strategy whose `page_size` template references the + response evaluates it with the response in context inside `next_page_token`, so the two can disagree - + pre-existing, and only observable for a `page_size` that is not a constant. + + :return: the number of records this paginator asks for per page, or None if it does not define one + """ + return None + @abstractmethod def path( self, diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py index 73a644b5ba..1e600a7bdd 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py @@ -77,7 +77,16 @@ def next_page_token( last_page_size: int, last_record: Optional[Record], last_page_token_value: Optional[Any] = None, + page_size_override: Optional[int] = None, ) -> Optional[Any]: + # The next page is a cursor read from the response, so `page_size_override` does not change how the token + # is computed. It is still exposed to the interpolation context as `page_size` because a `stop_condition` + # comparing `last_page_size` to a hardcoded page size would read a full reduced page as a short page and + # end the pagination early, silently dropping the rest of the partition. Writing the condition as + # `{{ last_page_size < page_size }}` keeps it correct while a reduction is in effect. + requested_page_size = ( + page_size_override if page_size_override is not None else self._page_size + ) decoded_response = next(self.decoder.decode(response)) # The default way that link is presented in requests.Response is a string of various links (last, next, etc). This # is not indexable or useful for parsing the cursor, so we replace it with the link dictionary from response.links @@ -90,6 +99,7 @@ def next_page_token( headers=headers, last_record=last_record, last_page_size=last_page_size, + page_size=requested_page_size, ) if should_stop: return None @@ -99,6 +109,7 @@ def next_page_token( headers=headers, last_record=last_record, last_page_size=last_page_size, + page_size=requested_page_size, ) return token if token else None diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py index 4370155dec..46939402c4 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py @@ -74,6 +74,7 @@ def next_page_token( last_page_size: int, last_record: Optional[Record], last_page_token_value: Optional[Any] = None, + page_size_override: Optional[int] = None, ) -> Optional[Any]: decoded_response = next(self.decoder.decode(response)) @@ -85,11 +86,20 @@ def next_page_token( page_size_from_response if page_size_from_response is not None else last_page_size ) - # Stop paginating when there are fewer records than the page size or the current page has no records - if ( - self._page_size - and last_page_size < self._page_size.eval(self.config, response=decoded_response) - ) or last_page_size == 0: + # Stop paginating when there are fewer records than the page size or the current page has no records. + # The comparison uses the page size that was actually requested: after a `REDUCE_PAGE_SIZE` reduction, a + # full page is smaller than the configured page size and comparing against the latter would end the + # pagination early and skip records. + requested_page_size = ( + page_size_override + if page_size_override is not None + else ( + self._page_size.eval(self.config, response=decoded_response) + if self._page_size + else None + ) + ) + if (requested_page_size and last_page_size < requested_page_size) or last_page_size == 0: return None elif last_page_token_value is None: # If the OffsetIncrement strategy does not inject on the first request, the incoming last_page_token_value diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py index f3bad7152b..026e0452d1 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py @@ -7,12 +7,14 @@ import requests +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import ( PaginationStrategy, ) from airbyte_cdk.sources.types import Config, Record +from airbyte_cdk.utils.traced_exception import AirbyteTracedException @dataclass @@ -53,7 +55,23 @@ def next_page_token( last_page_size: int, last_record: Optional[Record], last_page_token_value: Optional[Any], + page_size_override: Optional[int] = None, ) -> Optional[Any]: + if page_size_override is not None: + # Reachable only when the factory is bypassed: a manifest naming PageIncrement, and a + # CustomPaginationStrategy that subclasses it, are both rejected at config time. An unrecognized + # ValueError here would be reported as a generic system error even though the message describes a + # configuration mistake. `next_page_token` runs after the page's records have been emitted, so an + # out-of-tree caller gets this mid-stream rather than at startup - still better than silently + # ignoring the override and skipping records. + raise AirbyteTracedException( + internal_message="PageIncrement received a page_size_override", + message="PageIncrement does not support reducing the page size while paginating: pages are " + "addressed as page number * page size, so a smaller page size shifts every following page " + "boundary and would skip records. Use OffsetIncrement or CursorPagination instead.", + failure_type=FailureType.config_error, + ) + if self.extractor: # The record count is dependent on the records returned from the response which may not always # align with the size of pages emitted. For example, a record filter can reduce the number of diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py index dae02ba138..9be3af80ed 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py @@ -31,12 +31,16 @@ def next_page_token( last_page_size: int, last_record: Optional[Record], last_page_token_value: Optional[Any], + page_size_override: Optional[int] = None, ) -> Optional[Any]: """ :param response: response to process :param last_page_size: the number of records read from the response :param last_record: the last record extracted from the response :param last_page_token_value: The current value of the page token made on the last request + :param page_size_override: the page size that was actually requested, when it differs from the configured + one because of a `REDUCE_PAGE_SIZE` response action. Strategies that use their configured page size as + a stop condition must honor this value, else they end the pagination early and skip records. :return: next page token. Returns None if there are no more pages to fetch """ pass diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py index 068df72cb8..6df7a3a344 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py @@ -7,6 +7,9 @@ import requests +from airbyte_cdk.sources.declarative.requesters.paginators.paginator import ( + page_size_override_kwargs, +) from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import ( PaginationStrategy, ) @@ -47,13 +50,18 @@ def next_page_token( last_page_size: int, last_record: Optional[Record], last_page_token_value: Optional[Any] = None, + page_size_override: Optional[int] = None, ) -> Optional[Any]: # We evaluate in reverse order because the assumption is that most of the APIs using data feed structure # will return records in descending order. In terms of performance/memory, we return the records lazily if last_record and self._stop_condition.is_met(last_record): return None return self._delegate.next_page_token( - response, last_page_size, last_record, last_page_token_value + response, + last_page_size, + last_record, + last_page_token_value, + **page_size_override_kwargs(page_size_override), ) def get_page_size(self) -> Optional[int]: diff --git a/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py b/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py new file mode 100644 index 0000000000..ecb8854ed9 --- /dev/null +++ b/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py @@ -0,0 +1,182 @@ +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. + +import logging +import time +from dataclasses import dataclass +from enum import Enum +from typing import Callable, Optional + +from airbyte_cdk.models import FailureType +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + +LOGGER = logging.getLogger("airbyte") + + +class PageSizeResetPolicy(Enum): + NEVER = "NEVER" + AFTER_SUCCESSFUL_PAGE = "AFTER_SUCCESSFUL_PAGE" + + +@dataclass(frozen=True) +class PageSizeReduction: + """ + How much to shrink the page size when an error handler resolves to `ResponseAction.REDUCE_PAGE_SIZE`. + + This is immutable configuration: it is created once per stream and shared, while the page size in effect + lives in a `PageSizeReducer` created per partition read. + """ + + reduction_factor: float = 2.0 + minimum_page_size: int = 1 + max_attempts: int = 5 + reset_policy: PageSizeResetPolicy = PageSizeResetPolicy.NEVER + + def __post_init__(self) -> None: + if self.reduction_factor <= 1: + raise ValueError( + f"The page size reduction factor needs to be greater than 1. Got {self.reduction_factor}" + ) + if self.minimum_page_size < 1: + raise ValueError( + f"The minimum page size needs to be strictly positive. Got {self.minimum_page_size}" + ) + if self.max_attempts < 1: + raise ValueError( + f"The maximum number of page size reductions needs to be strictly positive. Got {self.max_attempts}" + ) + + +class PageSizeReducer: + """ + Tracks the page size to use while reading one partition when the API asks for smaller pages. + + One instance is created per `SimpleRetriever._read_pages` call. This is deliberate: a retriever and its + paginator are shared by every partition of a stream and partitions are read concurrently, so the reduced + page size must not be stored on the paginator or on the retriever. + """ + + # `PageSizeReductionRequiredException` is deliberately neither a `BaseBackoffException` nor a transient + # exception, so the reduced page is re-issued outside of the HTTP retry budget and nothing else spaces + # those requests out. The wait is kept non-zero and grows with the number of reductions so an endpoint + # that fails whatever page size we ask for degrades to a slow retry instead of a burst of requests. + BACKOFF_SECONDS: float = 0.5 + + # Backstop that bounds the reductions for the whole partition regardless of the reset policy. Under + # `AFTER_SUCCESSFUL_PAGE` the `max_attempts` budget restarts on every successful page, which is what lets a + # long partition complete when the API needs one reduction per page - so something else has to guarantee + # that the partition cannot spend reductions forever. It is deliberately far above any sane `max_attempts` + # because reaching it is a pathology, not a tuning problem, and it is therefore not exposed in the schema. + MAX_TOTAL_REDUCTIONS: int = 1000 + + def __init__( + self, + config: PageSizeReduction, + configured_page_size: Optional[int], + stream_name: str = "", + sleep: Callable[[float], None] = time.sleep, + ) -> None: + self._config = config + self._configured_page_size = configured_page_size + self._stream_name = stream_name + self._sleep = sleep + self._current_page_size: Optional[int] = None + self._attempts = 0 + self._total_reductions = 0 + + @property + def page_size_override(self) -> Optional[int]: + """ + :return: the reduced page size to request, or None while the configured page size is in effect + """ + return self._current_page_size + + def reduce(self) -> None: + """ + Shrink the page size used for the next request. Raises once the page size cannot be shrunk any further + so that an endpoint that keeps failing does not loop forever. + """ + if self._configured_page_size is None: + raise AirbyteTracedException( + internal_message=f"Stream {self._stream_name} received a REDUCE_PAGE_SIZE response action but its paginator does not inject a page size", + message="The connector is set up to reduce its page size on error but does not define one. Set `page_size` on the pagination strategy and `page_size_option` on the paginator.", + failure_type=FailureType.config_error, + ) + + current_page_size = ( + self._current_page_size + if self._current_page_size is not None + else self._configured_page_size + ) + if not isinstance(current_page_size, int) or isinstance(current_page_size, bool): + # A custom pagination strategy can return anything from `get_page_size`. Reducing + # is arithmetic, so a non-integer would otherwise fail with a bare TypeError in + # the middle of a sync. + raise AirbyteTracedException( + internal_message=f"Stream {self._stream_name} has a page size of type {type(current_page_size).__name__}: {current_page_size!r}", + message="The connector is set up to reduce its page size on error but its page size is not a whole number. " + "Make sure the pagination strategy's `get_page_size` returns an integer.", + failure_type=FailureType.config_error, + ) + + self._attempts += 1 + self._total_reductions += 1 + if ( + self._attempts > self._config.max_attempts + or self._total_reductions > self.MAX_TOTAL_REDUCTIONS + ): + raise AirbyteTracedException( + internal_message=f"Stream {self._stream_name} reduced its page size {self._total_reductions - 1} times while reading a single partition ({self._attempts - 1} of them since the last successful page), which is the maximum allowed", + message=f"The source kept failing while the connector requested smaller and smaller pages (down to {current_page_size} records per page). The API is likely unable to serve these requests. Try syncing fewer streams at once, or contact the API provider.", + failure_type=FailureType.transient_error, + ) + + reduced_page_size = max( + self._config.minimum_page_size, + int(current_page_size // self._config.reduction_factor), + ) + if reduced_page_size >= current_page_size: + if self._current_page_size is None: + # No reduction was ever applied, so the configured page size is already at or below the + # minimum. Nothing about the response can fix that, which makes it a configuration error + # rather than something the platform should retry the whole job for. + raise AirbyteTracedException( + internal_message=f"Stream {self._stream_name} has a configured page size of {current_page_size} which is not greater than the configured minimum page size of {self._config.minimum_page_size}, so it can never be reduced", + message=f"The connector is set up to reduce its page size on error but its page size ({current_page_size}) is already at or below the configured minimum of {self._config.minimum_page_size}. Lower `minimum_page_size` or raise the pagination strategy's `page_size`.", + failure_type=FailureType.config_error, + ) + raise AirbyteTracedException( + internal_message=f"Stream {self._stream_name} still fails with a page size of {current_page_size}, which is the smallest page size allowed by the configured minimum of {self._config.minimum_page_size}", + message=f"The source is still failing with the smallest page the connector is allowed to request ({current_page_size} records per page). The API is likely unable to serve this request. Try syncing fewer streams at once, or contact the API provider.", + failure_type=FailureType.transient_error, + ) + + backoff = self.BACKOFF_SECONDS * self._attempts + LOGGER.info( + f"Reducing the page size of stream {self._stream_name} from {current_page_size} to {reduced_page_size} " + f"and retrying the same page in {backoff}s." + ) + self._current_page_size = reduced_page_size + self._sleep(backoff) + + def on_successful_page(self) -> None: + """ + Called after each page that did not require a reduction. + + Under `NEVER` nothing happens: the reduced page size stays in effect and `max_attempts` keeps bounding + the reductions for the whole partition, which is the right budget when reductions are one-off. + + Under `AFTER_SUCCESSFUL_PAGE` the page size is restored and the `max_attempts` budget restarts. The + reduction count has to restart with it: this policy exists for an API that rejects the configured page + size on every page, so every page legitimately costs one reduction, and a budget spanning the whole + partition would fail the sync at page `max_attempts + 1` no matter how healthy the reads are. + `MAX_TOTAL_REDUCTIONS` still bounds the partition, so the sync cannot run forever. + """ + if self._config.reset_policy != PageSizeResetPolicy.AFTER_SUCCESSFUL_PAGE: + return + + self._attempts = 0 + if self._current_page_size is not None: + LOGGER.info( + f"Restoring the page size of stream {self._stream_name} from {self._current_page_size} to {self._configured_page_size}." + ) + self._current_page_size = None diff --git a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index 6f82b8ebd9..adfa504867 100644 --- a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -23,6 +23,7 @@ import requests from typing_extensions import deprecated +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector from airbyte_cdk.sources.declarative.extractors.record_filter import ( ClientSideIncrementalRecordFilterDecorator, @@ -32,23 +33,35 @@ SinglePartitionRouter, ) from airbyte_cdk.sources.declarative.requesters.paginators.no_pagination import NoPagination -from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator +from airbyte_cdk.sources.declarative.requesters.paginators.paginator import ( + Paginator, + page_size_override_kwargs, +) from airbyte_cdk.sources.declarative.requesters.query_properties import QueryProperties from airbyte_cdk.sources.declarative.requesters.request_options import ( DefaultRequestOptionsProvider, RequestOptionsProvider, ) from airbyte_cdk.sources.declarative.requesters.requester import Requester +from airbyte_cdk.sources.declarative.retrievers.page_size_reducer import ( + PageSizeReducer, + PageSizeReduction, +) from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer from airbyte_cdk.sources.source import ExperimentalClassWarning from airbyte_cdk.sources.streams.core import StreamData +from airbyte_cdk.sources.streams.http.page_size_reduction_exception import ( + PageSizeReductionNotSupportedException, + PageSizeReductionRequiredException, +) from airbyte_cdk.sources.streams.http.pagination_reset_exception import ( PaginationResetRequiredException, ) from airbyte_cdk.sources.types import Config, Record, StreamSlice from airbyte_cdk.utils.mapping_helpers import combine_mappings +from airbyte_cdk.utils.traced_exception import AirbyteTracedException FULL_REFRESH_SYNC_COMPLETE_KEY = "__ab_full_refresh_sync_complete" LOGGER = logging.getLogger("airbyte") @@ -77,6 +90,8 @@ class SimpleRetriever(Retriever): parameters (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation post_pagination_filter (Optional[ClientSideIncrementalRecordFilterDecorator]): Set for data feed streams only. Records the cursor considers already synced are dropped once pagination has observed them + page_size_reduction (Optional[PageSizeReduction]): How much to shrink the page size when an error handler + resolves to `ResponseAction.REDUCE_PAGE_SIZE`. `None` disables page size reduction entirely """ requester: Requester @@ -101,6 +116,7 @@ class SimpleRetriever(Retriever): default_factory=lambda: lambda: PaginationTracker() ) post_pagination_filter: Optional[ClientSideIncrementalRecordFilterDecorator] = None + page_size_reduction: Optional[PageSizeReduction] = None def __post_init__(self, parameters: Mapping[str, Any]) -> None: self._paginator = self.paginator or NoPagination(parameters=parameters) @@ -145,6 +161,7 @@ def _get_request_options( next_page_token: Optional[Mapping[str, Any]], paginator_method: Callable[..., Optional[Union[Mapping[str, Any], str]]], stream_slicer_method: Callable[..., Optional[Union[Mapping[str, Any], str]]], + page_size_override: Optional[int] = None, ) -> Union[Mapping[str, Any], str]: """ Get the request_option from the paginator and the stream slicer. @@ -157,6 +174,7 @@ def _get_request_options( paginator_method( stream_slice=stream_slice, next_page_token=next_page_token, + **page_size_override_kwargs(page_size_override), ), ] if not next_page_token or not self.ignore_stream_slicer_parameters_on_paginated_requests: @@ -172,6 +190,7 @@ def _request_headers( self, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Mapping[str, Any]: """ Specifies request headers. @@ -182,6 +201,7 @@ def _request_headers( next_page_token, self._paginator.get_request_headers, self.request_option_provider.get_request_headers, + **page_size_override_kwargs(page_size_override), ) if isinstance(headers, str): raise ValueError("Request headers cannot be a string") @@ -191,6 +211,7 @@ def _request_params( self, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Mapping[str, Any]: """ Specifies the query parameters that should be set on an outgoing HTTP request given the inputs. @@ -202,6 +223,7 @@ def _request_params( next_page_token, self._paginator.get_request_params, self.request_option_provider.get_request_params, + **page_size_override_kwargs(page_size_override), ) if isinstance(params, str): raise ValueError("Request params cannot be a string") @@ -211,6 +233,7 @@ def _request_body_data( self, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Union[Mapping[str, Any], str]: """ Specifies how to populate the body of the request with a non-JSON payload. @@ -226,12 +249,14 @@ def _request_body_data( next_page_token, self._paginator.get_request_body_data, self.request_option_provider.get_request_body_data, + **page_size_override_kwargs(page_size_override), ) def _request_body_json( self, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Optional[Mapping[str, Any]]: """ Specifies how to populate the body of the request with a JSON payload. @@ -243,6 +268,7 @@ def _request_body_json( next_page_token, self._paginator.get_request_body_json, self.request_option_provider.get_request_body_json, + **page_size_override_kwargs(page_size_override), ) if isinstance(body_json, str): raise ValueError("Request body json cannot be a string") @@ -298,6 +324,7 @@ def _next_page_token( last_page_size: int, last_record: Optional[Record], last_page_token_value: Optional[Any], + page_size_override: Optional[int] = None, ) -> Optional[Mapping[str, Any]]: """ Specifies a pagination strategy. @@ -311,12 +338,14 @@ def _next_page_token( last_page_size=last_page_size, last_record=last_record, last_page_token_value=last_page_token_value, + **page_size_override_kwargs(page_size_override), ) def _fetch_next_page( self, stream_slice: StreamSlice, next_page_token: Optional[Mapping[str, Any]] = None, + page_size_override: Optional[int] = None, ) -> Optional[requests.Response]: return self.requester.send_request( path=self._paginator_path( @@ -329,18 +358,22 @@ def _fetch_next_page( request_headers=self._request_headers( stream_slice=stream_slice, next_page_token=next_page_token, + **page_size_override_kwargs(page_size_override), ), request_params=self._request_params( stream_slice=stream_slice, next_page_token=next_page_token, + **page_size_override_kwargs(page_size_override), ), request_body_data=self._request_body_data( stream_slice=stream_slice, next_page_token=next_page_token, + **page_size_override_kwargs(page_size_override), ), request_body_json=self._request_body_json( stream_slice=stream_slice, next_page_token=next_page_token, + **page_size_override_kwargs(page_size_override), ), log_formatter=self.log_formatter, ) @@ -353,9 +386,20 @@ def _read_pages( ) -> Iterable[Record]: original_stream_slice = stream_slice pagination_tracker = self.pagination_tracker_factory() + page_size_reducer = ( + PageSizeReducer( + self.page_size_reduction, + self._paginator.get_page_size(), + stream_name=self.name, + ) + if self.page_size_reduction + else None + ) reset_pagination = False + reduce_page_size = False next_page_token = self._get_initial_next_page_token() while True: + page_size_override = page_size_reducer.page_size_override if page_size_reducer else None merged_records: MutableMapping[str, Any] = defaultdict(dict) last_page_size = 0 last_record: Optional[Record] = None @@ -371,7 +415,11 @@ def _read_pages( cursor_slice=stream_slice.cursor_slice or {}, extra_fields={"query_properties": properties}, ) - response = self._fetch_next_page(stream_slice, next_page_token) + response = self._fetch_next_page( + stream_slice, + next_page_token, + **page_size_override_kwargs(page_size_override), + ) for current_record in records_generator_fn(response): if self.additional_query_properties.property_chunking: @@ -401,7 +449,11 @@ def _read_pages( last_record = record yield record else: - response = self._fetch_next_page(stream_slice, next_page_token) + response = self._fetch_next_page( + stream_slice, + next_page_token, + **page_size_override_kwargs(page_size_override), + ) for current_record in records_generator_fn(response): pagination_tracker.observe(current_record) last_page_size += 1 @@ -409,10 +461,38 @@ def _read_pages( yield current_record except PaginationResetRequiredException: reset_pagination = True + except PageSizeReductionRequiredException: + if page_size_reducer is None: + # The action can be attached to a requester we cannot validate at config time, such as one + # built by a custom error handler or a CustomRequester. Re-raise as the misconfiguration it + # is: the exception being handled is the neutral "the API asked for a smaller page" signal. + raise PageSizeReductionNotSupportedException(stream_name=self.name) + if last_page_size: + # The reduction is safe only because it re-issues a page whose records were not emitted. + # Every in-CDK way of reaching this raises from `_fetch_next_page`, before the record loop, + # and the factory rejects the manifest constructs that would not, but a custom extractor, + # filter or transformation can issue its own request from inside the record generator. + # Re-issuing the page then duplicates the records already yielded, so fail instead. + raise AirbyteTracedException( + internal_message=f"Stream {self.name} requested a page size reduction after {last_page_size} records of the page had already been emitted", + message=f"Stream {self.name} asked for a smaller page size in the middle of a page. The page cannot be requested again without duplicating the records already read from it. Move the REDUCE_PAGE_SIZE action to the error handler of the stream's main requester.", + failure_type=FailureType.config_error, + ) + # Raises once the page size cannot be reduced any further, which is what stops the loop when + # the API keeps failing. + page_size_reducer.reduce() + reduce_page_size = True else: + if page_size_reducer: + page_size_reducer.on_successful_page() if not response: break + if reduce_page_size: + # Retry the very same page: neither the token nor the slice change, only the page size does. + reduce_page_size = False + continue + if reset_pagination or pagination_tracker.has_reached_limit(): next_page_token = self._get_initial_next_page_token() previous_slice = stream_slice @@ -432,6 +512,7 @@ def _read_pages( last_page_size=last_page_size, last_record=last_record, last_page_token_value=last_page_token_value, + **page_size_override_kwargs(page_size_override), ) if not next_page_token: break diff --git a/airbyte_cdk/sources/streams/http/error_handlers/response_models.py b/airbyte_cdk/sources/streams/http/error_handlers/response_models.py index 082d580d53..10416c1a7e 100644 --- a/airbyte_cdk/sources/streams/http/error_handlers/response_models.py +++ b/airbyte_cdk/sources/streams/http/error_handlers/response_models.py @@ -19,6 +19,7 @@ class ResponseAction(Enum): RESET_PAGINATION = "RESET_PAGINATION" RATE_LIMITED = "RATE_LIMITED" REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" + REDUCE_PAGE_SIZE = "REDUCE_PAGE_SIZE" @dataclass diff --git a/airbyte_cdk/sources/streams/http/http_client.py b/airbyte_cdk/sources/streams/http/http_client.py index c9008d1e64..825a59d4d2 100644 --- a/airbyte_cdk/sources/streams/http/http_client.py +++ b/airbyte_cdk/sources/streams/http/http_client.py @@ -42,6 +42,9 @@ RequestBodyException, UserDefinedBackoffException, ) +from airbyte_cdk.sources.streams.http.page_size_reduction_exception import ( + PageSizeReductionRequiredException, +) from airbyte_cdk.sources.streams.http.pagination_reset_exception import ( PaginationResetRequiredException, ) @@ -95,6 +98,22 @@ def monkey_patched_get_item(self, key): # type: ignore # this interface is a co requests_cache.SQLiteDict.__getitem__ = monkey_patched_get_item # type: ignore # see the method doc for more information +def _as_auxiliary_request_log(log_message: Any) -> Any: + """ + Flag an already-formatted request/response log as an auxiliary request. + + The Connector Builder builds one page per non-auxiliary HTTP log and bounds a slice by the number of those + pages, so a request that will not produce a page has to be marked here or it inflates that count. The log + formatter is connector-supplied and only the CDK's own one is guaranteed to have an `http` object, hence + the defensive check. + """ + if isinstance(log_message, dict): + http = log_message.get("http") + if isinstance(http, dict): + http["is_auxiliary"] = True + return log_message + + class HttpClient: _DEFAULT_MAX_RETRY: int = 5 _DEFAULT_MAX_TIME: int = 60 * 10 @@ -447,9 +466,16 @@ def _send( and self._message_repository is not None ): formatter = log_formatter + # A response resolving to REDUCE_PAGE_SIZE is not a page of the stream: the retriever discards it + # and re-issues the same page with a smaller page size. Logging it as an auxiliary request keeps it + # visible in the Connector Builder while keeping it out of the per-slice page count, which would + # otherwise report "limit reached" on a read that only retried. + log_as_auxiliary = error_resolution.response_action == ResponseAction.REDUCE_PAGE_SIZE self._message_repository.log_message( Level.DEBUG, - lambda: formatter(response), + lambda: _as_auxiliary_request_log(formatter(response)) + if log_as_auxiliary + else formatter(response), ) self._handle_error_resolution( @@ -510,6 +536,11 @@ def _handle_error_resolution( if error_resolution.response_action == ResponseAction.RESET_PAGINATION: raise PaginationResetRequiredException() + if error_resolution.response_action == ResponseAction.REDUCE_PAGE_SIZE: + raise PageSizeReductionRequiredException( + stream_name=self._name, error_message=error_resolution.error_message + ) + # Emit stream status RUNNING with the reason RATE_LIMITED to log that the rate limit has been reached if error_resolution.response_action == ResponseAction.RATE_LIMITED: # TODO: Update to handle with message repository when concurrent message repository is ready diff --git a/airbyte_cdk/sources/streams/http/page_size_reduction_exception.py b/airbyte_cdk/sources/streams/http/page_size_reduction_exception.py new file mode 100644 index 0000000000..13be064efb --- /dev/null +++ b/airbyte_cdk/sources/streams/http/page_size_reduction_exception.py @@ -0,0 +1,51 @@ +# +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. +# + +from typing import Optional + +from airbyte_cdk.models import FailureType +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + + +class PageSizeReductionRequiredException(AirbyteTracedException): + """ + Raised when an error handler resolves to `ResponseAction.REDUCE_PAGE_SIZE`. + + This is control flow rather than a failure: `SimpleRetriever._read_pages` catches it and re-issues the same + page with a smaller page size. It is raised on every reduction, including the ones a correctly configured + connector is expected to make, so the message describes what happened and nothing else - it must not read + as a bug report when it surfaces as the `__context__` of a later failure. + + A reduction the connector cannot honor raises `PageSizeReductionNotSupportedException` instead. + """ + + def __init__( + self, stream_name: Optional[str] = None, error_message: Optional[str] = None + ) -> None: + stream = f" of stream {stream_name}" if stream_name else "" + # The error handler's own `error_message` has no other outlet: the raise precedes every site that + # logs it, so an author who writes one on the reduction filter would otherwise never see it. + detail = f": {error_message}" if error_message else "" + super().__init__( + internal_message=f"An error handler{stream} resolved to REDUCE_PAGE_SIZE{detail}", + message=f"The API rejected a page{stream}. The connector is requesting the same page again with a smaller page size.", + failure_type=FailureType.transient_error, + ) + + +class PageSizeReductionNotSupportedException(AirbyteTracedException): + """ + Raised when a reduction is requested on a retriever that cannot honor it. + + The factory rejects this at config time wherever it can see the error handler, so reaching this means the + action came from somewhere it cannot inspect: a custom error handler, or a custom requester or retriever. + """ + + def __init__(self, stream_name: Optional[str] = None) -> None: + stream = f"Stream {stream_name}" if stream_name else "The stream" + super().__init__( + internal_message=f"An error handler of stream {stream_name} resolved to REDUCE_PAGE_SIZE but the retriever it is attached to defines no page_size_reduction. The action is only supported on the main requester of a SimpleRetriever that defines page_size_reduction.", + message=f"{stream} resolves an API response to the REDUCE_PAGE_SIZE action but is not set up to send a smaller page. Add `page_size_reduction` to the stream's retriever, or remove the REDUCE_PAGE_SIZE action from its error handler.", + failure_type=FailureType.config_error, + ) diff --git a/unit_tests/connector_builder/test_connector_builder_handler.py b/unit_tests/connector_builder/test_connector_builder_handler.py index 5842e86162..9223f77746 100644 --- a/unit_tests/connector_builder/test_connector_builder_handler.py +++ b/unit_tests/connector_builder/test_connector_builder_handler.py @@ -1919,3 +1919,128 @@ def test_full_resolve_manifest(valid_resolve_manifest_config_file): } assert resolved_manifest.record.data["manifest"] == expected_resolved_manifest assert resolved_manifest.record.stream == "full_resolve_manifest" + + +_PAGE_SIZE_REDUCTION_STREAM_NAME = "reducing_stream" +_PAGE_SIZE_REDUCTION_MANIFEST = { + "version": "0.30.3", + "type": "DeclarativeSource", + "check": {"type": "CheckStream", "stream_names": [_PAGE_SIZE_REDUCTION_STREAM_NAME]}, + "streams": [ + { + "type": "DeclarativeStream", + "name": _PAGE_SIZE_REDUCTION_STREAM_NAME, + "schema_loader": {"type": "InlineSchemaLoader", "schema": {"type": "object"}}, + "retriever": { + "type": "SimpleRetriever", + "page_size_reduction": {"type": "PageSizeReduction"}, + "requester": { + "type": "HttpRequester", + "url_base": "https://demonslayers.com/api/v1/", + "path": "hashiras", + "http_method": "GET", + "error_handler": { + "type": "DefaultErrorHandler", + "response_filters": [ + { + "type": "HttpResponseFilter", + "http_codes": [502], + "action": "REDUCE_PAGE_SIZE", + } + ], + }, + }, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": ["result"]}, + }, + "paginator": { + "type": "DefaultPaginator", + "page_size_option": { + "type": "RequestOption", + "inject_into": "request_parameter", + "field_name": "first", + }, + "page_token_option": {"type": "RequestPath"}, + "pagination_strategy": { + "type": "CursorPagination", + "page_size": 100, + "cursor_value": "{{ response._metadata.next }}", + "stop_condition": "{{ not response._metadata.next }}", + }, + }, + }, + } + ], + "spec": { + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": [], + "properties": {}, + }, + "documentation_url": "https://example.org", + "type": "Spec", + }, +} + + +def _create_502_page_response(): + response = requests.Response() + response.status_code = 502 + response._content = b"{}" + response.headers["Content-Type"] = "application/json" + response.request = _create_request() + return response + + +@patch( + "airbyte_cdk.sources.declarative.retrievers.page_size_reducer.PageSizeReducer.BACKOFF_SECONDS", + 0, +) +@patch.object( + requests.Session, + "send", + side_effect=( + _create_502_page_response(), + _create_page_response({"result": [{"id": 0}], "_metadata": {"next": "next"}}), + _create_page_response({"result": [{"id": 1}], "_metadata": {}}), + ), +) +def test_given_page_size_reduction_when_test_read_then_the_retry_does_not_count_as_a_page( + mock_http_stream, +): + """ + The Connector Builder bounds a slice by the number of request/response logs it sees, not by the paginator's + own counter. A response that only triggers a reduction is never a page of the stream, so counting it would + show empty error pages and report "limit reached" on a read that merely retried. + """ + # three responses are served but only two of them are pages, so the limit is not reached + limits = TestLimits(max_records=100, max_pages_per_slice=3, max_slices=2) + catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream( + name=_PAGE_SIZE_REDUCTION_STREAM_NAME, + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh], + ), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.append, + ) + ] + ) + config = {"__injected_declarative_manifest": _PAGE_SIZE_REDUCTION_MANIFEST} + source = create_source(config=config, limits=limits, catalog=catalog, state=None) + + output_data = read_stream(source, config, catalog, None, limits).record.data + + pages = output_data["slices"][0]["pages"] + assert [page["response"]["status"] for page in pages] == [200, 200] + assert [record["id"] for page in pages for record in page["records"]] == [0, 1] + assert output_data["test_read_limit_reached"] is False + # the failed attempt stays visible, just not as a page + assert [ + auxiliary_request["response"]["status"] + for auxiliary_request in output_data["auxiliary_requests"] + ] == [502] diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index 21c99adc71..2600ebf907 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -79,6 +79,7 @@ from airbyte_cdk.sources.declarative.models import DeclarativeStream as DeclarativeStreamModel from airbyte_cdk.sources.declarative.models import DefaultPaginator as DefaultPaginatorModel from airbyte_cdk.sources.declarative.models import DpathExtractor as DpathExtractorModel +from airbyte_cdk.sources.declarative.models import FileUploader as FileUploaderModel from airbyte_cdk.sources.declarative.models import ( GroupingPartitionRouter as GroupingPartitionRouterModel, ) @@ -88,6 +89,9 @@ from airbyte_cdk.sources.declarative.models import OAuthAuthenticator as OAuthAuthenticatorModel from airbyte_cdk.sources.declarative.models import PropertyChunking as PropertyChunkingModel from airbyte_cdk.sources.declarative.models import RecordSelector as RecordSelectorModel +from airbyte_cdk.sources.declarative.models import ( + SessionTokenAuthenticator as SessionTokenAuthenticatorModel, +) from airbyte_cdk.sources.declarative.models import SimpleRetriever as SimpleRetrieverModel from airbyte_cdk.sources.declarative.models import Spec as SpecModel from airbyte_cdk.sources.declarative.models import ( @@ -163,6 +167,9 @@ PageIncrement, StopConditionPaginationStrategyDecorator, ) +from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import ( + PaginationStrategy, +) from airbyte_cdk.sources.declarative.requesters.query_properties import ( PropertiesFromEndpoint, PropertyChunking, @@ -191,6 +198,10 @@ LazySimpleRetriever, SimpleRetriever, ) +from airbyte_cdk.sources.declarative.retrievers.page_size_reducer import ( + PageSizeReduction, + PageSizeResetPolicy, +) from airbyte_cdk.sources.declarative.schema import InlineSchemaLoader, JsonFileSchemaLoader from airbyte_cdk.sources.declarative.schema.caching_schema_loader_decorator import ( CachingSchemaLoaderDecorator, @@ -6424,6 +6435,804 @@ def get_schema_loader(stream: DefaultStream): return stream._stream_partition_generator._partition_factory._schema_loader._decorated +_PAGE_SIZE_REDUCTION_STREAM = """ +type: DeclarativeStream +name: Test +primary_key: id +schema_loader: + type: InlineSchemaLoader + schema: + type: object +retriever: + type: SimpleRetriever + {page_size_reduction} + requester: + type: HttpRequester + url_base: "https://airbyte.io" + path: "/graphql" + http_method: POST + error_handler: + type: DefaultErrorHandler + response_filters: + - type: HttpResponseFilter + http_codes: [502, 504] + action: {action} + paginator: + type: DefaultPaginator + pagination_strategy: + {pagination_strategy} + {page_size_option} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["items"] +""" + +_CURSOR_PAGINATION_STRATEGY = ( + 'type: CursorPagination\n page_size: 100\n cursor_value: "{{ response.next }}"' +) +_PAGE_SIZE_OPTION = """page_size_option: + type: RequestOption + inject_into: request_parameter + field_name: first""" + + +def _page_size_reduction_stream( + page_size_reduction="page_size_reduction:\n type: PageSizeReduction", + action="REDUCE_PAGE_SIZE", + pagination_strategy=_CURSOR_PAGINATION_STRATEGY, + page_size_option=_PAGE_SIZE_OPTION, +): + content = _PAGE_SIZE_REDUCTION_STREAM.format( + page_size_reduction=page_size_reduction, + action=action, + pagination_strategy=pagination_strategy, + page_size_option=page_size_option, + ) + stream_manifest = transformer.propagate_types_and_parameters( + "", resolver.preprocess_manifest(YamlDeclarativeSource._parse(content)), {} + ) + return factory.create_component( + model_type=DeclarativeStreamModel, component_definition=stream_manifest, config={} + ) + + +def test_given_page_size_reduction_then_create_retriever_with_defaults(): + retriever = get_retriever(_page_size_reduction_stream()) + + assert retriever.page_size_reduction == PageSizeReduction( + reduction_factor=2, + minimum_page_size=1, + max_attempts=5, + reset_policy=PageSizeResetPolicy.NEVER, + ) + + +def test_given_page_size_reduction_values_then_create_retriever_with_those_values(): + retriever = get_retriever( + _page_size_reduction_stream( + page_size_reduction=( + "page_size_reduction:\n" + " type: PageSizeReduction\n" + " reduction_factor: 4\n" + " minimum_page_size: 10\n" + " max_attempts: 2\n" + " reset_policy: AFTER_SUCCESSFUL_PAGE" + ) + ) + ) + + assert retriever.page_size_reduction == PageSizeReduction( + reduction_factor=4, + minimum_page_size=10, + max_attempts=2, + reset_policy=PageSizeResetPolicy.AFTER_SUCCESSFUL_PAGE, + ) + + +def test_given_no_page_size_reduction_then_retriever_has_none(): + retriever = get_retriever(_page_size_reduction_stream(page_size_reduction="", action="RETRY")) + + assert retriever.page_size_reduction is None + + +def test_given_reduce_page_size_action_without_page_size_reduction_then_raise(): + with pytest.raises(ValueError) as exception: + _page_size_reduction_stream(page_size_reduction="") + + assert "REDUCE_PAGE_SIZE" in str(exception.value) + + +def test_given_page_increment_and_page_size_reduction_then_raise(): + with pytest.raises(ValueError) as exception: + _page_size_reduction_stream(pagination_strategy="type: PageIncrement\n page_size: 100") + + assert "PageIncrement" in str(exception.value) + + +def test_given_no_page_size_option_and_page_size_reduction_then_raise(): + with pytest.raises(ValueError) as exception: + _page_size_reduction_stream(page_size_option="") + + assert "page_size_option" in str(exception.value) + + +class _StrategyHonoringOverride(PaginationStrategy): + """A custom strategy that can be told the reduced page size.""" + + @property + def initial_token(self): + return None + + def next_page_token( + self, + response, + last_page_size, + last_record, + last_page_token_value=None, + page_size_override=None, + ): + return None + + def get_page_size(self): + return 100 + + +class _StrategyIgnoringOverride(PaginationStrategy): + """A custom strategy predating the feature: it would raise TypeError on the first reduction.""" + + @property + def initial_token(self): + return None + + def next_page_token(self, response, last_page_size, last_record, last_page_token_value=None): + return None + + def get_page_size(self): + return 100 + + +def test_given_custom_pagination_strategy_accepting_the_override_and_page_size_reduction_then_create_retriever(): + """A custom strategy is written by whoever enables the reduction, so it is allowed as long + as it can receive the reduced page size. Rejecting every custom strategy would exclude the + GraphQL streams this feature exists for.""" + retriever = get_retriever( + _page_size_reduction_stream( + pagination_strategy=( + "type: CustomPaginationStrategy\n" + " class_name: unit_tests.sources.declarative.parsers.test_model_to_component_factory._StrategyHonoringOverride" + ) + ) + ) + + assert retriever.page_size_reduction is not None + + +def test_given_custom_pagination_strategy_ignoring_the_override_and_page_size_reduction_then_raise(): + with pytest.raises(ValueError, match="page_size_override"): + _page_size_reduction_stream( + pagination_strategy=( + "type: CustomPaginationStrategy\n" + " class_name: unit_tests.sources.declarative.parsers.test_model_to_component_factory._StrategyIgnoringOverride" + ) + ) + + +class _StrategyAcceptingKwargs(PaginationStrategy): + """A custom strategy that swallows the override through `**kwargs` rather than naming it.""" + + @property + def initial_token(self): + return None + + def next_page_token( + self, response, last_page_size, last_record, last_page_token_value=None, **kwargs + ): + return None + + def get_page_size(self): + return 100 + + +class _StrategySubclassingPageIncrement(PageIncrement): + """A custom strategy that inherits `next_page_token` - and therefore its rejection - from PageIncrement.""" + + +class _NotAPaginationStrategy: + """A class_name that builds but is not a pagination strategy: it defines no `next_page_token`.""" + + def __init__(self, **kwargs): + pass + + @property + def initial_token(self): + return None + + def get_page_size(self): + return 100 + + +def test_given_custom_pagination_strategy_accepting_kwargs_and_page_size_reduction_then_create_retriever(): + retriever = get_retriever( + _page_size_reduction_stream( + pagination_strategy=( + "type: CustomPaginationStrategy\n" + " class_name: unit_tests.sources.declarative.parsers.test_model_to_component_factory._StrategyAcceptingKwargs" + ) + ) + ) + + assert retriever.page_size_reduction is not None + + +def test_given_custom_pagination_strategy_subclassing_page_increment_then_raise(): + """ + `PageIncrement.next_page_token` declares `page_size_override` only to reject it, so a subclass that does + not override the method satisfies the signature check while being unable to honor a reduction. + """ + with pytest.raises(ValueError, match="PageIncrement"): + _page_size_reduction_stream( + pagination_strategy=( + "type: CustomPaginationStrategy\n" + " page_size: 100\n" + " class_name: unit_tests.sources.declarative.parsers.test_model_to_component_factory._StrategySubclassingPageIncrement" + ) + ) + + +def test_given_custom_pagination_strategy_without_next_page_token_then_raise_value_error(): + """Every other config-time failure in the factory is a ValueError; this one used to escape as a bare + AttributeError, which is reported as a system error rather than a manifest problem.""" + with pytest.raises(ValueError, match="next_page_token"): + _page_size_reduction_stream( + pagination_strategy=( + "type: CustomPaginationStrategy\n" + " class_name: unit_tests.sources.declarative.parsers.test_model_to_component_factory._NotAPaginationStrategy" + ) + ) + + +_CURSOR_PAGINATION_WITH_STOP_CONDITION = ( + 'type: CursorPagination\n page_size: 100\n cursor_value: "{{{{ response.next }}}}"\n' + ' stop_condition: "{condition}"' +) + + +def test_given_stop_condition_compares_last_page_size_to_a_constant_then_raise(): + """ + A full page at the reduced size satisfies `last_page_size < 100`, so the pagination would end early and + silently drop the rest of the partition. + """ + with pytest.raises(ValueError, match="last_page_size"): + _page_size_reduction_stream( + pagination_strategy=_CURSOR_PAGINATION_WITH_STOP_CONDITION.format( + condition="{{ last_page_size < 100 }}" + ) + ) + + +def test_given_stop_condition_compares_last_page_size_to_the_requested_page_size_then_create_retriever(): + retriever = get_retriever( + _page_size_reduction_stream( + pagination_strategy=_CURSOR_PAGINATION_WITH_STOP_CONDITION.format( + condition="{{ last_page_size < page_size }}" + ) + ) + ) + + assert retriever.page_size_reduction is not None + + +def test_given_stop_condition_does_not_mention_last_page_size_then_create_retriever(): + retriever = get_retriever( + _page_size_reduction_stream( + pagination_strategy=_CURSOR_PAGINATION_WITH_STOP_CONDITION.format( + condition="{{ not response.next }}" + ) + ) + ) + + assert retriever.page_size_reduction is not None + + +def test_given_no_paginator_and_page_size_reduction_then_raise(): + content = """ +type: DeclarativeStream +name: Test +primary_key: id +schema_loader: + type: InlineSchemaLoader + schema: + type: object +retriever: + type: SimpleRetriever + page_size_reduction: + type: PageSizeReduction + requester: + type: HttpRequester + url_base: "https://airbyte.io" + path: "/items" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["items"] +""" + stream_manifest = transformer.propagate_types_and_parameters( + "", resolver.preprocess_manifest(YamlDeclarativeSource._parse(content)), {} + ) + + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=DeclarativeStreamModel, component_definition=stream_manifest, config={} + ) + + assert "DefaultPaginator" in str(exception.value) + + +_OFFSET_INCREMENT_STRATEGY = "type: OffsetIncrement\n page_size: 100" + + +def test_given_offset_increment_and_page_size_reduction_then_create_retriever(): + """ + `OffsetIncrement` is the only strategy whose stop condition depends on the page size, so a typo in the + validator's isinstance tuple would reject every manifest this feature is meant to support. + """ + retriever = get_retriever( + _page_size_reduction_stream(pagination_strategy=_OFFSET_INCREMENT_STRATEGY) + ) + + assert retriever.page_size_reduction == PageSizeReduction() + + +def test_given_no_page_size_on_the_strategy_and_page_size_reduction_then_raise(): + """ + Without a `page_size` the paginator injects nothing, so the reduction is a dead end that would only + surface on the first failing response, after records have been emitted. + """ + with pytest.raises(ValueError) as exception: + _page_size_reduction_stream(pagination_strategy="type: OffsetIncrement") + + assert "page_size" in str(exception.value) + + +def test_given_minimum_page_size_not_below_the_page_size_then_raise(): + with pytest.raises(ValueError) as exception: + _page_size_reduction_stream( + page_size_reduction=( + "page_size_reduction:\n type: PageSizeReduction\n minimum_page_size: 100" + ) + ) + + assert "minimum_page_size" in str(exception.value) + + +def test_given_composite_error_handler_with_reduce_page_size_action_then_require_page_size_reduction(): + """The action can be nested in a CompositeErrorHandler, which the guard has to recurse into.""" + stream_definition = { + "type": "DeclarativeStream", + "name": "Test", + "primary_key": "id", + "schema_loader": {"type": "InlineSchemaLoader", "schema": {"type": "object"}}, + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/graphql", + "http_method": "POST", + "error_handler": { + "type": "CompositeErrorHandler", + "error_handlers": [ + { + "type": "DefaultErrorHandler", + "response_filters": [ + { + "type": "HttpResponseFilter", + "http_codes": [429], + "action": "RATE_LIMITED", + } + ], + }, + { + "type": "DefaultErrorHandler", + "response_filters": [ + { + "type": "HttpResponseFilter", + "http_codes": [502], + "action": "REDUCE_PAGE_SIZE", + } + ], + }, + ], + }, + }, + "paginator": { + "type": "DefaultPaginator", + "page_size_option": { + "type": "RequestOption", + "inject_into": "request_parameter", + "field_name": "first", + }, + "pagination_strategy": { + "type": "CursorPagination", + "page_size": 100, + "cursor_value": "{{ response.next }}", + }, + }, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": ["items"]}, + }, + }, + } + + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=DeclarativeStreamModel, + component_definition=stream_definition, + config={}, + ) + + assert "REDUCE_PAGE_SIZE" in str(exception.value) + + # and the same manifest with the block present builds + stream_definition["retriever"]["page_size_reduction"] = {"type": "PageSizeReduction"} + retriever = get_retriever( + factory.create_component( + model_type=DeclarativeStreamModel, + component_definition=stream_definition, + config={}, + ) + ) + assert retriever.page_size_reduction == PageSizeReduction() + + +def test_given_page_size_reduction_without_reduce_page_size_action_then_warn(caplog): + """ + A `CustomErrorHandler` can resolve to the action without being inspectable, so this cannot raise. It must + not stay silent either: the feature would be dead on a stream that only exists because it would fail. + """ + with caplog.at_level(logging.WARNING, logger="airbyte.model_to_component_factory"): + retriever = get_retriever(_page_size_reduction_stream(action="RETRY")) + + assert retriever.page_size_reduction == PageSizeReduction() + assert "REDUCE_PAGE_SIZE" in caplog.text + + +def test_given_query_properties_and_page_size_reduction_then_raise(): + """ + Records of the earlier property chunks were already emitted when a later chunk asks for a smaller page, so + re-issuing the page would emit them twice. + """ + content = _PAGE_SIZE_REDUCTION_STREAM.format( + page_size_reduction="page_size_reduction:\n type: PageSizeReduction", + action="REDUCE_PAGE_SIZE", + pagination_strategy=_CURSOR_PAGINATION_STRATEGY, + page_size_option=_PAGE_SIZE_OPTION, + ).replace( + " http_method: POST\n", + " http_method: POST\n" + " query_properties:\n" + " type: QueryProperties\n" + ' property_list: ["a", "b"]\n', + ) + stream_manifest = transformer.propagate_types_and_parameters( + "", resolver.preprocess_manifest(YamlDeclarativeSource._parse(content)), {} + ) + + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=DeclarativeStreamModel, component_definition=stream_manifest, config={} + ) + + assert "query properties" in str(exception.value) + + +def _lazy_read_stream_definition(page_size_reduction, action): + return { + "type": "DeclarativeStream", + "name": "items", + "primary_key": [], + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": {"type": "object", "properties": {}}, + }, + "retriever": { + "type": "SimpleRetriever", + **page_size_reduction, + "requester": { + "type": "HttpRequester", + "url_base": "https://api.test.com", + "path": "parent/{{ stream_partition.parent_id }}/items", + "http_method": "GET", + "error_handler": { + "type": "DefaultErrorHandler", + "response_filters": [ + {"type": "HttpResponseFilter", "http_codes": [502], "action": action} + ], + }, + }, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": ["data"]}, + }, + "paginator": { + "type": "DefaultPaginator", + "page_size_option": { + "type": "RequestOption", + "inject_into": "request_parameter", + "field_name": "first", + }, + "pagination_strategy": { + "type": "CursorPagination", + "page_size": 100, + "cursor_value": '{{ response["data"][-1]["id"] }}', + }, + }, + "partition_router": { + "type": "SubstreamPartitionRouter", + "parent_stream_configs": [ + { + "type": "ParentStreamConfig", + "parent_key": "id", + "partition_field": "parent_id", + "lazy_read_pointer": ["items"], + "stream": { + "type": "DeclarativeStream", + "name": "parent", + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": {"type": "object", "properties": {}}, + }, + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url_base": "https://api.test.com", + "path": "/parents", + "http_method": "GET", + }, + "record_selector": { + "type": "RecordSelector", + "extractor": { + "type": "DpathExtractor", + "field_path": ["data"], + }, + }, + }, + }, + } + ], + }, + }, + } + + +@pytest.mark.parametrize( + "page_size_reduction, action", + [ + pytest.param( + {"page_size_reduction": {"type": "PageSizeReduction"}}, + "RETRY", + id="page_size_reduction_without_the_action", + ), + pytest.param({}, "REDUCE_PAGE_SIZE", id="action_without_page_size_reduction"), + pytest.param( + {"page_size_reduction": {"type": "PageSizeReduction"}}, + "REDUCE_PAGE_SIZE", + id="both", + ), + ], +) +@pytest.mark.parametrize( + "has_state", + [pytest.param(False, id="first_sync"), pytest.param(True, id="resumed_sync")], +) +def test_given_lazy_read_pointer_and_page_size_reduction_then_raise( + page_size_reduction, action, has_state +): + """ + `LazySimpleRetriever` paginates the parent's embedded pages, so there is no page of its own to re-issue. + The rejection must not depend on the presence of state: gating it on the lazy branch, which only applies + while a stream has no state, would accept the same manifest from the second sync onwards. + """ + connector_state_manager = ConnectorStateManager( + state=[ + AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="items"), + stream_state=AirbyteStateBlob({"created": "2025-01-01T00:00:00+0000"}), + ), + ) + ] + if has_state + else [] + ) + + with pytest.raises(ValueError) as exception: + ModelToComponentFactory(connector_state_manager=connector_state_manager).create_component( + model_type=DeclarativeStreamModel, + component_definition=_lazy_read_stream_definition(page_size_reduction, action), + config=input_config, + ) + + assert "lazy_read_pointer" in str(exception.value) + + +def test_given_file_uploader_and_page_size_reduction_then_raise(): + """ + The file uploader sends one request per record from inside the page's record generator, so a reduction + asked for halfway through a page would re-emit the records the generator already yielded. + """ + content = ( + _PAGE_SIZE_REDUCTION_STREAM.format( + page_size_reduction="page_size_reduction:\n type: PageSizeReduction", + action="REDUCE_PAGE_SIZE", + pagination_strategy=_CURSOR_PAGINATION_STRATEGY, + page_size_option=_PAGE_SIZE_OPTION, + ) + + """file_uploader: + type: FileUploader + requester: + type: HttpRequester + url_base: "https://airbyte.io" + path: "/download" + download_target_extractor: + type: DpathExtractor + field_path: ["url"] +""" + ) + stream_manifest = transformer.propagate_types_and_parameters( + "", resolver.preprocess_manifest(YamlDeclarativeSource._parse(content)), {} + ) + + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=DeclarativeStreamModel, component_definition=stream_manifest, config={} + ) + + assert "file_uploader" in str(exception.value) + + +_REDUCE_PAGE_SIZE_ERROR_HANDLER = { + "type": "DefaultErrorHandler", + "response_filters": [ + {"type": "HttpResponseFilter", "http_codes": [502], "action": "REDUCE_PAGE_SIZE"} + ], +} + + +def test_given_reduce_page_size_action_on_the_file_uploader_requester_then_raise(): + """ + `error_handler` is defined on `HttpRequester`, so the action is manifest-legal on requesters that have no + page of their own. Nothing there can honor it, so it is rejected rather than raised mid-sync. + """ + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=FileUploaderModel, + component_definition={ + "type": "FileUploader", + "requester": { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/download", + "error_handler": _REDUCE_PAGE_SIZE_ERROR_HANDLER, + }, + "download_target_extractor": { + "type": "DpathExtractor", + "field_path": ["url"], + }, + }, + config={}, + ) + + assert "REDUCE_PAGE_SIZE" in str(exception.value) + + +def test_given_reduce_page_size_action_on_the_login_requester_then_raise(): + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=SessionTokenAuthenticatorModel, + component_definition={ + "type": "SessionTokenAuthenticator", + "login_requester": { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/login", + "http_method": "POST", + "error_handler": _REDUCE_PAGE_SIZE_ERROR_HANDLER, + }, + "session_token_path": ["token"], + "request_authentication": { + "type": "ApiKey", + "inject_into": { + "type": "RequestOption", + "inject_into": "header", + "field_name": "Authorization", + }, + }, + }, + config={}, + name="a_stream", + ) + + assert "REDUCE_PAGE_SIZE" in str(exception.value) + + +@pytest.mark.parametrize( + "requester_field", + [ + "creation_requester", + "polling_requester", + "download_requester", + "download_target_requester", + "abort_requester", + "delete_requester", + ], +) +def test_given_reduce_page_size_action_on_an_async_retriever_requester_then_raise(requester_field): + definition = { + "type": "AsyncRetriever", + "status_mapping": { + "type": "AsyncJobStatusMap", + "running": ["running"], + "completed": ["ready"], + "failed": ["failed"], + "timeout": ["timeout"], + }, + "status_extractor": {"type": "DpathExtractor", "field_path": ["status"]}, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": ["items"]}, + }, + "creation_requester": { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/jobs", + "http_method": "POST", + }, + "polling_requester": { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/jobs/{{ creation_response.id }}", + }, + "download_requester": { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/jobs/{{ creation_response.id }}/download", + }, + } + definition[requester_field] = { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/jobs", + "error_handler": _REDUCE_PAGE_SIZE_ERROR_HANDLER, + } + if requester_field == "download_target_requester": + # Without it the `download_target_extractor` guard fires first and the assertion below would pass for + # the wrong reason. + definition["download_target_extractor"] = { + "type": "DpathExtractor", + "field_path": ["url"], + } + + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=AsyncRetrieverModel, + component_definition=definition, + config={}, + name="a_stream", + primary_key=None, + stream_slicer=None, + transformations=[], + ) + + assert "REDUCE_PAGE_SIZE" in str(exception.value) + + def get_retriever(stream: Union[DeclarativeStream, DefaultStream]): return ( stream.retriever diff --git a/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py b/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py index 765b879de3..afc767e5ea 100644 --- a/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +++ b/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py @@ -349,3 +349,26 @@ def test_composite_error_handler_always_uses_first_strategy(): assert len(composite_handler.backoff_strategies) == 2 assert isinstance(composite_handler.backoff_strategies[0], ConstantBackoffStrategy) assert composite_handler.backoff_strategies[1], ConstantBackoffStrategy + + +def test_given_reduce_page_size_when_interpret_response_then_stop_at_the_matching_handler(): + """ + Without the short circuit, a later handler resolving to FAIL would win over the page size reduction. + """ + reducing_handler = MagicMock() + reducing_handler.interpret_response.return_value = ErrorResolution( + response_action=ResponseAction.REDUCE_PAGE_SIZE, + failure_type=FailureType.transient_error, + ) + failing_handler = MagicMock() + failing_handler.interpret_response.return_value = ErrorResolution( + response_action=ResponseAction.FAIL, failure_type=FailureType.system_error + ) + error_handler = CompositeErrorHandler( + error_handlers=[reducing_handler, failing_handler], parameters={} + ) + + error_resolution = error_handler.interpret_response(MagicMock()) + + assert error_resolution.response_action == ResponseAction.REDUCE_PAGE_SIZE + assert failing_handler.interpret_response.call_count == 0 diff --git a/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py b/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py index da21e1074c..79b8729439 100644 --- a/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +++ b/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py @@ -152,3 +152,60 @@ def test_interpolated_page_size_raises_on_non_integer(): config={"page_size": "invalid"}, parameters={}, ) + + +def test_given_page_size_override_then_token_is_unchanged(): + strategy = CursorPaginationStrategy( + page_size=100, cursor_value="{{ response.next }}", config={}, parameters={} + ) + response = requests.Response() + response._content = json.dumps({"next": "a token"}).encode("utf-8") + + assert ( + strategy.next_page_token(response, 50, None, None, page_size_override=50) + == strategy.next_page_token(response, 100, None, None) + == "a token" + ) + + +def test_given_stop_condition_uses_page_size_and_page_is_full_at_the_reduced_size_then_keep_paginating(): + """ + Regression test for the silent truncation a page size reduction used to cause: a full page at the reduced + size satisfies `last_page_size < 100` and would end the pagination, dropping the rest of the partition. + `page_size` holds the size that was actually requested, so the same condition keeps paginating. + """ + strategy = CursorPaginationStrategy( + page_size=100, + cursor_value="{{ response.next }}", + stop_condition="{{ last_page_size < page_size }}", + config={}, + parameters={}, + ) + response = requests.Response() + response._content = json.dumps({"next": "a token"}).encode("utf-8") + + assert strategy.next_page_token(response, 50, None, None, page_size_override=50) == "a token" + assert strategy.next_page_token(response, 100, None, None) == "a token" + + +def test_given_stop_condition_uses_page_size_and_page_is_short_then_stop(): + strategy = CursorPaginationStrategy( + page_size=100, + cursor_value="{{ response.next }}", + stop_condition="{{ last_page_size < page_size }}", + config={}, + parameters={}, + ) + response = requests.Response() + response._content = json.dumps({"next": "a token"}).encode("utf-8") + + assert strategy.next_page_token(response, 49, None, None, page_size_override=50) is None + assert strategy.next_page_token(response, 99, None, None) is None + + +def test_given_no_page_size_then_page_size_interpolates_to_none(): + strategy = CursorPaginationStrategy(cursor_value="{{ page_size }}", config={}, parameters={}) + response = requests.Response() + response._content = json.dumps({}).encode("utf-8") + + assert strategy.next_page_token(response, 10, None, None) is None diff --git a/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py b/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py index ee9f12b123..5254e69d3d 100644 --- a/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +++ b/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py @@ -539,3 +539,138 @@ def test_path_returns_none_when_option_not_request_path() -> None: ) result = paginator.path(next_page_token) assert result is None + + +def _paginator_with_page_size(page_size=100, inject_into=RequestOptionType.request_parameter): + return DefaultPaginator( + page_size_option=RequestOption( + inject_into=inject_into, field_name="page_size", parameters={} + ), + page_token_option=RequestOption( + inject_into=RequestOptionType.request_parameter, field_name="after", parameters={} + ), + pagination_strategy=CursorPaginationStrategy( + page_size=page_size, cursor_value="{{ response.next }}", config={}, parameters={} + ), + config={}, + url_base="https://airbyte.io", + parameters={}, + ) + + +def test_get_page_size_returns_the_strategy_page_size(): + assert _paginator_with_page_size().get_page_size() == 100 + + +def test_given_page_size_override_then_injected_instead_of_the_configured_page_size(): + paginator = _paginator_with_page_size() + + assert paginator.get_request_params(page_size_override=25) == {"page_size": 25} + assert paginator.get_request_params() == {"page_size": 100} + + +def test_given_page_size_override_when_injected_into_body_json_then_use_override(): + paginator = _paginator_with_page_size(inject_into=RequestOptionType.body_json) + + assert paginator.get_request_body_json(page_size_override=25) == {"page_size": 25} + assert paginator.get_request_headers(page_size_override=25) == {} + + +def test_given_page_size_override_when_next_page_token_then_forward_to_strategy(): + strategy = Mock() + strategy.next_page_token.return_value = "a token" + paginator = DefaultPaginator( + pagination_strategy=strategy, config={}, url_base="https://airbyte.io", parameters={} + ) + response = requests.Response() + + paginator.next_page_token(response, 25, None, None, page_size_override=25) + + assert strategy.next_page_token.call_args.kwargs["page_size_override"] == 25 + + +def test_given_no_page_size_override_when_next_page_token_then_strategy_called_without_the_argument(): + strategy = Mock() + strategy.next_page_token.return_value = "a token" + paginator = DefaultPaginator( + pagination_strategy=strategy, config={}, url_base="https://airbyte.io", parameters={} + ) + response = requests.Response() + + paginator.next_page_token(response, 25, None, None) + + assert "page_size_override" not in strategy.next_page_token.call_args.kwargs + + +def test_test_read_decorator_delegates_page_size_override(): + decorated = _paginator_with_page_size() + paginator = PaginatorTestReadDecorator(decorated, 5) + + assert paginator.get_page_size() == 100 + assert paginator.get_request_params(page_size_override=25) == {"page_size": 25} + assert paginator.get_request_params() == {"page_size": 100} + + +_TEST_READ_DECORATOR_REQUEST_OPTION_METHODS = [ + "get_request_params", + "get_request_headers", + "get_request_body_data", + "get_request_body_json", +] + + +@pytest.mark.parametrize("method_name", _TEST_READ_DECORATOR_REQUEST_OPTION_METHODS) +def test_given_page_size_override_when_test_read_decorator_request_options_then_forward_it( + method_name, +): + """ + This is the Connector Builder path: a method that dropped the override would make a Builder test read of a + reduced stream request the configured page size again, or stop after the first reduced page. + """ + decorated = Mock() + paginator = PaginatorTestReadDecorator(decorated, 5) + + getattr(paginator, method_name)(page_size_override=25) + + assert getattr(decorated, method_name).call_args.kwargs["page_size_override"] == 25 + + +@pytest.mark.parametrize("method_name", _TEST_READ_DECORATOR_REQUEST_OPTION_METHODS) +def test_given_no_page_size_override_when_test_read_decorator_request_options_then_omit_it( + method_name, +): + decorated = Mock() + paginator = PaginatorTestReadDecorator(decorated, 5) + + getattr(paginator, method_name)() + + assert "page_size_override" not in getattr(decorated, method_name).call_args.kwargs + + +def test_given_page_size_override_when_test_read_decorator_next_page_token_then_forward_it(): + decorated = Mock() + decorated.next_page_token.return_value = "a token" + paginator = PaginatorTestReadDecorator(decorated, 5) + response = requests.Response() + + assert paginator.next_page_token(response, 25, None, None, page_size_override=25) == "a token" + assert decorated.next_page_token.call_args.kwargs["page_size_override"] == 25 + + +def test_given_no_page_size_override_when_test_read_decorator_next_page_token_then_omit_it(): + decorated = Mock() + decorated.next_page_token.return_value = "a token" + paginator = PaginatorTestReadDecorator(decorated, 5) + response = requests.Response() + + paginator.next_page_token(response, 25, None, None) + + assert "page_size_override" not in decorated.next_page_token.call_args.kwargs + + +def test_test_read_decorator_delegates_get_page_size(): + decorated = Mock() + decorated.get_page_size.return_value = 100 + paginator = PaginatorTestReadDecorator(decorated, 5) + + assert paginator.get_page_size() == 100 diff --git a/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py b/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py index 28f6717f54..a12bd1e5a1 100644 --- a/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +++ b/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py @@ -146,3 +146,98 @@ def test_offset_increment_paginator_strategy_initial_token( ) assert paginator_strategy.initial_token == expected_initial_token + + +def _response(records): + response = requests.Response() + response._content = json.dumps({"results": records}).encode("utf-8") + return response + + +def test_given_page_size_override_when_page_is_full_for_the_override_then_keep_paginating(): + """ + A page that is full for the reduced page size is not the last page, even though it is smaller than the + configured page size. + """ + strategy = OffsetIncrement(page_size=100, extractor=None, config={}, parameters={}) + + next_page_token = strategy.next_page_token( + response=_response([{"id": index} for index in range(50)]), + last_page_size=50, + last_record=None, + last_page_token_value=0, + page_size_override=50, + ) + + assert next_page_token == 50 + + +def test_given_page_size_override_when_page_is_not_full_for_the_override_then_stop_paginating(): + strategy = OffsetIncrement(page_size=100, extractor=None, config={}, parameters={}) + + next_page_token = strategy.next_page_token( + response=_response([{"id": index} for index in range(30)]), + last_page_size=30, + last_record=None, + last_page_token_value=0, + page_size_override=50, + ) + + assert next_page_token is None + + +def test_given_page_size_override_then_offset_follows_the_records_actually_returned(): + strategy = OffsetIncrement(page_size=100, extractor=None, config={}, parameters={}) + + assert ( + strategy.next_page_token( + response=_response([{"id": index} for index in range(100)]), + last_page_size=100, + last_record=None, + last_page_token_value=0, + ) + == 100 + ) + assert ( + strategy.next_page_token( + response=_response([{"id": index} for index in range(50)]), + last_page_size=50, + last_record=None, + last_page_token_value=100, + page_size_override=50, + ) + == 150 + ) + + +def test_given_page_size_interpolates_to_empty_string_then_paginate_until_an_empty_page(): + """ + `page_size` interpolating to an empty string is the one behaviour change on the no-override path: the stop + condition used to compare the rendered value with `<`, which raised a TypeError for a string. It is now + treated as "no page size known", so pagination runs until a page comes back empty. + + This is only reachable when the paginator has no `page_size_option`, since `get_page_size` raises for a + non-integer page size before the request is built. + """ + strategy = OffsetIncrement( + page_size="{{ config['page_size'] }}", extractor=None, config={}, parameters={} + ) + + assert ( + strategy.next_page_token( + response=_response([{"id": index} for index in range(30)]), + last_page_size=30, + last_record=None, + last_page_token_value=0, + ) + == 30 + ) + assert ( + strategy.next_page_token( + response=_response([]), + last_page_size=0, + last_record=None, + last_page_token_value=30, + ) + is None + ) diff --git a/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py b/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py index ecb458b951..ef8339fe55 100644 --- a/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +++ b/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py @@ -8,10 +8,12 @@ import pytest import requests +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.extractors import DpathExtractor from airbyte_cdk.sources.declarative.requesters.paginators.strategies.page_increment import ( PageIncrement, ) +from airbyte_cdk.utils.traced_exception import AirbyteTracedException @pytest.mark.parametrize( @@ -160,3 +162,24 @@ def test_page_increment_paginator_strategy_initial_token( ) assert paginator_strategy.initial_token == expected_initial_token + + +def test_given_page_size_override_then_raise_config_error(): + """ + Reducing the page size would move every following page boundary, so PageIncrement refuses it. This is only + reachable when the factory is bypassed, so it has to report itself as a configuration error rather than + surfacing as a generic system error. + """ + strategy = PageIncrement(page_size=100, config={}, parameters={}, start_from_page=1) + response = requests.Response() + + with pytest.raises(AirbyteTracedException) as exception: + strategy.next_page_token( + response=response, + last_page_size=50, + last_record=None, + last_page_token_value=1, + page_size_override=50, + ) + + assert exception.value.failure_type == FailureType.config_error diff --git a/unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py b/unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py index b89baf4430..cb2ccc5987 100644 --- a/unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py +++ b/unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py @@ -116,3 +116,18 @@ def test_when_get_page_size_then_delegate(mocked_pagination_strategy, mocked_sto assert page_size == mocked_pagination_strategy.get_page_size.return_value mocked_pagination_strategy.get_page_size.assert_called_once_with() + + +def test_given_page_size_override_when_next_page_token_then_forward_to_delegate( + mocked_pagination_strategy, mocked_stop_condition +): + mocked_stop_condition.is_met.return_value = False + decorator = StopConditionPaginationStrategyDecorator( + mocked_pagination_strategy, mocked_stop_condition + ) + + decorator.next_page_token(ANY_RESPONSE, 25, NO_RECORD, None, page_size_override=25) + + mocked_pagination_strategy.next_page_token.assert_called_once_with( + ANY_RESPONSE, 25, NO_RECORD, None, page_size_override=25 + ) diff --git a/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py b/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py new file mode 100644 index 0000000000..2378db9e93 --- /dev/null +++ b/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py @@ -0,0 +1,237 @@ +# +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. +# + +import pytest + +from airbyte_cdk.models import FailureType +from airbyte_cdk.sources.declarative.retrievers.page_size_reducer import ( + PageSizeReducer, + PageSizeReduction, + PageSizeResetPolicy, +) +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + +A_STREAM_NAME = "stream_name" + + +def _reducer(configured_page_size=100, sleeps=None, **kwargs): + return PageSizeReducer( + PageSizeReduction(**kwargs), + configured_page_size, + stream_name=A_STREAM_NAME, + # The reducer waits before each retry; tests record the waits instead of taking them. + sleep=sleeps.append if sleeps is not None else lambda _: None, + ) + + +def test_given_no_reduction_when_page_size_override_then_return_none(): + assert _reducer().page_size_override is None + + +def test_when_reduce_then_halve_page_size(): + reducer = _reducer() + + reducer.reduce() + assert reducer.page_size_override == 50 + + reducer.reduce() + assert reducer.page_size_override == 25 + + reducer.reduce() + assert reducer.page_size_override == 12 + + +def test_given_reduction_factor_when_reduce_then_use_factor(): + reducer = _reducer(reduction_factor=4) + + reducer.reduce() + + assert reducer.page_size_override == 25 + + +def test_given_minimum_page_size_when_reduce_then_clamp_to_minimum(): + reducer = _reducer(configured_page_size=30, minimum_page_size=20) + + reducer.reduce() + + assert reducer.page_size_override == 20 + + +def test_given_minimum_page_size_above_configured_page_size_when_reduce_then_raise_config_error(): + """ + No reduction can ever be applied here, so nothing about the response can fix it and the platform must not + retry the whole job for it. + """ + reducer = _reducer(configured_page_size=50, minimum_page_size=100) + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.config_error + assert "already at or below the configured minimum" in exception.value.message + + +def test_given_page_size_cannot_be_reduced_when_reduce_then_raise_config_error(): + reducer = _reducer(configured_page_size=1) + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.config_error + + +def test_given_already_at_minimum_when_reduce_then_raise_transient_error(): + reducer = _reducer(configured_page_size=4, minimum_page_size=2) + reducer.reduce() + assert reducer.page_size_override == 2 + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.transient_error + + +def test_given_more_reductions_than_max_attempts_when_reduce_then_raise_transient_error(): + reducer = _reducer(configured_page_size=1000, max_attempts=2) + reducer.reduce() + reducer.reduce() + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.transient_error + assert reducer.page_size_override == 250 + + +def test_given_paginator_has_no_page_size_when_reduce_then_raise_config_error(): + reducer = _reducer(configured_page_size=None) + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.config_error + + +def test_given_reset_policy_never_when_on_successful_page_then_keep_reduced_page_size(): + reducer = _reducer() + reducer.reduce() + + reducer.on_successful_page() + + assert reducer.page_size_override == 50 + + +def test_given_reset_policy_after_successful_page_when_on_successful_page_then_restore_page_size(): + reducer = _reducer(reset_policy=PageSizeResetPolicy.AFTER_SUCCESSFUL_PAGE) + reducer.reduce() + + reducer.on_successful_page() + + assert reducer.page_size_override is None + + +def test_given_reset_policy_after_successful_page_when_on_successful_page_then_attempts_are_reset(): + """ + This policy exists for an API that rejects the configured page size on every page, so every page costs one + reduction. A budget spanning the whole partition would fail the sync at page `max_attempts + 1` however + healthy the reads are, which is the one workload the policy is for. + """ + reducer = _reducer(max_attempts=2, reset_policy=PageSizeResetPolicy.AFTER_SUCCESSFUL_PAGE) + + for _ in range(10): + reducer.reduce() + assert reducer.page_size_override == 50 + reducer.on_successful_page() + assert reducer.page_size_override is None + + +def test_given_reset_policy_after_successful_page_when_no_page_succeeds_then_max_attempts_still_applies(): + """The budget restarts on a successful page, not on a reduction, so an endpoint that fails whatever we ask + for still terminates.""" + reducer = _reducer(max_attempts=2, reset_policy=PageSizeResetPolicy.AFTER_SUCCESSFUL_PAGE) + reducer.reduce() + reducer.reduce() + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.transient_error + + +def test_given_reset_policy_never_when_pages_succeed_then_attempts_are_not_reset(): + reducer = _reducer(max_attempts=2) + reducer.reduce() + reducer.on_successful_page() + reducer.reduce() + reducer.on_successful_page() + + with pytest.raises(AirbyteTracedException): + reducer.reduce() + + +def test_given_reset_policy_after_successful_page_then_total_reductions_are_still_bounded(): + """`max_attempts` restarting on every successful page cannot be the only bound, or a partition could spend + reductions forever.""" + reducer = _reducer( + configured_page_size=1000, + max_attempts=2, + reset_policy=PageSizeResetPolicy.AFTER_SUCCESSFUL_PAGE, + ) + reducer.MAX_TOTAL_REDUCTIONS = 3 + + for _ in range(3): + reducer.reduce() + reducer.on_successful_page() + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.transient_error + + +@pytest.mark.parametrize( + "kwargs", + [ + pytest.param({"reduction_factor": 1}, id="reduction_factor_does_not_reduce"), + pytest.param({"minimum_page_size": 0}, id="minimum_page_size_is_not_positive"), + pytest.param({"max_attempts": 0}, id="max_attempts_is_not_positive"), + ], +) +def test_given_invalid_configuration_then_raise_value_error(kwargs): + with pytest.raises(ValueError): + PageSizeReduction(**kwargs) + + +def test_given_non_integer_page_size_when_reduce_then_raise_config_error(): + """A custom pagination strategy can return anything from `get_page_size`; reducing is + arithmetic, so a non-integer has to be reported rather than raising a bare TypeError.""" + reducer = PageSizeReducer( + PageSizeReduction(), + "{{ config['page_size'] }}", # type: ignore[arg-type] + stream_name="a_stream", + ) + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.config_error + assert "not a whole number" in exception.value.message + + +def test_when_reduce_then_wait_before_the_retry(): + """ + `PageSizeReductionRequiredException` bypasses the HTTP retry budget on purpose, so this wait is the only + thing keeping an endpoint that fails at every page size from being hit in a burst. + """ + sleeps: list = [] + reducer = _reducer(configured_page_size=1000, sleeps=sleeps) + + reducer.reduce() + reducer.reduce() + + assert sleeps == [ + PageSizeReducer.BACKOFF_SECONDS, + PageSizeReducer.BACKOFF_SECONDS * 2, + ] + assert all(wait > 0 for wait in sleeps) diff --git a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index e05c427acb..818f04642c 100644 --- a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -3,6 +3,9 @@ # import json +import threading +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor from functools import partial from typing import Any, Iterable, Mapping, Optional from unittest.mock import MagicMock, Mock, patch @@ -13,6 +16,7 @@ from airbyte_cdk.models import ( AirbyteLogMessage, AirbyteMessage, + FailureType, Level, SyncMode, Type, @@ -37,15 +41,27 @@ GroupByKey, PropertyLimitType, ) -from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType +from airbyte_cdk.sources.declarative.requesters.request_option import ( + RequestOption, + RequestOptionType, +) from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod, Requester +from airbyte_cdk.sources.declarative.retrievers.page_size_reducer import ( + PageSizeReducer, + PageSizeReduction, + PageSizeResetPolicy, +) from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever +from airbyte_cdk.sources.streams.http.page_size_reduction_exception import ( + PageSizeReductionRequiredException, +) from airbyte_cdk.sources.streams.http.pagination_reset_exception import ( PaginationResetRequiredException, ) from airbyte_cdk.sources.types import Record, StreamSlice from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from airbyte_cdk.utils.traced_exception import AirbyteTracedException A_RECORD_SCHEMA = {} A_SLICE_STATE = {"slice_state": "slice state value"} @@ -1426,6 +1442,418 @@ def test_given_reach_pagination_limit_after_two_pages_when_read_records_than_red } +@pytest.fixture(autouse=True) +def _no_page_size_reduction_backoff(monkeypatch): + """The reducer waits between reduction retries; taking those waits for real adds seconds to every CI run.""" + monkeypatch.setattr(PageSizeReducer, "BACKOFF_SECONDS", 0) + + +def _page_size_reduction_retriever( + requester: Requester, + paginator: Paginator, + record_selector: HttpSelector, + page_size_reduction: PageSizeReduction, +) -> SimpleRetriever: + return SimpleRetriever( + name=A_STREAM_NAME, + primary_key=primary_key, + requester=requester, + record_selector=record_selector, + paginator=paginator, + page_size_reduction=page_size_reduction, + parameters={}, + config={}, + ) + + +def test_given_page_size_reduction_when_read_records_then_retry_same_page_with_reduced_page_size(): + requester = Mock(spec=Requester) + requester.send_request.side_effect = [ + PageSizeReductionRequiredException(), + [{"id": 1}], + ] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.return_value = [{"id": 1}] + paginator = _mock_paginator() + paginator.get_page_size.return_value = 100 + paginator.get_initial_token.return_value = "a token" + paginator.next_page_token.return_value = None + + retriever = _page_size_reduction_retriever( + requester, paginator, record_selector, PageSizeReduction() + ) + + records = list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + assert records == [{"id": 1}] + assert requester.send_request.call_count == 2 + # the same page is requested again: only the page size changes + assert [call.kwargs["next_page_token"] for call in requester.send_request.call_args_list] == [ + {"next_page_token": "a token"}, + {"next_page_token": "a token"}, + ] + assert [call.kwargs["stream_slice"] for call in requester.send_request.call_args_list] == [ + A_STREAM_SLICE, + A_STREAM_SLICE, + ] + assert [ + call.kwargs.get("page_size_override") + for call in paginator.get_request_params.call_args_list + ] == [None, 50] + + +def test_given_page_size_reduction_when_read_records_then_next_page_token_not_computed_for_failed_page(): + requester = Mock(spec=Requester) + requester.send_request.side_effect = [ + PageSizeReductionRequiredException(), + [{"id": 1}], + ] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.return_value = [{"id": 1}] + paginator = _mock_paginator() + paginator.get_page_size.return_value = 100 + paginator.get_initial_token.return_value = None + paginator.next_page_token.return_value = None + + retriever = _page_size_reduction_retriever( + requester, paginator, record_selector, PageSizeReduction() + ) + + list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + assert paginator.next_page_token.call_count == 1 + assert paginator.next_page_token.call_args.kwargs["page_size_override"] == 50 + + +def test_given_reset_policy_never_when_page_succeeds_then_following_pages_stay_reduced(): + requester = Mock(spec=Requester) + requester.send_request.side_effect = [ + PageSizeReductionRequiredException(), + [{"id": 1}], + [{"id": 2}], + ] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.side_effect = [[{"id": 1}], [{"id": 2}]] + paginator = _mock_paginator() + paginator.get_page_size.return_value = 100 + paginator.get_initial_token.return_value = None + paginator.next_page_token.side_effect = [{"next_page_token": 2}, None] + + retriever = _page_size_reduction_retriever( + requester, paginator, record_selector, PageSizeReduction() + ) + + list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + assert [ + call.kwargs.get("page_size_override") + for call in paginator.get_request_params.call_args_list + ] == [None, 50, 50] + + +def test_given_reset_policy_after_successful_page_when_page_succeeds_then_page_size_restored(): + requester = Mock(spec=Requester) + requester.send_request.side_effect = [ + PageSizeReductionRequiredException(), + [{"id": 1}], + [{"id": 2}], + ] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.side_effect = [[{"id": 1}], [{"id": 2}]] + paginator = _mock_paginator() + paginator.get_page_size.return_value = 100 + paginator.get_initial_token.return_value = None + paginator.next_page_token.side_effect = [{"next_page_token": 2}, None] + + retriever = _page_size_reduction_retriever( + requester, + paginator, + record_selector, + PageSizeReduction(reset_policy=PageSizeResetPolicy.AFTER_SUCCESSFUL_PAGE), + ) + + list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + # the reduced page size applies to the retry only, the following page is back to the configured one + assert [ + call.kwargs.get("page_size_override") + for call in paginator.get_request_params.call_args_list + ] == [None, 50, None] + + +def test_given_reductions_exhausted_when_read_records_then_raise_transient_error(): + requester = Mock(spec=Requester) + requester.send_request.side_effect = PageSizeReductionRequiredException() + record_selector = Mock(spec=HttpSelector) + paginator = _mock_paginator() + paginator.get_page_size.return_value = 100 + paginator.get_initial_token.return_value = None + + retriever = _page_size_reduction_retriever( + requester, paginator, record_selector, PageSizeReduction(max_attempts=2) + ) + + with pytest.raises(AirbyteTracedException) as exception: + list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + assert exception.value.failure_type == FailureType.transient_error + assert requester.send_request.call_count == 3 + + +def test_given_no_page_size_reduction_when_reduce_page_size_required_then_raise_config_error(): + requester = Mock(spec=Requester) + requester.send_request.side_effect = PageSizeReductionRequiredException() + record_selector = Mock(spec=HttpSelector) + paginator = _mock_paginator() + paginator.get_initial_token.return_value = None + + retriever = SimpleRetriever( + name=A_STREAM_NAME, + primary_key=primary_key, + requester=requester, + record_selector=record_selector, + paginator=paginator, + parameters={}, + config={}, + ) + + with pytest.raises(AirbyteTracedException) as exception: + list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + assert exception.value.failure_type == FailureType.config_error + # the neutral "the API asked for a smaller page" message is replaced by the one describing the + # misconfiguration, which is what this branch actually means + assert "not set up to send a smaller page" in exception.value.message + + +def test_given_records_already_emitted_when_reduce_page_size_required_then_raise_instead_of_retrying(): + """ + Re-issuing a page is only safe while none of its records have been emitted. Every in-CDK path raises from + the fetch, but a custom extractor, filter or transformation can issue its own request from inside the + record generator, and retrying then would emit those records twice. + """ + requester = Mock(spec=Requester) + requester.send_request.return_value = [{"id": 1}] + paginator = _mock_paginator() + paginator.get_page_size.return_value = 100 + paginator.get_initial_token.return_value = None + + def select_records(**kwargs): + yield Record(data={"id": 1}, stream_name=A_STREAM_NAME) + raise PageSizeReductionRequiredException() + + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.side_effect = select_records + + retriever = _page_size_reduction_retriever( + requester, paginator, record_selector, PageSizeReduction() + ) + + with pytest.raises(AirbyteTracedException) as exception: + list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + assert exception.value.failure_type == FailureType.config_error + assert "middle of a page" in exception.value.message + assert requester.send_request.call_count == 1 + + +def test_given_page_size_reduction_and_pagination_limit_reached_when_read_records_then_reduce_before_resetting(): + """ + The reduction retry runs before the pagination limit check, so a page that failed defers the reset by one + iteration. That is correct - the failed page observed no record, so the limit it reports is stale - but the + two features never met in a test. + """ + requester = Mock(spec=Requester) + requester.send_request.side_effect = [ + PageSizeReductionRequiredException(), + [{"id": 1}], + [{"id": 2}], + ] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.side_effect = [[{"id": 1}], [{"id": 2}]] + pagination_tracker = Mock(spec=PaginationTracker) + pagination_tracker.has_reached_limit.side_effect = [True, False] + paginator = _mock_paginator() + paginator.get_page_size.return_value = 100 + paginator.get_initial_token.return_value = 1 + paginator.next_page_token.return_value = None + + retriever = SimpleRetriever( + name=A_STREAM_NAME, + primary_key=primary_key, + requester=requester, + record_selector=record_selector, + paginator=paginator, + pagination_tracker_factory=lambda: pagination_tracker, + page_size_reduction=PageSizeReduction(), + parameters={}, + config={}, + ) + + records = list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + assert len(records) == 2 + # the failed page is retried on the same slice with a smaller page, and only the page after it resets + assert pagination_tracker.has_reached_limit.call_count == 2 + assert pagination_tracker.reduce_slice_range_if_possible.call_count == 1 + assert [ + call.kwargs.get("page_size_override") + for call in paginator.get_request_params.call_args_list + ] == [None, 50, 50] + assert requester.send_request.call_args_list[1].kwargs["stream_slice"] == A_STREAM_SLICE + assert ( + requester.send_request.call_args_list[2].kwargs["stream_slice"] + == pagination_tracker.reduce_slice_range_if_possible.return_value + ) + + +def test_given_partitions_read_concurrently_when_one_reduces_then_others_keep_configured_page_size(): + """ + One retriever instance is shared by every partition of a stream, so the page size in effect must not leak + from one partition to another. + + The handshake has to make partition b read the page size in effect *after* partition a has reduced, which + means blocking b's first response until a is reduced and giving b a second page: the page size is read at + the top of the page loop, before the request is built, so a barrier inside the request building would come + too late and the assertion would hold even with a single shared reducer. + """ + slice_a = StreamSlice(cursor_slice={}, partition={"id": "a"}) + slice_b = StreamSlice(cursor_slice={}, partition={"id": "b"}) + partition_a_reduced = threading.Event() + requested_page_sizes = defaultdict(list) + + paginator = _mock_paginator() + paginator.get_page_size.return_value = 100 + paginator.get_initial_token.return_value = None + + def get_request_params(*, stream_slice, next_page_token, page_size_override=None): + requested_page_sizes[stream_slice.partition["id"]].append(page_size_override) + return {} + + paginator.get_request_params.side_effect = get_request_params + + def next_page_token(*, response, last_page_token_value, **kwargs): + # only partition b has a second page, which it requests once partition a is known to be reduced + if response[0]["id"] == "b" and last_page_token_value is None: + return {"next_page_token": "b page 2"} + return None + + paginator.next_page_token.side_effect = next_page_token + + def send_request(*args, **kwargs): + partition = kwargs["stream_slice"].partition["id"] + if partition == "a": + if len(requested_page_sizes["a"]) == 1: + raise PageSizeReductionRequiredException() + # the retry is in flight with the reduced page size + partition_a_reduced.set() + elif len(requested_page_sizes["b"]) == 1: + # hold partition b's first page open until partition a has reduced, so that b reads the page + # size in effect strictly after the reduction happened + assert partition_a_reduced.wait(timeout=10) + return [{"id": partition}] + + requester = Mock(spec=Requester) + requester.send_request.side_effect = send_request + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.side_effect = lambda **kwargs: [{"id": 1}] + + retriever = _page_size_reduction_retriever( + requester, paginator, record_selector, PageSizeReduction() + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [ + executor.submit(lambda s=s: list(retriever.read_records(A_RECORD_SCHEMA, s))) + for s in (slice_a, slice_b) + ] + for future in futures: + future.result() + + assert requested_page_sizes["a"] == [None, 50] + assert requested_page_sizes["b"] == [None, None] + + +def test_given_partitions_read_concurrently_then_each_read_owns_its_page_size_reducer(): + """ + Cheap and fully deterministic counterpart to the test above: two concurrent reads of the same retriever + must not share the object that holds the page size in effect. + """ + reducers = [] + original_init = PageSizeReducer.__init__ + + def record_reducer(self, *args, **kwargs): + original_init(self, *args, **kwargs) + reducers.append(self) + + requester = Mock(spec=Requester) + requester.send_request.return_value = [{"id": 1}] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.return_value = [{"id": 1}] + paginator = _mock_paginator() + paginator.get_page_size.return_value = 100 + paginator.get_initial_token.return_value = None + paginator.next_page_token.return_value = None + + retriever = _page_size_reduction_retriever( + requester, paginator, record_selector, PageSizeReduction() + ) + + with patch.object(PageSizeReducer, "__init__", record_reducer): + for partition in ("a", "b"): + list( + retriever.read_records( + A_RECORD_SCHEMA, StreamSlice(cursor_slice={}, partition={"id": partition}) + ) + ) + + assert len(reducers) == 2 + assert reducers[0] is not reducers[1] + + +def test_given_page_size_reduction_when_read_records_then_outgoing_request_carries_reduced_page_size(): + """ + The tests above assert that the retriever hands the reduced page size to the paginator. This one uses a + real DefaultPaginator so that a break anywhere between the retriever and `inject_into_request` is caught. + """ + response = requests.Response() + response.status_code = 200 + response._content = b"{}" + requester = Mock(spec=Requester) + requester.send_request.side_effect = [ + PageSizeReductionRequiredException(), + response, + ] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.return_value = [{"id": 1}] + paginator = DefaultPaginator( + page_size_option=RequestOption( + field_name="limit", inject_into=RequestOptionType.request_parameter, parameters={} + ), + page_token_option=RequestOption( + field_name="cursor", inject_into=RequestOptionType.request_parameter, parameters={} + ), + pagination_strategy=CursorPaginationStrategy( + page_size=100, cursor_value="{{ None }}", config={}, parameters={} + ), + config={}, + url_base="https://airbyte.io", + parameters={}, + ) + + retriever = _page_size_reduction_retriever( + requester, paginator, record_selector, PageSizeReduction() + ) + + records = list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + assert records == [{"id": 1}] + assert [call.kwargs["request_params"] for call in requester.send_request.call_args_list] == [ + {"limit": 100}, + {"limit": 50}, + ] + + def _mock_paginator(): paginator = Mock(spec=Paginator) paginator.get_request_params.__name__ = "get_request_params" diff --git a/unit_tests/sources/declarative/test_concurrent_declarative_source.py b/unit_tests/sources/declarative/test_concurrent_declarative_source.py index e2fef671c9..58db9e1c59 100644 --- a/unit_tests/sources/declarative/test_concurrent_declarative_source.py +++ b/unit_tests/sources/declarative/test_concurrent_declarative_source.py @@ -60,6 +60,7 @@ from airbyte_cdk.sources.declarative.resolvers.http_components_resolver import ( HttpComponentsResolver, ) +from airbyte_cdk.sources.declarative.retrievers.page_size_reducer import PageSizeReducer from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever from airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_generator import ( StreamSlicerPartitionGenerator, @@ -4981,6 +4982,189 @@ def test_given_response_action_is_pagination_reset_when_read_then_reset_paginati assert len(list(filter(lambda message: message.type == Type.RECORD, messages))) +def _page_size_reduction_manifest(pagination_strategy, page_token_option=None): + paginator = { + "type": "DefaultPaginator", + "pagination_strategy": pagination_strategy, + "page_size_option": { + "type": "RequestOption", + "inject_into": "request_parameter", + "field_name": "first", + }, + } + if page_token_option: + paginator["page_token_option"] = page_token_option + return { + "version": "0.34.2", + "type": "DeclarativeSource", + "check": {"type": "CheckStream", "stream_names": ["Test"]}, + "streams": [ + { + "type": "DeclarativeStream", + "name": "Test", + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": {"type": "object"}, + }, + "retriever": { + "type": "SimpleRetriever", + "page_size_reduction": {"type": "PageSizeReduction"}, + "requester": { + "type": "HttpRequester", + "url_base": "https://example.org", + "path": "/test", + "authenticator": {"type": "NoAuth"}, + "error_handler": { + "type": "DefaultErrorHandler", + "response_filters": [ + { + "type": "HttpResponseFilter", + "http_codes": [502], + # no `failure_type`: HttpResponseFilter only applies it to the FAIL + # action, and the failure the user sees comes from PageSizeReducer + "action": "REDUCE_PAGE_SIZE", + }, + ], + }, + }, + "paginator": paginator, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": ["items"]}, + }, + }, + } + ], + "spec": { + "type": "Spec", + "documentation_url": "https://example.org", + "connection_specification": {}, + }, + } + + +def _read_page_size_reduction_source(manifest): + catalog = create_catalog("Test") + source = ConcurrentDeclarativeSource( + source_config=manifest, + config={}, + catalog=catalog, + state=None, + ) + # the reducer waits before each reduction retry; taking those waits for real adds seconds to every CI run + with patch.object(PageSizeReducer, "BACKOFF_SECONDS", 0): + yield from source.read(logger=source.logger, config={}, catalog=catalog, state=[]) + + +def test_given_reduce_page_size_action_when_read_then_retry_page_with_smaller_page_size(): + """ + The call counts are asserted explicitly: the context-manager form of `HttpMocker` does not validate that + every matcher was called, so without them the test would also pass if the connector had started at the + reduced page size and never requested the configured one. + """ + manifest = _page_size_reduction_manifest( + { + "type": "CursorPagination", + "page_size": 100, + "cursor_value": "{{ response.next }}", + "stop_condition": "{{ not response.next }}", + } + ) + full_page_request = HttpRequest("https://example.org/test", query_params={"first": "100"}) + reduced_page_request = HttpRequest("https://example.org/test", query_params={"first": "50"}) + + with HttpMocker() as http_mocker: + http_mocker.get(full_page_request, HttpResponse("", 502)) + http_mocker.get(reduced_page_request, HttpResponse(json.dumps({"items": [{"id": 1}]}), 200)) + + messages = list(_read_page_size_reduction_source(manifest)) + + http_mocker.assert_number_of_calls(full_page_request, 1) + http_mocker.assert_number_of_calls(reduced_page_request, 1) + + assert [message.record.data["id"] for message in messages if message.type == Type.RECORD] == [1] + + +def test_given_offset_increment_and_reduce_page_size_action_when_read_then_keep_paginating(): + """ + `OffsetIncrement` is the only strategy whose stop condition depends on the page size. A page that is full + for the reduced size is smaller than the configured size, so comparing against the configured size would + end the pagination there and silently drop the tail of the partition. + """ + manifest = _page_size_reduction_manifest( + {"type": "OffsetIncrement", "page_size": 100}, + page_token_option={ + "type": "RequestOption", + "inject_into": "request_parameter", + "field_name": "offset", + }, + ) + full_page_request = HttpRequest("https://example.org/test", query_params={"first": "100"}) + first_reduced_page_request = HttpRequest( + "https://example.org/test", query_params={"first": "50"} + ) + second_reduced_page_request = HttpRequest( + "https://example.org/test", query_params={"first": "50", "offset": "50"} + ) + with HttpMocker() as http_mocker: + http_mocker.get(full_page_request, HttpResponse("", 502)) + http_mocker.get( + first_reduced_page_request, + HttpResponse(json.dumps({"items": [{"id": index} for index in range(50)]}), 200), + ) + http_mocker.get( + second_reduced_page_request, + HttpResponse(json.dumps({"items": [{"id": 50 + index} for index in range(20)]}), 200), + ) + + messages = list(_read_page_size_reduction_source(manifest)) + + http_mocker.assert_number_of_calls(full_page_request, 1) + http_mocker.assert_number_of_calls(first_reduced_page_request, 1) + http_mocker.assert_number_of_calls(second_reduced_page_request, 1) + + assert [ + message.record.data["id"] for message in messages if message.type == Type.RECORD + ] == list(range(70)) + + +def test_given_reductions_exhausted_when_read_then_emit_a_transient_error(): + """ + The failure type decides whether the platform retries the whole job, and an endpoint that refuses every + page size is the case the reduction budget exists for. + """ + manifest = _page_size_reduction_manifest( + { + "type": "CursorPagination", + "page_size": 100, + "cursor_value": "{{ response.next }}", + "stop_condition": "{{ not response.next }}", + } + ) + manifest["streams"][0]["retriever"]["page_size_reduction"]["max_attempts"] = 2 + + messages = [] + with HttpMocker() as http_mocker: + for page_size in ("100", "50", "25"): + http_mocker.get( + HttpRequest("https://example.org/test", query_params={"first": page_size}), + HttpResponse("", 502), + ) + + # the read fails, which is the point: the messages emitted before it are what the platform sees + with pytest.raises(AirbyteTracedException): + messages.extend(_read_page_size_reduction_source(manifest)) + + errors = [ + message.trace.error + for message in messages + if message.type == Type.TRACE and message.trace.type == TraceType.ERROR + ] + assert errors + assert all(error.failure_type == FailureType.transient_error for error in errors) + assert any("smaller and smaller pages" in error.message for error in errors) + + def test_given_pagination_limit_reached_when_read_then_reset_pagination(): input_config = {} manifest = { diff --git a/unit_tests/sources/streams/http/test_http_client.py b/unit_tests/sources/streams/http/test_http_client.py index 2c642ab9b1..23cd84a26e 100644 --- a/unit_tests/sources/streams/http/test_http_client.py +++ b/unit_tests/sources/streams/http/test_http_client.py @@ -1,5 +1,6 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. +import json import logging import os import time @@ -11,7 +12,9 @@ from pympler import asizeof from requests_cache import CachedRequest -from airbyte_cdk.models import FailureType +from airbyte_cdk.models import FailureType, Level +from airbyte_cdk.sources.http_logger import format_http_message +from airbyte_cdk.sources.message import InMemoryMessageRepository from airbyte_cdk.sources.streams.call_rate import CachedLimiterSession, LimiterSession from airbyte_cdk.sources.streams.http import HttpClient from airbyte_cdk.sources.streams.http.error_handlers import ( @@ -27,6 +30,9 @@ UserDefinedBackoffException, ) from airbyte_cdk.sources.streams.http.http_client import MessageRepresentationAirbyteTracedErrors +from airbyte_cdk.sources.streams.http.page_size_reduction_exception import ( + PageSizeReductionRequiredException, +) from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from airbyte_cdk.utils.traced_exception import AirbyteTracedException @@ -1441,3 +1447,110 @@ def test_deprecated_alias_is_catchable_as_airbyte_traced_exception(): internal_message="test", message="test user message", ) + + +def test_send_raises_page_size_reduction_required_exception_with_reduce_page_size_response_action(): + mocked_session = MagicMock(spec=requests.Session) + http_client = HttpClient( + name="test", + logger=MagicMock(), + error_handler=HttpStatusErrorHandler( + logger=MagicMock(), + error_mapping={ + 502: ErrorResolution( + ResponseAction.REDUCE_PAGE_SIZE, + FailureType.transient_error, + "test reduce page size message", + ) + }, + ), + session=mocked_session, + ) + mocked_response = requests.Response() + mocked_response.status_code = 502 + mocked_session.send.return_value = mocked_response + + # the retriever is responsible for retrying with a smaller page, so the backoff handlers must not retry + with pytest.raises(PageSizeReductionRequiredException) as exception: + http_client.send_request(http_method="get", url="https://airbyte.io", request_kwargs={}) + + assert http_client._session.send.call_count == 1 + assert "test" in exception.value.internal_message + # the exception is raised on every reduction, including the ones a correctly configured connector makes, + # so its message must describe the event rather than accuse the connector of a bug + assert "should be reported" not in exception.value.message + assert exception.value.message == ( + "The API rejected a page of stream test. The connector is requesting the same page again with a " + "smaller page size." + ) + + +def test_given_reduce_page_size_action_then_log_the_response_as_an_auxiliary_request(): + """ + The Connector Builder builds one page per non-auxiliary HTTP log and bounds a slice by the number of those + pages. A response resolving to REDUCE_PAGE_SIZE never becomes a page - the retriever re-issues it - so + counting it would report "limit reached" on a read that only retried. + """ + message_repository = InMemoryMessageRepository(Level.DEBUG) + mocked_session = MagicMock(spec=requests.Session) + http_client = HttpClient( + name="test", + logger=MagicMock(), + error_handler=HttpStatusErrorHandler( + logger=MagicMock(), + error_mapping={ + 502: ErrorResolution( + ResponseAction.REDUCE_PAGE_SIZE, + FailureType.transient_error, + "test reduce page size message", + ) + }, + ), + session=mocked_session, + message_repository=message_repository, + ) + mocked_response = requests.Response() + mocked_response.status_code = 502 + mocked_response.request = requests.Request(method="GET", url="https://airbyte.io").prepare() + mocked_session.send.return_value = mocked_response + + with pytest.raises(PageSizeReductionRequiredException): + http_client.send_request( + http_method="get", + url="https://airbyte.io", + request_kwargs={}, + log_formatter=lambda response: format_http_message( + response, "a title", "a description", "test" + ), + ) + + logged = [json.loads(message.log.message) for message in message_repository.consume_queue()] + assert [entry["http"]["is_auxiliary"] for entry in logged] == [True] + + +def test_given_no_reduce_page_size_action_then_log_the_response_as_a_page(): + message_repository = InMemoryMessageRepository(Level.DEBUG) + mocked_session = MagicMock(spec=requests.Session) + http_client = HttpClient( + name="test", + logger=MagicMock(), + error_handler=HttpStatusErrorHandler(logger=MagicMock()), + session=mocked_session, + message_repository=message_repository, + ) + mocked_response = requests.Response() + mocked_response.status_code = 200 + mocked_response.request = requests.Request(method="GET", url="https://airbyte.io").prepare() + mocked_session.send.return_value = mocked_response + + http_client.send_request( + http_method="get", + url="https://airbyte.io", + request_kwargs={}, + log_formatter=lambda response: format_http_message( + response, "a title", "a description", "test" + ), + ) + + logged = [json.loads(message.log.message) for message in message_repository.consume_queue()] + assert [entry["http"].get("is_auxiliary") for entry in logged] == [None] From 70ab89d66024e08333ebbf7d72b88b04800ffa4e Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Tue, 15 Sep 2026 21:08:45 +0300 Subject: [PATCH 02/13] fix: make the stop-condition gate correct in both directions and stop failing healthy long partitions Three round-2 review findings, two of them introduced by the round-1 fixes. The `stop_condition` gate matched the expression as a string, and a regex cannot do that job in either direction. `\bpage_size\b` matches inside `config['page_size']` because `'` is a non-word character, so `{{ last_page_size < config['page_size'] }}` passed the gate and still silently truncated - the exact bug the gate exists to prevent. And `{{ last_page_size == 0 }}` was rejected although no reduction can make a full page empty; that is 11 of the 14 `last_page_size` stop conditions in the monorepo, including all 6 in source-zendesk-support, which this PR names as an adopter. Patching the regex was not the fix: two bugs in one pattern meant the mechanism was wrong. The condition is now parsed as a Jinja expression and the comparisons `last_page_size` takes part in are classified on the AST, where a bare `page_size` Name is distinguishable from a Getitem on `config`. The rule is whether the condition can be true for a page that is full at the size that was requested: an emptiness test and a threshold at or below `minimum_page_size` cannot be, a lower bound can only stop being satisfied as the page shrinks, and an upper bound is safe only when it follows the reduction. A shape the analysis cannot classify is warned about rather than rejected, because this runs at stream construction and a false rejection takes `check` and `discover` down with `read`. Reverting to the round-1 regex fails 12 of the new factory tests, in both directions. `MAX_TOTAL_REDUCTIONS = 1000` guaranteed termination but moved the `AFTER_SUCCESSFUL_PAGE` cliff from page 6 to page 1001 instead of removing it, and the failure it produced was wrong three ways: `transient_error` on a job that could never succeed, "the source kept failing" on a partition where every page succeeded, and a page size that was restored rather than requested. A stream that gets every page through after one reduction is healthy and has to complete, so the cap is gone. `max_attempts` now bounds only the reductions made in a row *without* a page succeeding, which is what separates a stuck partition from an expensive one. Termination still holds: only a successful page restarts the budget and `_read_pages` calls that once per page it consumed, so every restart costs a page of progress. The schema's `max_attempts` and `reset_policy` descriptions say so, and two tests pin the healthy long stream and the stuck partition - the first one fails against the previous revision. Also: - `PageSizeReductionRequiredException` goes back to `config_error`. On the SimpleRetriever path it is always caught and the type is inert; the only way it escapes is a retriever that never re-issues the page, where a job-level retry would fail identically forever. The escape path is kept deliberately - it is the only signal such a stream gets - and the message no longer claims a retry that will not happen. - Two assertions passed vacuously. `"test" in internal_message` also matched the stream name, so it certified nothing about the error handler's `error_message` reaching the exception; it now asserts the mapping's text. `test_given_no_page_size_then_page_size_interpolates_to_none` held equally if `page_size` were never bound at all; it is now parametrized with two positive cases that pin the variable. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 2 + .../declarative_component_schema.yaml | 16 +- .../models/declarative_component_schema.py | 6 +- .../parsers/model_to_component_factory.py | 46 +++- .../parsers/stop_condition_safety.py | 220 ++++++++++++++++++ .../retrievers/page_size_reducer.py | 34 +-- .../http/page_size_reduction_exception.py | 11 +- .../test_model_to_component_factory.py | 96 ++++++++ .../parsers/test_stop_condition_safety.py | 165 +++++++++++++ .../test_cursor_pagination_strategy.py | 25 +- .../retrievers/test_page_size_reducer.py | 24 +- .../sources/streams/http/test_http_client.py | 11 +- 12 files changed, 599 insertions(+), 57 deletions(-) create mode 100644 airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py create mode 100644 unit_tests/sources/declarative/parsers/test_stop_condition_safety.py diff --git a/.gitignore b/.gitignore index 6644e574e2..ac4466df53 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,8 @@ node_modules # sqlite file that requests_cache creates on macOS for the `file::memory:?cache=shared` URI file::memory:?cache=shared unit_tests/sources/declarative/extractors/test_response.csv +# ResponseToFileExtractor names its temporary file after a uuid4 and writes it to the working directory +/????????-????-????-????-???????????? dist .ruff_cache diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 89ecaa52c2..4d1a176e3d 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -776,7 +776,9 @@ definitions: description: >- Template string evaluating when to stop paginating. Compare last_page_size against page_size rather than against a hardcoded number: page_size is the page size that was actually requested, so the condition stays - correct when page_size_reduction shrinks it. + correct when page_size_reduction shrinks it. Testing the page for emptiness with last_page_size == 0 is + equally safe. A stream that enables page_size_reduction is rejected when its stop condition compares + last_page_size against anything else, since a full page at a reduced size would then read as a short page. type: string interpolation_context: - config @@ -4338,11 +4340,12 @@ definitions: max_attempts: title: Maximum Reduction Attempts description: >- - Maximum number of consecutive page size reductions allowed before the sync fails with a transient error. - Every reduction follows a request that failed, so at most max_attempts + 1 failing requests are issued - before giving up. With reset_policy NEVER this bounds the reductions for the whole partition; with + Maximum number of page size reductions made in a row without a single page succeeding, before the sync + fails with a transient error. Every reduction follows a request that failed, so at most + max_attempts + 1 failing requests are issued before giving up. With reset_policy NEVER this bounds the + reductions for the whole partition, since the reduced page size is never restored; with AFTER_SUCCESSFUL_PAGE the budget restarts after every page that succeeds, so it bounds the reductions - needed to get a single page through. + needed to get a single page through and not the number of pages a partition may have. type: integer default: 5 minimum: 1 @@ -4356,6 +4359,9 @@ definitions: the rest of the partition. AFTER_SUCCESSFUL_PAGE restores it as soon as one page succeeds, which means hitting the same error again on every page - use it only when the reduction is worth one extra request per page, for instance because the configured page size usually works and only some pages are too heavy. + AFTER_SUCCESSFUL_PAGE also restarts the max_attempts budget on every page that succeeds, so there is no + limit on how many reductions a partition may make in total: a stream that needs one reduction per page + reads to the end however many pages it has. What is bounded is the reductions that get no page through. type: string enum: - NEVER diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 2f6b52defb..778c88673b 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -140,7 +140,7 @@ class CursorPagination(BaseModel): ) stop_condition: Optional[str] = Field( None, - description="Template string evaluating when to stop paginating. Compare last_page_size against page_size rather than against a hardcoded number: page_size is the page size that was actually requested, so the condition stays correct when page_size_reduction shrinks it.", + description="Template string evaluating when to stop paginating. Compare last_page_size against page_size rather than against a hardcoded number: page_size is the page size that was actually requested, so the condition stays correct when page_size_reduction shrinks it. Testing the page for emptiness with last_page_size == 0 is equally safe. A stream that enables page_size_reduction is rejected when its stop condition compares last_page_size against anything else, since a full page at a reduced size would then read as a short page.", examples=[ "{{ response.data.has_more is false }}", "{{ 'next' not in headers['link'] }}", @@ -1436,14 +1436,14 @@ class PageSizeReduction(BaseModel): ) max_attempts: Optional[int] = Field( 5, - description="Maximum number of consecutive page size reductions allowed before the sync fails with a transient error. Every reduction follows a request that failed, so at most max_attempts + 1 failing requests are issued before giving up. With reset_policy NEVER this bounds the reductions for the whole partition; with AFTER_SUCCESSFUL_PAGE the budget restarts after every page that succeeds, so it bounds the reductions needed to get a single page through.", + description="Maximum number of page size reductions made in a row without a single page succeeding, before the sync fails with a transient error. Every reduction follows a request that failed, so at most max_attempts + 1 failing requests are issued before giving up. With reset_policy NEVER this bounds the reductions for the whole partition, since the reduced page size is never restored; with AFTER_SUCCESSFUL_PAGE the budget restarts after every page that succeeds, so it bounds the reductions needed to get a single page through and not the number of pages a partition may have.", examples=[5, 10], ge=1, title="Maximum Reduction Attempts", ) reset_policy: Optional[ResetPolicy] = Field( ResetPolicy.NEVER, - description="When to restore the page size configured on the pagination strategy. NEVER keeps the reduced page size for the rest of the partition. AFTER_SUCCESSFUL_PAGE restores it as soon as one page succeeds, which means hitting the same error again on every page - use it only when the reduction is worth one extra request per page, for instance because the configured page size usually works and only some pages are too heavy.", + description="When to restore the page size configured on the pagination strategy. NEVER keeps the reduced page size for the rest of the partition. AFTER_SUCCESSFUL_PAGE restores it as soon as one page succeeds, which means hitting the same error again on every page - use it only when the reduction is worth one extra request per page, for instance because the configured page size usually works and only some pages are too heavy. AFTER_SUCCESSFUL_PAGE also restarts the max_attempts budget on every page that succeeds, so there is no limit on how many reductions a partition may make in total: a stream that needs one reduction per page reads to the end however many pages it has. What is bounded is the reductions that get no page through.", title="Reset Policy", ) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index b8d7ddf87b..8d5aacdc83 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -499,6 +499,10 @@ AirbyteCustomCodeNotPermittedError, custom_code_execution_permitted, ) +from airbyte_cdk.sources.declarative.parsers.stop_condition_safety import ( + StopConditionSafety, + classify_stop_condition, +) from airbyte_cdk.sources.declarative.partition_routers import ( CartesianProductStreamSlicer, GroupingPartitionRouter, @@ -3851,34 +3855,52 @@ def _validate_page_size_reduction_is_supported( # that only surfaces on the first failing response - after records have been emitted. self._validate_page_size_is_reducible(strategy, model.page_size_reduction, name) if isinstance(strategy, CursorPaginationModel): - self._validate_stop_condition_is_reduction_aware(strategy, name) + self._validate_stop_condition_is_reduction_aware( + strategy, model.page_size_reduction, name + ) @staticmethod def _validate_stop_condition_is_reduction_aware( - strategy: CursorPaginationModel, name: str + strategy: CursorPaginationModel, + page_size_reduction: Optional[PageSizeReductionModel], + name: str, ) -> None: """ A `stop_condition` comparing `last_page_size` to a hardcoded page size reads a full reduced page as a short page and ends the pagination early, dropping the rest of the partition without failing. The strategy exposes the page size that was actually requested as `page_size`, so the condition can be written correctly - but only if it is, which is what this checks. + + The condition is parsed as a Jinja expression rather than matched as a string: only the AST tells + `page_size`, which follows the reduction, apart from `config['page_size']`, which does not, and only the + AST tells an inequality, which a reduction can invalidate, apart from `last_page_size == 0`, which it + cannot. A shape the analysis does not understand is warned about rather than rejected - this runs at + stream construction, so a false rejection takes `check`, `discover` and `read` down with it. """ stop_condition = strategy.stop_condition if not stop_condition: return - # `\b` does not match between the `_` and the `p` of `last_page_size`, so the second pattern only - # matches a standalone `page_size` reference. - uses_last_page_size = re.search(r"\blast_page_size\b", stop_condition) - uses_requested_page_size = re.search(r"\bpage_size\b", stop_condition) - if uses_last_page_size and not uses_requested_page_size: + minimum_page_size = ( + page_size_reduction.minimum_page_size if page_size_reduction else None + ) or 1 + verdict, reason = classify_stop_condition(stop_condition, minimum_page_size) + if verdict is StopConditionSafety.TRUNCATES: raise ValueError( f"`page_size_reduction` on stream {name} cannot be used with the `stop_condition` " - f"{stop_condition!r}: it compares `last_page_size` to a value that does not follow the " - f"reduction, so a full page at the reduced size would read as a short page and end the " - f"pagination early, silently dropping the rest of the partition. Compare against the " - f"`page_size` interpolation variable instead, which holds the page size that was actually " - f"requested (for example `{{{{ last_page_size < page_size }}}}`)." + f"{stop_condition!r}: {reason}. The pagination would then end early, silently dropping the " + f"rest of the partition. Compare against the `page_size` interpolation variable instead, " + f"which holds the page size that was actually requested (for example " + f"`{{{{ last_page_size < page_size }}}}`), or test the page for emptiness with " + f"`{{{{ last_page_size == 0 }}}}`." + ) + if verdict is StopConditionSafety.UNKNOWN: + LOGGER.warning( + f"Stream {name} uses `page_size_reduction` with the `stop_condition` {stop_condition!r}, " + f"which could not be checked against the reduction because {reason}. Make sure a page that is " + f"full at a reduced page size does not satisfy it, otherwise the pagination ends early and the " + f"rest of the partition is silently dropped. Comparing against the `page_size` interpolation " + f"variable, which holds the page size that was actually requested, is always safe." ) @staticmethod diff --git a/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py b/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py new file mode 100644 index 0000000000..f270e3efde --- /dev/null +++ b/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py @@ -0,0 +1,220 @@ +# +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. +# + +""" +Config-time analysis of a `CursorPagination` `stop_condition` against a page size reduction. + +A condition that infers "this was the last page" from a short page is only correct relative to the page size +that was actually requested. When `page_size_reduction` shrinks that size, a *full* page at the reduced size +reads as a short page against the configured size, the pagination ends, and the rest of the partition is +dropped without anything failing. + +Matching the expression as a string cannot tell the reduction-aware `page_size` variable apart from a +config-derived constant - `\\bpage_size\\b` matches inside `config['page_size']` just as well - and it cannot +tell an inequality, which a reduction can invalidate, apart from `last_page_size == 0`, which no reduction can. +So the expression is parsed with Jinja and the comparisons involving `last_page_size` are inspected on the AST, +where a bare `Name` node is distinguishable from a `Getitem` on `config`. + +The safety rule is a single question: can the condition be true for a page that is *full at the size that was +requested*? A full page holds exactly `requested_page_size` records, and a reduction only ever lowers that +number, so: + +- `last_page_size == 0` and `last_page_size != 0` cannot change verdict: a full page is never empty, because + `minimum_page_size` is at least 1. +- `last_page_size > x` and `last_page_size >= x` can only stop being true as the page shrinks, so a reduction + cannot introduce a stop that would not have happened anyway. +- `last_page_size < x` and `last_page_size <= x` are the dangerous shape. They are safe only when `x` follows + the reduction (it references the `page_size` variable) or when `x` is a literal at or below + `minimum_page_size`, which makes the comparison a rewrite of "the page is empty". + +Anything else is reported as unclassifiable rather than as a truncation: this analysis rejects a manifest at +stream construction, so a shape it does not understand must not be treated as a defect. +""" + +from enum import Enum +from typing import Iterator, List, Tuple + +from jinja2 import nodes +from jinja2.environment import Environment +from jinja2.exceptions import TemplateSyntaxError + +LAST_PAGE_SIZE_VARIABLE = "last_page_size" +REQUESTED_PAGE_SIZE_VARIABLE = "page_size" + +# Parsing is all this environment is used for: no filter, test or global is resolved, so the plain environment +# parses everything the interpolation environment accepts. +_PARSING_ENVIRONMENT = Environment() + +_MIRRORED_OPERATORS = { + "lt": "gt", + "lteq": "gteq", + "gt": "lt", + "gteq": "lteq", + "eq": "eq", + "ne": "ne", +} + + +class StopConditionSafety(Enum): + """Whether a page size reduction can change what a `stop_condition` decides.""" + + SAFE = "SAFE" + """No reduction can make the condition stop the pagination earlier than it already would.""" + + TRUNCATES = "TRUNCATES" + """A full page at a reduced size satisfies the condition, so the partition would be cut short.""" + + UNKNOWN = "UNKNOWN" + """The condition uses `last_page_size` in a shape this analysis cannot reason about.""" + + +def classify_stop_condition( + stop_condition: str, minimum_page_size: int +) -> Tuple[StopConditionSafety, str]: + """ + :param stop_condition: the raw `stop_condition` template from the manifest + :param minimum_page_size: the smallest page size the reduction is allowed to request + :return: the verdict and a human readable reason for it + """ + try: + template = _PARSING_ENVIRONMENT.parse(stop_condition) + except TemplateSyntaxError as exception: + return ( + StopConditionSafety.UNKNOWN, + f"it is not a valid Jinja expression ({exception.message})", + ) + + if not _references(template, LAST_PAGE_SIZE_VARIABLE): + return ( + StopConditionSafety.SAFE, + f"it does not use `{LAST_PAGE_SIZE_VARIABLE}`, so the page size it was requested with is irrelevant", + ) + + unclassifiable: List[str] = [] + understood = 0 + for left, operator, right in _comparisons(template): + if not _references(left, LAST_PAGE_SIZE_VARIABLE) and not _references( + right, LAST_PAGE_SIZE_VARIABLE + ): + continue + verdict, reason = _classify_comparison(left, operator, right, minimum_page_size) + if verdict is StopConditionSafety.TRUNCATES: + return verdict, reason + if verdict is StopConditionSafety.UNKNOWN: + unclassifiable.append(reason) + else: + understood += 1 + + if unclassifiable: + return StopConditionSafety.UNKNOWN, unclassifiable[0] + if understood: + return ( + StopConditionSafety.SAFE, + f"every comparison it makes against `{LAST_PAGE_SIZE_VARIABLE}` holds whatever page size was requested", + ) + return ( + StopConditionSafety.UNKNOWN, + f"it uses `{LAST_PAGE_SIZE_VARIABLE}` outside of a comparison", + ) + + +def _comparisons(template: nodes.Template) -> Iterator[Tuple[nodes.Node, str, nodes.Node]]: + """Flatten every comparison, including the chained ones, into (left, operator, right) triples.""" + for comparison in template.find_all(nodes.Compare): + left = comparison.expr + for operand in comparison.ops: + yield left, operand.op, operand.expr + left = operand.expr + + +def _classify_comparison( + left: nodes.Node, operator: str, right: nodes.Node, minimum_page_size: int +) -> Tuple[StopConditionSafety, str]: + if _references(right, LAST_PAGE_SIZE_VARIABLE) and not _references( + left, LAST_PAGE_SIZE_VARIABLE + ): + left, right, operator = right, left, _MIRRORED_OPERATORS.get(operator, operator) + elif _references(left, LAST_PAGE_SIZE_VARIABLE) and _references(right, LAST_PAGE_SIZE_VARIABLE): + return ( + StopConditionSafety.UNKNOWN, + f"it compares `{LAST_PAGE_SIZE_VARIABLE}` against itself", + ) + + is_bare = isinstance(left, nodes.Name) and left.name == LAST_PAGE_SIZE_VARIABLE + + if operator in ("eq", "ne"): + if is_bare and isinstance(right, nodes.Const) and right.value == 0: + # A page that is full at the requested size holds at least `minimum_page_size` records, which is + # at least 1, so no reduction can make an emptiness test fire. + return ( + StopConditionSafety.SAFE, + f"`{LAST_PAGE_SIZE_VARIABLE}` is only tested for emptiness", + ) + return ( + StopConditionSafety.UNKNOWN, + f"it tests `{LAST_PAGE_SIZE_VARIABLE}` for equality against {_describe(right)} rather than against 0", + ) + + if operator in ("gt", "gteq"): + if is_bare: + # A reduction only lowers the size of a full page, so a lower bound can only stop being satisfied. + return ( + StopConditionSafety.SAFE, + f"a smaller page can only make `{LAST_PAGE_SIZE_VARIABLE} {'>' if operator == 'gt' else '>='} " + f"{_describe(right)}` less true, never more", + ) + return ( + StopConditionSafety.UNKNOWN, + f"`{LAST_PAGE_SIZE_VARIABLE}` is transformed before being compared", + ) + + if operator in ("lt", "lteq"): + if _references(right, REQUESTED_PAGE_SIZE_VARIABLE): + return ( + StopConditionSafety.SAFE, + f"it compares `{LAST_PAGE_SIZE_VARIABLE}` against `{REQUESTED_PAGE_SIZE_VARIABLE}`, " + f"which follows the reduction", + ) + if isinstance(right, nodes.Const) and _is_whole_number(right.value): + # `last_page_size < k` is false for every full page exactly when k is at or below the smallest page + # the connector is allowed to request, which makes it another way of writing "the page is empty". + threshold = right.value if operator == "lt" else right.value + 1 + if threshold <= minimum_page_size: + return ( + StopConditionSafety.SAFE, + f"no page the connector is allowed to request is smaller than {threshold}", + ) + return ( + StopConditionSafety.TRUNCATES, + f"it stops as soon as a page is smaller than {_describe(right)}, which does not follow the " + f"reduction, so a full page at a reduced size reads as a short page", + ) + + return ( + StopConditionSafety.UNKNOWN, + f"it uses `{LAST_PAGE_SIZE_VARIABLE}` with the `{operator}` operator", + ) + + +def _references(node: nodes.Node, name: str) -> bool: + if isinstance(node, nodes.Name): + return bool(node.name == name) + return any(referenced.name == name for referenced in node.find_all(nodes.Name)) + + +def _is_whole_number(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + + +def _describe(node: nodes.Node) -> str: + """Render the expression a `last_page_size` comparison is made against, for an error message.""" + if isinstance(node, nodes.Const): + return repr(node.value) + if isinstance(node, nodes.Name): + return f"`{node.name}`" + if isinstance(node, nodes.Getitem) and isinstance(node.arg, nodes.Const): + return f"`{_describe(node.node).strip('`')}[{node.arg.value!r}]`" + if isinstance(node, nodes.Getattr): + return f"`{_describe(node.node).strip('`')}.{node.attr}`" + return "a value the connector computes itself" diff --git a/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py b/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py index ecb8854ed9..872d1755e9 100644 --- a/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py +++ b/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py @@ -61,13 +61,6 @@ class PageSizeReducer: # that fails whatever page size we ask for degrades to a slow retry instead of a burst of requests. BACKOFF_SECONDS: float = 0.5 - # Backstop that bounds the reductions for the whole partition regardless of the reset policy. Under - # `AFTER_SUCCESSFUL_PAGE` the `max_attempts` budget restarts on every successful page, which is what lets a - # long partition complete when the API needs one reduction per page - so something else has to guarantee - # that the partition cannot spend reductions forever. It is deliberately far above any sane `max_attempts` - # because reaching it is a pathology, not a tuning problem, and it is therefore not exposed in the schema. - MAX_TOTAL_REDUCTIONS: int = 1000 - def __init__( self, config: PageSizeReduction, @@ -120,12 +113,13 @@ def reduce(self) -> None: self._attempts += 1 self._total_reductions += 1 - if ( - self._attempts > self._config.max_attempts - or self._total_reductions > self.MAX_TOTAL_REDUCTIONS - ): + if self._attempts > self._config.max_attempts: + # The budget counts the reductions that did *not* get a page through, which is what separates a + # partition that is stuck from one that is merely expensive. A partition where every page succeeds + # after a reduction resets this counter on each page and reads to the end, however many pages it + # has; a partition where nothing gets through burns the budget and fails here. raise AirbyteTracedException( - internal_message=f"Stream {self._stream_name} reduced its page size {self._total_reductions - 1} times while reading a single partition ({self._attempts - 1} of them since the last successful page), which is the maximum allowed", + internal_message=f"Stream {self._stream_name} reduced its page size {self._attempts - 1} times in a row without a single page succeeding, which is the configured maximum of {self._config.max_attempts} ({self._total_reductions - 1} reductions so far while reading this partition)", message=f"The source kept failing while the connector requested smaller and smaller pages (down to {current_page_size} records per page). The API is likely unable to serve these requests. Try syncing fewer streams at once, or contact the API provider.", failure_type=FailureType.transient_error, ) @@ -165,11 +159,17 @@ def on_successful_page(self) -> None: Under `NEVER` nothing happens: the reduced page size stays in effect and `max_attempts` keeps bounding the reductions for the whole partition, which is the right budget when reductions are one-off. - Under `AFTER_SUCCESSFUL_PAGE` the page size is restored and the `max_attempts` budget restarts. The - reduction count has to restart with it: this policy exists for an API that rejects the configured page - size on every page, so every page legitimately costs one reduction, and a budget spanning the whole - partition would fail the sync at page `max_attempts + 1` no matter how healthy the reads are. - `MAX_TOTAL_REDUCTIONS` still bounds the partition, so the sync cannot run forever. + Under `AFTER_SUCCESSFUL_PAGE` the page size is restored and the `max_attempts` budget restarts. This + policy exists for an API that rejects the configured page size on every page, so every page legitimately + costs one reduction, and a budget spanning the whole partition would fail the sync at page + `max_attempts + 1` no matter how healthy the reads are. There is deliberately no partition-wide cap on + top of it: a stream where every page gets through is healthy and has to sync to completion, and a cap + would only move the same cliff further out. + + The budget still terminates the read, because only a page that succeeded can restart it and only + `_read_pages` calls this, once per page it consumed. So between any two restarts the partition made one + page of progress, and the reductions that make no progress are bounded by `max_attempts`. Under `NEVER` + the reduced page size is also never restored, so `minimum_page_size` bounds the reductions on its own. """ if self._config.reset_policy != PageSizeResetPolicy.AFTER_SUCCESSFUL_PAGE: return diff --git a/airbyte_cdk/sources/streams/http/page_size_reduction_exception.py b/airbyte_cdk/sources/streams/http/page_size_reduction_exception.py index 13be064efb..451149eecd 100644 --- a/airbyte_cdk/sources/streams/http/page_size_reduction_exception.py +++ b/airbyte_cdk/sources/streams/http/page_size_reduction_exception.py @@ -17,7 +17,12 @@ class PageSizeReductionRequiredException(AirbyteTracedException): connector is expected to make, so the message describes what happened and nothing else - it must not read as a bug report when it surfaces as the `__context__` of a later failure. - A reduction the connector cannot honor raises `PageSizeReductionNotSupportedException` instead. + A reduction the connector cannot honor raises `PageSizeReductionNotSupportedException` instead, but only on + the `SimpleRetriever` path, which is the only one that knows a reduction was impossible. A `CustomRetriever`, + or a plain Python-CDK `HttpStream` whose error handler returns `ResponseAction.REDUCE_PAGE_SIZE`, never + catches this and lets it escape. That escape is kept deliberately - it is the only signal such a stream gets + - and it is typed `config_error` because nothing retries the page there, so a job-level retry would fail the + same way forever. On the `SimpleRetriever` path the type is inert, since the exception never leaves the loop. """ def __init__( @@ -29,8 +34,8 @@ def __init__( detail = f": {error_message}" if error_message else "" super().__init__( internal_message=f"An error handler{stream} resolved to REDUCE_PAGE_SIZE{detail}", - message=f"The API rejected a page{stream}. The connector is requesting the same page again with a smaller page size.", - failure_type=FailureType.transient_error, + message=f"The API rejected a page{stream} and asked the connector for a smaller one. If this message ends a sync, the stream is not set up to request a smaller page: add `page_size_reduction` to its retriever, or remove the REDUCE_PAGE_SIZE action from its error handler.", + failure_type=FailureType.config_error, ) diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index 2600ebf907..dc5bc092e1 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -6736,6 +6736,102 @@ def test_given_stop_condition_does_not_mention_last_page_size_then_create_retrie assert retriever.page_size_reduction is not None +@pytest.mark.parametrize( + "condition", + [ + # the three literal forms in the monorepo today: source-discord x3 + pytest.param("{{ last_page_size < 100 }}", id="literal_100"), + pytest.param("{{ last_page_size < 200 }}", id="literal_200"), + pytest.param("{{ last_page_size < 1000 }}", id="literal_1000"), + # the form the string-matching gate let through: `\bpage_size\b` matches inside `config['page_size']` + pytest.param("{{ last_page_size < config['page_size'] }}", id="config_reference"), + pytest.param("{{ last_page_size < config.page_size }}", id="config_attribute_reference"), + pytest.param("{{ last_page_size < parameters['page_size'] }}", id="parameters_reference"), + pytest.param("{{ last_page_size <= 99 }}", id="less_than_or_equal_to_a_literal"), + pytest.param("{{ 100 > last_page_size }}", id="reversed_operands"), + pytest.param( + "{{ last_page_size < 100 or not response.next }}", id="inside_a_larger_expression" + ), + pytest.param("{{ last_page_size | int < 100 }}", id="through_a_filter"), + ], +) +def test_given_stop_condition_compares_last_page_size_to_a_value_that_does_not_follow_the_reduction_then_raise( + condition, +): + """ + A full page at the reduced size satisfies each of these, so the pagination would end early and silently + drop the rest of the partition. `config['page_size']` is the one the previous string-matching gate + accepted: it contains the substring `page_size` but holds the configured size, not the requested one. + """ + with pytest.raises(ValueError, match="last_page_size"): + _page_size_reduction_stream( + pagination_strategy=_CURSOR_PAGINATION_WITH_STOP_CONDITION.format(condition=condition) + ) + + +@pytest.mark.parametrize( + "condition", + [ + # 11 of the 14 `last_page_size` stop conditions in the monorepo, including all 6 in + # source-zendesk-support and all 5 in source-trello. A reduction cannot make an empty page non-empty. + pytest.param("{{ last_page_size == 0 }}", id="emptiness_test"), + pytest.param("{{ 0 == last_page_size }}", id="emptiness_test_reversed"), + pytest.param( + "{{ last_page_size == 0 or not response.next }}", id="emptiness_test_or_no_cursor" + ), + # equivalent to emptiness, because `minimum_page_size` defaults to 1 + pytest.param("{{ last_page_size < 1 }}", id="less_than_the_minimum_page_size"), + pytest.param("{{ last_page_size <= 0 }}", id="at_most_zero"), + # the sanctioned form, and variations on it that still follow the reduction + pytest.param("{{ last_page_size < page_size }}", id="requested_page_size"), + pytest.param( + "{{ last_page_size < page_size | int }}", id="requested_page_size_through_a_filter" + ), + pytest.param("{{ page_size > last_page_size }}", id="requested_page_size_reversed"), + # a reduction only makes a page smaller, so a lower bound can only stop being satisfied + pytest.param("{{ last_page_size > 1000 }}", id="lower_bound"), + ], +) +def test_given_reduction_safe_stop_condition_then_create_retriever(condition): + retriever = get_retriever( + _page_size_reduction_stream( + pagination_strategy=_CURSOR_PAGINATION_WITH_STOP_CONDITION.format(condition=condition) + ) + ) + + assert retriever.page_size_reduction is not None + + +@pytest.mark.parametrize( + "condition", + [ + pytest.param( + "{{ last_page_size == config['page_size'] }}", id="equality_against_a_config_value" + ), + pytest.param("{{ last_page_size is lt(100) }}", id="jinja_test_rather_than_a_comparison"), + pytest.param("{{ last_page_size < }}", id="not_a_valid_jinja_expression"), + ], +) +def test_given_stop_condition_shape_cannot_be_classified_then_warn_and_create_retriever( + caplog, condition +): + """ + The gate runs at stream construction, so rejecting a manifest it merely does not understand would take + `check` and `discover` down with `read`. A shape outside the analysis warns instead. + """ + with caplog.at_level(logging.WARNING): + retriever = get_retriever( + _page_size_reduction_stream( + pagination_strategy=_CURSOR_PAGINATION_WITH_STOP_CONDITION.format( + condition=condition + ) + ) + ) + + assert retriever.page_size_reduction is not None + assert "could not be checked against the reduction" in caplog.text + + def test_given_no_paginator_and_page_size_reduction_then_raise(): content = """ type: DeclarativeStream diff --git a/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py b/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py new file mode 100644 index 0000000000..3ed34e1cd5 --- /dev/null +++ b/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py @@ -0,0 +1,165 @@ +# +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. +# + +import pytest + +from airbyte_cdk.sources.declarative.parsers.stop_condition_safety import ( + StopConditionSafety, + classify_stop_condition, +) + + +@pytest.mark.parametrize( + "stop_condition,minimum_page_size,expected", + [ + # The 11 monorepo occurrences of the emptiness test - source-zendesk-support x6, source-trello x5 - + # which the previous string-matching gate rejected. No reduction can make a full page empty. + pytest.param("{{ last_page_size == 0 }}", 1, StopConditionSafety.SAFE, id="emptiness_test"), + pytest.param( + "{{ 0 == last_page_size }}", 1, StopConditionSafety.SAFE, id="emptiness_test_reversed" + ), + pytest.param( + "{{ last_page_size != 0 }}", 1, StopConditionSafety.SAFE, id="non_emptiness_test" + ), + pytest.param( + "{{ last_page_size == 0 or not response.next }}", + 1, + StopConditionSafety.SAFE, + id="emptiness_test_in_a_larger_expression", + ), + # The 3 monorepo occurrences of the literal form - source-discord x3 - plus the equivalents. + pytest.param( + "{{ last_page_size < 100 }}", 1, StopConditionSafety.TRUNCATES, id="literal_100" + ), + pytest.param( + "{{ last_page_size < 200 }}", 1, StopConditionSafety.TRUNCATES, id="literal_200" + ), + pytest.param( + "{{ last_page_size < 1000 }}", 1, StopConditionSafety.TRUNCATES, id="literal_1000" + ), + # The false negative of the string-matching gate: `\bpage_size\b` matches inside `config['page_size']` + # because `'` is a non-word character, so the gate read a configured constant as reduction-aware. + pytest.param( + "{{ last_page_size < config['page_size'] }}", + 1, + StopConditionSafety.TRUNCATES, + id="config_getitem", + ), + pytest.param( + "{{ last_page_size < config.page_size }}", + 1, + StopConditionSafety.TRUNCATES, + id="config_getattr", + ), + pytest.param( + "{{ last_page_size < parameters['page_size'] }}", + 1, + StopConditionSafety.TRUNCATES, + id="parameters_getitem", + ), + # The sanctioned form and the shapes that still follow the reduction. + pytest.param( + "{{ last_page_size < page_size }}", + 1, + StopConditionSafety.SAFE, + id="requested_page_size", + ), + pytest.param( + "{{ last_page_size < page_size | int }}", + 1, + StopConditionSafety.SAFE, + id="requested_page_size_through_a_filter", + ), + pytest.param( + "{{ page_size > last_page_size }}", + 1, + StopConditionSafety.SAFE, + id="requested_page_size_reversed", + ), + # A threshold at or below the smallest page the connector may request is a rewrite of the emptiness + # test, so it is safe - and it stops being safe as soon as the minimum rises above it. + pytest.param("{{ last_page_size < 1 }}", 1, StopConditionSafety.SAFE, id="below_minimum"), + pytest.param("{{ last_page_size <= 0 }}", 1, StopConditionSafety.SAFE, id="at_most_zero"), + pytest.param( + "{{ last_page_size < 10 }}", 10, StopConditionSafety.SAFE, id="at_a_raised_minimum" + ), + pytest.param( + "{{ last_page_size < 11 }}", + 10, + StopConditionSafety.TRUNCATES, + id="above_a_raised_minimum", + ), + # A reduction only makes a full page smaller, so a lower bound can only stop being satisfied. + pytest.param("{{ last_page_size > 1000 }}", 1, StopConditionSafety.SAFE, id="lower_bound"), + pytest.param( + "{{ last_page_size >= 1000 }}", 1, StopConditionSafety.SAFE, id="inclusive_lower_bound" + ), + # A condition that never looks at the page size is unaffected by the reduction. + pytest.param( + "{{ not response.next }}", 1, StopConditionSafety.SAFE, id="no_last_page_size" + ), + # Shapes the analysis cannot reason about are reported as unknown so the caller can warn rather than + # reject: this runs at stream construction, where a false rejection also breaks `check` and `discover`. + pytest.param( + "{{ last_page_size == config['page_size'] }}", + 1, + StopConditionSafety.UNKNOWN, + id="equality_against_a_config_value", + ), + pytest.param( + "{{ last_page_size is lt(100) }}", + 1, + StopConditionSafety.UNKNOWN, + id="jinja_test_rather_than_a_comparison", + ), + pytest.param( + "{{ last_page_size }}", 1, StopConditionSafety.UNKNOWN, id="no_comparison_at_all" + ), + pytest.param( + "{{ last_page_size < }}", + 1, + StopConditionSafety.UNKNOWN, + id="not_a_valid_jinja_expression", + ), + pytest.param( + "{{ last_page_size > last_page_size }}", + 1, + StopConditionSafety.UNKNOWN, + id="compared_against_itself", + ), + # A transform makes the equality and the lower bound unreadable, but a shrinking upper bound is still + # a truncation whatever `last_page_size` was piped through. + pytest.param( + "{{ last_page_size | int > 100 }}", + 1, + StopConditionSafety.UNKNOWN, + id="lower_bound_through_a_filter", + ), + pytest.param( + "{{ last_page_size | int < 100 }}", + 1, + StopConditionSafety.TRUNCATES, + id="upper_bound_through_a_filter", + ), + ], +) +def test_classify_stop_condition(stop_condition, minimum_page_size, expected): + verdict, reason = classify_stop_condition(stop_condition, minimum_page_size) + + assert verdict is expected + assert reason + + +def test_given_one_truncating_comparison_among_several_then_truncates(): + verdict, _ = classify_stop_condition( + "{{ last_page_size == 0 or last_page_size < config['page_size'] }}", 1 + ) + + assert verdict is StopConditionSafety.TRUNCATES + + +def test_reason_names_the_value_the_page_size_is_compared_against(): + _, reason = classify_stop_condition("{{ last_page_size < config['page_size'] }}", 1) + + assert "config['page_size']" in reason diff --git a/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py b/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py index 79b8729439..7527e0c736 100644 --- a/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +++ b/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py @@ -203,9 +203,28 @@ def test_given_stop_condition_uses_page_size_and_page_is_short_then_stop(): assert strategy.next_page_token(response, 99, None, None) is None -def test_given_no_page_size_then_page_size_interpolates_to_none(): - strategy = CursorPaginationStrategy(cursor_value="{{ page_size }}", config={}, parameters={}) +@pytest.mark.parametrize( + "page_size,page_size_override,expected_token", + [ + pytest.param(100, None, 100, id="test_configured_page_size_is_bound"), + pytest.param(100, 50, 50, id="test_reduced_page_size_is_bound"), + pytest.param(None, None, None, id="test_no_page_size_interpolates_to_none"), + ], +) +def test_page_size_is_bound_in_the_cursor_value_interpolation_context( + page_size, page_size_override, expected_token +): + """ + `page_size` has to resolve to the size that was actually requested. The None case alone would also pass if + the variable were never bound at all, so the two positive cases are what pin it. + """ + strategy = CursorPaginationStrategy( + page_size=page_size, cursor_value="{{ page_size }}", config={}, parameters={} + ) response = requests.Response() response._content = json.dumps({}).encode("utf-8") - assert strategy.next_page_token(response, 10, None, None) is None + assert ( + strategy.next_page_token(response, 10, None, None, page_size_override=page_size_override) + == expected_token + ) diff --git a/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py b/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py index 2378db9e93..5db30fedeb 100644 --- a/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py +++ b/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py @@ -148,7 +148,7 @@ def test_given_reset_policy_after_successful_page_when_on_successful_page_then_a def test_given_reset_policy_after_successful_page_when_no_page_succeeds_then_max_attempts_still_applies(): """The budget restarts on a successful page, not on a reduction, so an endpoint that fails whatever we ask - for still terminates.""" + for still terminates. This is the genuinely-stuck case: nothing got through, so the read has to end.""" reducer = _reducer(max_attempts=2, reset_policy=PageSizeResetPolicy.AFTER_SUCCESSFUL_PAGE) reducer.reduce() reducer.reduce() @@ -157,6 +157,9 @@ def test_given_reset_policy_after_successful_page_when_no_page_succeeds_then_max reducer.reduce() assert exception.value.failure_type == FailureType.transient_error + # the size named is the one that was just requested and failed, not the configured one + assert "down to 25 records per page" in exception.value.message + assert "2 times in a row without a single page succeeding" in exception.value.internal_message def test_given_reset_policy_never_when_pages_succeed_then_attempts_are_not_reset(): @@ -170,24 +173,23 @@ def test_given_reset_policy_never_when_pages_succeed_then_attempts_are_not_reset reducer.reduce() -def test_given_reset_policy_after_successful_page_then_total_reductions_are_still_bounded(): - """`max_attempts` restarting on every successful page cannot be the only bound, or a partition could spend - reductions forever.""" +def test_given_reset_policy_after_successful_page_when_every_page_succeeds_then_never_fail(): + """ + A partition where every page gets through after one reduction is healthy, however long it is: this policy + exists for an API that rejects the configured page size on every page. There is no partition-wide cap on + the number of reductions, because any such cap would fail this stream at the page it happens to sit on. + """ reducer = _reducer( configured_page_size=1000, max_attempts=2, reset_policy=PageSizeResetPolicy.AFTER_SUCCESSFUL_PAGE, ) - reducer.MAX_TOTAL_REDUCTIONS = 3 - for _ in range(3): + for _ in range(5_000): reducer.reduce() + assert reducer.page_size_override == 500 reducer.on_successful_page() - - with pytest.raises(AirbyteTracedException) as exception: - reducer.reduce() - - assert exception.value.failure_type == FailureType.transient_error + assert reducer.page_size_override is None @pytest.mark.parametrize( diff --git a/unit_tests/sources/streams/http/test_http_client.py b/unit_tests/sources/streams/http/test_http_client.py index 23cd84a26e..bf2832093b 100644 --- a/unit_tests/sources/streams/http/test_http_client.py +++ b/unit_tests/sources/streams/http/test_http_client.py @@ -1475,14 +1475,19 @@ def test_send_raises_page_size_reduction_required_exception_with_reduce_page_siz http_client.send_request(http_method="get", url="https://airbyte.io", request_kwargs={}) assert http_client._session.send.call_count == 1 - assert "test" in exception.value.internal_message + # the error handler's own error_message has no other outlet, so it must reach the internal message. The + # stream is also called "test", so this asserts the mapping's text rather than any occurrence of "test". + assert "test reduce page size message" in exception.value.internal_message # the exception is raised on every reduction, including the ones a correctly configured connector makes, # so its message must describe the event rather than accuse the connector of a bug assert "should be reported" not in exception.value.message assert exception.value.message == ( - "The API rejected a page of stream test. The connector is requesting the same page again with a " - "smaller page size." + "The API rejected a page of stream test and asked the connector for a smaller one. If this message " + "ends a sync, the stream is not set up to request a smaller page: add `page_size_reduction` to its " + "retriever, or remove the REDUCE_PAGE_SIZE action from its error handler." ) + # a retriever that cannot re-issue the page never retries it, so a job-level retry cannot help + assert exception.value.failure_type == FailureType.config_error def test_given_reduce_page_size_action_then_log_the_response_as_an_auxiliary_request(): From 09882ecf035fa45c75ba2ddc5851aabd11652939 Mon Sep 17 00:00:00 2001 From: "octavia-bot[bot]" <108746235+octavia-bot[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 18:23:09 +0000 Subject: [PATCH 03/13] Auto-committed changes from Poe command `build` --- .../models/declarative_component_schema.py | 524 +++++++++--------- 1 file changed, 259 insertions(+), 265 deletions(-) diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 778c88673b..e34dcc5a55 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -6,7 +6,7 @@ from enum import Enum from typing import Any, Dict, List, Literal, Optional, Union -from pydantic.v1 import BaseModel, Extra, Field +from pydantic.v1 import BaseModel, Extra, Field, confloat, conint from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( BaseModelWithDeprecations, @@ -18,12 +18,6 @@ class AuthFlowType(Enum): oauth1_0 = "oauth1.0" -class ScopesJoinStrategy(Enum): - space = "space" - comma = "comma" - plus = "plus" - - class BasicHttpAuthenticator(BaseModel): type: Literal["BasicHttpAuthenticator"] username: str = Field( @@ -52,15 +46,50 @@ class BearerAuthenticator(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") +class HttpMethod(Enum): + GET = "GET" + POST = "POST" + + +class QuotaStatusSource(BaseModel): + type: Literal["QuotaStatusSource"] + url: str = Field( + ..., + description="The full URL of the quota status endpoint.", + examples=[ + "https://api.github.com/rate_limit", + "{{ config.get('api_url', 'https://api.github.com') }}/rate_limit", + ], + title="URL", + ) + http_method: Optional[HttpMethod] = Field( + HttpMethod.GET, + description="The HTTP method used to fetch the quota status.", + title="HTTP Method", + ) + request_headers: Optional[Dict[str, str]] = Field( + None, + description="Additional headers to send with the quota status request.", + title="Request Headers", + ) + unavailable_status_codes: Optional[List[int]] = Field( + None, + description="Status codes from the quota status endpoint that mean quota tracking is unavailable rather than broken, such as a self-hosted deployment with rate limiting turned off. Every pool of the token whose request returned that status is then treated as untracked, so the authenticator stops waiting for quota resets, stops throttling proactively and stops rotating on exhaustion for it, while still signing requests. A token untracked this way stays untracked for the rest of the sync, because the endpoint is never consulted for it again, so a status the endpoint can also return transiently costs quota tracking for the whole run. If only some tokens return that status the others stay tracked, but they are no longer refreshed either, because the authenticator stops waiting for quota resets as soon as one token is untracked; once their counters are locally spent all traffic moves onto the untracked tokens. Rate limiting reported by ordinary responses is still handled by the stream's error handler, so one that retries 429 or 403 keeps working, and a retry rotates onto the next token; it pays the backoff the response asks for rather than the shortened one a tracked pool would get, since an untracked pool has no counters with which to argue the rejection was about that credential. Any status not listed still fails the connection, and this field never excuses a quota path missing from a response the endpoint did answer, so list only the codes the endpoint uses to report that rate limiting is not enabled. Do not list authentication or authorization statuses, since a 401 or 403 from a revoked credential would then be read as quota tracking being unavailable rather than as a credentials failure.", + examples=[[404]], + title="Unavailable Status Codes", + unique_items=True, + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class DynamicStreamCheckConfig(BaseModel): type: Literal["DynamicStreamCheckConfig"] dynamic_stream_name: str = Field( ..., description="The dynamic stream name.", title="Dynamic Stream Name" ) - stream_count: Optional[int] = Field( + stream_count: Optional[conint(ge=1)] = Field( None, description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", - ge=1, title="Stream Count", ) @@ -104,17 +133,16 @@ class ConcurrencyLevel(BaseModel): class ConstantBackoffStrategy(BaseModel): type: Literal["ConstantBackoffStrategy"] - backoff_time_in_seconds: Union[float, str] = Field( + backoff_time_in_seconds: Union[confloat(ge=0.0), str] = Field( ..., description="Backoff time in seconds.", examples=[30, 30.5, "{{ config['backoff_time'] }}"], title="Backoff Time", ) - jitter_range_in_seconds: Optional[float] = Field( + jitter_range_in_seconds: Optional[confloat(ge=0.0)] = Field( None, description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", examples=[15], - ge=0, title="Jitter Range", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -503,7 +531,7 @@ class Config: ) weight: Optional[Union[int, str]] = Field( None, - description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", + description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.\n", title="Weight", ) @@ -523,6 +551,32 @@ class OnNoRecords(Enum): emit_parent = "emit_parent" +class RecordExpander(BaseModel): + type: Literal["RecordExpander"] + expand_records_from_field: List[str] = Field( + ..., + description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", + examples=[ + ["lines", "data"], + ["items"], + ["nested", "array"], + ["sections", "*", "items"], + ], + title="Expand Records From Field", + ) + remain_original_record: Optional[bool] = Field( + False, + description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', + title="Remain Original Record", + ) + on_no_records: Optional[OnNoRecords] = Field( + OnNoRecords.skip, + description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', + title="On No Records", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class ExponentialBackoffStrategy(BaseModel): type: Literal["ExponentialBackoffStrategy"] factor: Optional[Union[float, str]] = Field( @@ -531,11 +585,10 @@ class ExponentialBackoffStrategy(BaseModel): examples=[5, 5.5, "10"], title="Factor", ) - jitter_range_in_seconds: Optional[float] = Field( + jitter_range_in_seconds: Optional[confloat(ge=0.0)] = Field( None, description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", examples=[2], - ge=0, title="Jitter Range", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -556,157 +609,6 @@ class SessionTokenRequestBearerAuthenticator(BaseModel): type: Literal["Bearer"] -class HttpMethod(Enum): - GET = "GET" - POST = "POST" - - -class QuotaStatusSource(BaseModel): - type: Literal["QuotaStatusSource"] - url: str = Field( - ..., - description="The full URL of the quota status endpoint.", - examples=[ - "https://api.github.com/rate_limit", - "{{ config.get('api_url', 'https://api.github.com') }}/rate_limit", - ], - title="URL", - ) - http_method: Optional[HttpMethod] = Field( - HttpMethod.GET, - description="The HTTP method used to fetch the quota status.", - title="HTTP Method", - ) - request_headers: Optional[Dict[str, str]] = Field( - None, - description="Additional headers to send with the quota status request.", - title="Request Headers", - ) - unavailable_status_codes: Optional[List[int]] = Field( - None, - description="Status codes from the quota status endpoint that mean quota tracking is unavailable rather than broken, such as a self-hosted deployment with rate limiting turned off. Every pool of the token whose request returned that status is then treated as untracked, so the authenticator stops waiting for quota resets, stops throttling proactively and stops rotating on exhaustion for it, while still signing requests. A token untracked this way stays untracked for the rest of the sync, because the endpoint is never consulted for it again, so a status the endpoint can also return transiently costs quota tracking for the whole run. If only some tokens return that status the others stay tracked, but they are no longer refreshed either, because the authenticator stops waiting for quota resets as soon as one token is untracked; once their counters are locally spent all traffic moves onto the untracked tokens. Rate limiting reported by ordinary responses is still handled by the stream's error handler, so one that retries 429 or 403 keeps working, and a retry rotates onto the next token; it pays the backoff the response asks for rather than the shortened one a tracked pool would get, since an untracked pool has no counters with which to argue the rejection was about that credential. Any status not listed still fails the connection, and this field never excuses a quota path missing from a response the endpoint did answer, so list only the codes the endpoint uses to report that rate limiting is not enabled. Do not list authentication or authorization statuses, since a 401 or 403 from a revoked credential would then be read as quota tracking being unavailable rather than as a credentials failure.", - examples=[[404]], - title="Unavailable Status Codes", - unique_items=True, - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - -class TokenQuota(BaseModel): - type: Literal["TokenQuota"] - name: str = Field( - ..., - description="Name of the quota pool.", - examples=["rest", "graphql"], - title="Name", - ) - remaining_path: List[str] = Field( - ..., - description="Path to the remaining call count for this pool in the quota status response.", - examples=[["resources", "core", "remaining"]], - title="Remaining Path", - ) - reset_path: List[str] = Field( - ..., - description="Path to the quota reset timestamp for this pool in the quota status response.", - examples=[["resources", "core", "reset"]], - title="Reset Path", - ) - limit_path: Optional[List[str]] = Field( - None, - description="Optional path to the total call limit for this pool in the quota status response. Used to compute the proactive throttling reserve; falls back to the initially observed remaining count when not set. Setting it on every pool is recommended so the reserve does not shrink when a sync starts with the pool already partially consumed.", - examples=[["resources", "core", "limit"]], - title="Limit Path", - ) - matchers: Optional[List[HttpRequestRegexMatcher]] = Field( - None, - description="List of matchers that classify outgoing requests into this quota pool. The first pool whose matcher matches a request is used. A pool with no matchers acts as the default pool.", - title="Matchers", - ) - remaining_header: Optional[str] = Field( - None, - description="Optional response header carrying the remaining call count for this pool. When set, the pool's counter is reconciled against this header on every response, which corrects drift caused by sharing the token with other clients, by requests in flight concurrently, or by a sync running long enough for the initial quota status read to go stale. Without it the pool is only ever seeded from the quota status endpoint.", - examples=["X-RateLimit-Remaining"], - title="Remaining Header", - ) - reset_header: Optional[str] = Field( - None, - description="Optional response header carrying the quota reset timestamp for this pool. Parsed with the same rules as `reset_path`, so epoch seconds and ISO 8601 both work. Used to tell a rolled-over quota window from the current one; a response proving the window has rolled over restores the pool to its limit. Most useful alongside `remaining_header`.", - examples=["X-RateLimit-Reset"], - title="Reset Header", - ) - limit_header: Optional[str] = Field( - None, - description="Optional response header carrying the total call limit for this pool, used to keep the proactive throttling reserve accurate as the limit changes.", - examples=["X-RateLimit-Limit"], - title="Limit Header", - ) - exhaustion_status_codes: Optional[List[int]] = Field( - None, - description="Response status codes that mean this token's pool is spent. These have two effects. A response carrying one of them but no remaining count sets the pool to zero, so the next request rotates to another token instead of waiting out the reset window. They also mark which responses may report a zero for a quota window that has already elapsed, so a rate limit whose reset header trails the value being held still stops the token being used; a zero on any other response is treated as the last call of a finished window and ignored. Leaving this empty means such trailing rejections are ignored unless their reset is within the skew tolerance of the current window. Only list codes the API uses exclusively for rate limiting -- a code that also signals other failures would park a healthy token.", - examples=[[429]], - title="Exhaustion Status Codes", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - -class RateLimitedMultipleTokenAuthenticator(BaseModel): - type: Literal["RateLimitedMultipleTokenAuthenticator"] - tokens: Union[str, List[str]] = Field( - ..., - description="The tokens to rotate between. Either an explicit list of tokens, or a single string containing multiple tokens separated by `token_delimiter`.", - examples=[ - "{{ config['credentials']['personal_access_token'] }}", - ["{{ config['token_1'] }}", "{{ config['token_2'] }}"], - ], - title="Tokens", - ) - token_delimiter: Optional[str] = Field( - ",", - description="Delimiter used to split a single token string into multiple tokens.", - title="Token Delimiter", - ) - auth_method: Optional[str] = Field( - "Bearer", - description="The prefix to prepend to the token in the auth header value (e.g. `Authorization: Bearer `).", - examples=["Bearer", "token"], - title="Auth Method", - ) - header: Optional[str] = Field( - "Authorization", - description="The name of the HTTP header in which to inject the token.", - title="Header Name", - ) - quota_status_source: QuotaStatusSource = Field( - ..., - description="Defines where to fetch each token's current quota status. Called once per token at startup and after an exhaustion wait, not per data request.", - title="Quota Status Source", - ) - quotas: List[TokenQuota] = Field( - ..., - description="Quota pools tracked per token. Each outgoing request is classified into the first pool whose matchers match the request; a pool with no matchers acts as the default. The `remaining_path` and `reset_path` locate each pool's values in the quota status response.\n", - min_items=1, - title="Quota Pools", - ) - max_wait_time: Optional[str] = Field( - "PT2H", - description="ISO 8601 duration. When all tokens are exhausted, the maximum time to wait for a quota reset before raising a transient error.", - examples=["PT2H", "PT30M", "PT{{ config.get('max_waiting_time', 120) }}M"], - title="Maximum Wait Time", - ) - budget_reserve_fraction: Optional[float] = Field( - 0.1, - description="Fraction of each token's quota to keep in reserve. When every token drops below its reserve, requests are proactively throttled to spread the remaining calls until the quota reset. Set to 0 (along with `budget_min_reserve`) to disable throttling.", - title="Budget Reserve Fraction", - ) - budget_min_reserve: Optional[int] = Field( - 50, - description="Minimum number of calls to keep in reserve per token before proactive throttling kicks in.", - title="Budget Minimum Reserve", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class Action(Enum): SUCCESS = "SUCCESS" FAIL = "FAIL" @@ -838,12 +740,13 @@ class JsonItemsDecoder(BaseModel): type: Literal["JsonItemsDecoder"] items_path: str = Field( ..., - description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax \u2014 do not include leading `$.` or trailing `[*]`.", + description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax — do not include leading `$.` or trailing `[*]`.", + examples=["dataByDepartmentAndSearchTerm", "dataByAsin", "data.users"], title="Items Path", ) encoding: Optional[str] = Field( "utf-8", - description="The character encoding of the JSON data. Defaults to UTF-8.", + description="Text encoding used to decode the streamed bytes before JSON parsing.", title="Encoding", ) @@ -1006,22 +909,32 @@ class NoPagination(BaseModel): type: Literal["NoPagination"] -class State(BaseModel): +class Scope(BaseModel): class Config: extra = Extra.allow - min: int - max: int + scope: str = Field(..., description="The OAuth scope string to request from the provider.") -class OAuthScope(BaseModel): +class OptionalScope(BaseModel): class Config: extra = Extra.allow - scope: str = Field( - ..., - description="The OAuth scope string to request from the provider.", - ) + scope: str = Field(..., description="The OAuth scope string to request from the provider.") + + +class ScopesJoinStrategy(Enum): + space = "space" + comma = "comma" + plus = "plus" + + +class State(BaseModel): + class Config: + extra = Extra.allow + + min: int + max: int class OauthConnectorInputSpecification(BaseModel): @@ -1043,17 +956,13 @@ class Config: examples=["user:read user:read_orders workspaces:read"], title="Scopes", ) - # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the - # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. - # The CDK schema defines the manifest contract; the platform reads these fields - # during the OAuth consent flow to build the authorization URL. - scopes: Optional[List[OAuthScope]] = Field( + scopes: Optional[List[Scope]] = Field( None, description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], title="Scopes", ) - optional_scopes: Optional[List[OAuthScope]] = Field( + optional_scopes: Optional[List[OptionalScope]] = Field( None, description="Optional OAuth scope objects that may or may not be granted.", examples=[[{"scope": "admin:read"}]], @@ -1420,25 +1329,22 @@ class ResetPolicy(Enum): class PageSizeReduction(BaseModel): type: Literal["PageSizeReduction"] - reduction_factor: Optional[float] = Field( + reduction_factor: Optional[confloat(gt=1.0)] = Field( 2, description="Divisor applied to the page size on each reduction. The new page size is floor(current page size / reduction factor).", examples=[2, 4], - gt=1.0, title="Reduction Factor", ) - minimum_page_size: Optional[int] = Field( + minimum_page_size: Optional[conint(ge=1)] = Field( 1, description="Page size below which the connector stops reducing and fails the sync. It must be smaller than the page size configured on the pagination strategy, otherwise no reduction could ever be applied.", examples=[1, 10], - ge=1, title="Minimum Page Size", ) - max_attempts: Optional[int] = Field( + max_attempts: Optional[conint(ge=1)] = Field( 5, description="Maximum number of page size reductions made in a row without a single page succeeding, before the sync fails with a transient error. Every reduction follows a request that failed, so at most max_attempts + 1 failing requests are issued before giving up. With reset_policy NEVER this bounds the reductions for the whole partition, since the reduced page size is never restored; with AFTER_SUCCESSFUL_PAGE the budget restarts after every page that succeeds, so it bounds the reductions needed to get a single page through and not the number of pages a partition may have.", examples=[5, 10], - ge=1, title="Maximum Reduction Attempts", ) reset_policy: Optional[ResetPolicy] = Field( @@ -1461,7 +1367,14 @@ class AsyncJobStatusMap(BaseModel): completed: List[str] failed: List[str] timeout: List[str] - skipped: Optional[List[str]] = None + skipped: Optional[List[str]] = Field( + None, + description="Statuses that indicate the job was skipped because there is no data to return. Jobs with these statuses will not be retried and no records will be fetched.", + ) + + +class BlockSimultaneousSyncsAction(BaseModel): + type: Literal["BlockSimultaneousSyncsAction"] class ValueType(Enum): @@ -1829,6 +1742,64 @@ class AuthFlow(BaseModel): oauth_config_specification: Optional[OAuthConfigSpecification] = None +class TokenQuota(BaseModel): + type: Literal["TokenQuota"] + name: str = Field( + ..., + description="Name of the quota pool.", + examples=["rest", "graphql"], + title="Name", + ) + remaining_path: List[str] = Field( + ..., + description="Path to the remaining call count for this pool in the quota status response.", + examples=[["resources", "core", "remaining"]], + title="Remaining Path", + ) + reset_path: List[str] = Field( + ..., + description="Path to the quota reset timestamp for this pool in the quota status response.", + examples=[["resources", "core", "reset"]], + title="Reset Path", + ) + limit_path: Optional[List[str]] = Field( + None, + description="Optional path to the total call limit for this pool in the quota status response. Used to compute the proactive throttling reserve; falls back to the initially observed remaining count when not set. Setting it on every pool is recommended so the reserve does not shrink when a sync starts with the pool already partially consumed.", + examples=[["resources", "core", "limit"]], + title="Limit Path", + ) + matchers: Optional[List[HttpRequestRegexMatcher]] = Field( + None, + description="List of matchers that classify outgoing requests into this quota pool. The first pool whose matcher matches a request is used. A pool with no matchers acts as the default pool.", + title="Matchers", + ) + remaining_header: Optional[str] = Field( + None, + description="Optional response header carrying the remaining call count for this pool. When set, the pool's counter is reconciled against this header on every response, which corrects drift caused by sharing the token with other clients, by requests in flight concurrently, or by a sync running long enough for the initial quota status read to go stale. Without it the pool is only ever seeded from the quota status endpoint.", + examples=["X-RateLimit-Remaining"], + title="Remaining Header", + ) + reset_header: Optional[str] = Field( + None, + description="Optional response header carrying the quota reset timestamp for this pool. Parsed with the same rules as `reset_path`, so epoch seconds and ISO 8601 both work. Used to tell a rolled-over quota window from the current one; a response proving the window has rolled over restores the pool to its limit. Most useful alongside `remaining_header`.", + examples=["X-RateLimit-Reset"], + title="Reset Header", + ) + limit_header: Optional[str] = Field( + None, + description="Optional response header carrying the total call limit for this pool, used to keep the proactive throttling reserve accurate as the limit changes.", + examples=["X-RateLimit-Limit"], + title="Limit Header", + ) + exhaustion_status_codes: Optional[List[int]] = Field( + None, + description="Response status codes that mean this token's pool is spent. These have two effects. A response carrying one of them but no remaining count sets the pool to zero, so the next request rotates to another token instead of waiting out the reset window. They also mark which responses may report a zero for a quota window that has already elapsed, so a rate limit whose reset header trails the value being held still stops the token being used; a zero on any other response is treated as the last call of a finished window and ignored. Leaving this empty means such trailing rejections are ignored unless their reset is within the skew tolerance of the current window. Only list codes the API uses exclusively for rate limiting -- a code that also signals other failures would park a healthy token.", + examples=[[429]], + title="Exhaustion Status Codes", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class CheckStream(BaseModel): type: Literal["CheckStream"] stream_names: Optional[List[str]] = Field( @@ -2320,28 +2291,23 @@ class DefaultPaginator(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") -class RecordExpander(BaseModel): - type: Literal["RecordExpander"] - expand_records_from_field: List[str] = Field( +class DpathExtractor(BaseModel): + type: Literal["DpathExtractor"] + field_path: List[str] = Field( ..., - description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", + description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', examples=[ - ["lines", "data"], - ["items"], - ["nested", "array"], - ["sections", "*", "items"], + ["data"], + ["data", "records"], + ["data", "{{ parameters.name }}"], + ["data", "*", "record"], ], - title="Expand Records From Field", - ) - remain_original_record: Optional[bool] = Field( - False, - description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', - title="Remain Original Record", + title="Field Path", ) - on_no_records: Optional[OnNoRecords] = Field( - OnNoRecords.skip, - description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', - title="On No Records", + record_expander: Optional[RecordExpander] = Field( + None, + description="Optional component to expand records by extracting items from nested array fields.", + title="Record Expander", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -2414,6 +2380,27 @@ class ListPartitionRouter(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") +class RecordSelector(BaseModel): + type: Literal["RecordSelector"] + extractor: Union[DpathExtractor, CustomRecordExtractor] + record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( + None, + description="Responsible for filtering records to be emitted by the Source.", + title="Record Filter", + ) + schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( + None, + description="Responsible for normalization according to the schema.", + title="Schema Normalization", + ) + transform_before_filtering: Optional[bool] = Field( + None, + description="If true, transformation will be applied before record filtering.", + title="Transform Before Filtering", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class PaginationReset(BaseModel): type: Literal["PaginationReset"] action: Action1 @@ -2489,6 +2476,63 @@ class ConfigAddFields(BaseModel): ) +class RateLimitedMultipleTokenAuthenticator(BaseModel): + type: Literal["RateLimitedMultipleTokenAuthenticator"] + tokens: Union[str, List[str]] = Field( + ..., + description="The tokens to rotate between. Either an explicit list of tokens, or a single string containing multiple tokens separated by `token_delimiter`.", + examples=[ + "{{ config['credentials']['personal_access_token'] }}", + ["{{ config['token_1'] }}", "{{ config['token_2'] }}"], + ], + title="Tokens", + ) + token_delimiter: Optional[str] = Field( + ",", + description="Delimiter used to split a single token string into multiple tokens.", + title="Token Delimiter", + ) + auth_method: Optional[str] = Field( + "Bearer", + description="The prefix to prepend to the token in the auth header value (e.g. `Authorization: Bearer `).", + examples=["Bearer", "token"], + title="Auth Method", + ) + header: Optional[str] = Field( + "Authorization", + description="The name of the HTTP header in which to inject the token.", + title="Header Name", + ) + quota_status_source: QuotaStatusSource = Field( + ..., + description="Defines where to fetch each token's current quota status. Called once per token at startup and after an exhaustion wait, not per data request.", + title="Quota Status Source", + ) + quotas: List[TokenQuota] = Field( + ..., + description="Quota pools tracked per token. Each outgoing request is classified into the first pool whose matchers match the request; a pool with no matchers acts as the default. The `remaining_path` and `reset_path` locate each pool's values in the quota status response.\n", + min_items=1, + title="Quota Pools", + ) + max_wait_time: Optional[str] = Field( + "PT2H", + description="ISO 8601 duration. When all tokens are exhausted, the maximum time to wait for a quota reset before raising a transient error.", + examples=["PT2H", "PT30M", "PT{{ config.get('max_waiting_time', 120) }}M"], + title="Maximum Wait Time", + ) + budget_reserve_fraction: Optional[float] = Field( + 0.1, + description="Fraction of each token's quota to keep in reserve. When every token drops below its reserve, requests are proactively throttled to spread the remaining calls until the quota reset. Set to 0 (along with `budget_min_reserve`) to disable throttling.", + title="Budget Reserve Fraction", + ) + budget_min_reserve: Optional[int] = Field( + 50, + description="Minimum number of calls to keep in reserve per token before proactive throttling kicks in.", + title="Budget Minimum Reserve", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class CompositeErrorHandler(BaseModel): type: Literal["CompositeErrorHandler"] error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( @@ -2534,27 +2578,6 @@ class Config: ) -class DpathExtractor(BaseModel): - type: Literal["DpathExtractor"] - field_path: List[str] = Field( - ..., - description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', - examples=[ - ["data"], - ["data", "records"], - ["data", "{{ parameters.name }}"], - ["data", "*", "record"], - ], - title="Field Path", - ) - record_expander: Optional[RecordExpander] = Field( - None, - description="Optional component to expand records by extracting items from nested array fields.", - title="Record Expander", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class ZipfileDecoder(BaseModel): class Config: extra = Extra.allow @@ -2567,27 +2590,6 @@ class Config: ) -class RecordSelector(BaseModel): - type: Literal["RecordSelector"] - extractor: Union[DpathExtractor, CustomRecordExtractor] - record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( - None, - description="Responsible for filtering records to be emitted by the Source.", - title="Record Filter", - ) - schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( - None, - description="Responsible for normalization according to the schema.", - title="Schema Normalization", - ) - transform_before_filtering: Optional[bool] = Field( - None, - description="If true, transformation will be applied before record filtering.", - title="Transform Before Filtering", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class ConfigMigration(BaseModel): type: Literal["ConfigMigration"] description: Optional[str] = Field( @@ -2680,7 +2682,7 @@ class Config: api_budget: Optional[HTTPAPIBudget] = None stream_groups: Optional[Dict[str, StreamGroup]] = Field( None, - description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", + description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.\n", title="Stream Groups", ) max_concurrent_async_job_count: Optional[Union[int, str]] = Field( @@ -2720,7 +2722,7 @@ class Config: api_budget: Optional[HTTPAPIBudget] = None stream_groups: Optional[Dict[str, StreamGroup]] = Field( None, - description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", + description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.\n", title="Stream Groups", ) max_concurrent_async_job_count: Optional[Union[int, str]] = Field( @@ -3218,7 +3220,7 @@ class StateDelegatingStream(BaseModel): ) api_retention_period: Optional[str] = Field( None, - description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", + description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n* **PT1H**: 1 hour\n* **P1D**: 1 day\n* **P1W**: 1 week\n* **P1M**: 1 month\n* **P1Y**: 1 year\n* **P30D**: 30 days\n", examples=["P30D", "P90D", "P1Y"], title="API Retention Period", ) @@ -3324,10 +3326,9 @@ class AsyncRetriever(BaseModel): None, description="The time in minutes after which the single Async Job should be considered as Timed Out.", ) - failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( + failed_retry_wait_time_in_seconds: Optional[Union[conint(ge=1), str]] = Field( None, description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", - ge=1, ) download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( None, @@ -3408,20 +3409,14 @@ class AsyncRetriever(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") -class BlockSimultaneousSyncsAction(BaseModel): - type: Literal["BlockSimultaneousSyncsAction"] - - class StreamGroup(BaseModel): - streams: List[str] = Field( + streams: List[DeclarativeStream] = Field( ..., - description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").', + description="List of references to streams that belong to this group.\n", title="Streams", ) action: BlockSimultaneousSyncsAction = Field( - ..., - description="The action to apply to streams in this group.", - title="Action", + ..., description="The action to apply to streams in this group.", title="Action" ) @@ -3446,7 +3441,7 @@ class GroupingPartitionRouter(BaseModel): underlying_partition_router: Union[ ListPartitionRouter, SubstreamPartitionRouter, - "UnionPartitionRouter", + UnionPartitionRouter, CustomPartitionRouter, ] = Field( ..., @@ -3534,4 +3529,3 @@ class DynamicDeclarativeStream(BaseModel): SimpleRetriever.update_forward_refs() AsyncRetriever.update_forward_refs() GroupingPartitionRouter.update_forward_refs() -UnionPartitionRouter.update_forward_refs() From 65572c8f34baa808e708307bd84d01acaa4ce6e0 Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Tue, 15 Sep 2026 22:02:15 +0300 Subject: [PATCH 04/13] revert: undo the `/poe build` codegen commit, keep the hand-written model Reverts 09882ecf ("Auto-committed changes from Poe command `build`"). Running the codegen pipeline produced a +259/-265 diff against the committed generated model and turned the MyPy Check red with 8 `valid-type` errors. Every one is a `conint(...)` or `confloat(...)` used as an annotation, which mypy rejects; the committed model on `main` contains none of these forms, so `main` is green and only a regenerated file fails. Three of the eight come from this PR (`PageSizeReduction.reduction_factor`, `.minimum_page_size`, `.max_attempts`). The other five are pre-existing and unrelated to this change: `DynamicStreamCheckConfig.stream_count`, `ConstantBackoffStrategy` (x2), `ExponentialBackoffStrategy.jitter_range_in_seconds`, and `AsyncRetriever.failed_retry_wait_time_in_seconds`. They are latent on every branch and surface the moment anyone regenerates. The root cause is in bin/generate_component_manifest_files.py: the datamodel-codegen invocation does not pass `--field-constraints`, so numeric `minimum` / `exclusiveMinimum` constraints become `conint()` / `confloat()` instead of `Field(ge=...)`. Fixing that regenerates the whole file and should be its own PR against main. Reverting loses no validation. The manifest is checked against declarative_component_schema.yaml by `jsonschema.validators.validate` in `_validate_source()`, so the YAML bounds are enforced regardless of what the generated model expresses - the same mechanism that enforces `minItems` on other components, which codegen also drops. Co-Authored-By: Claude Opus 5 (1M context) --- .../models/declarative_component_schema.py | 524 +++++++++--------- 1 file changed, 265 insertions(+), 259 deletions(-) diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index e34dcc5a55..778c88673b 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -6,7 +6,7 @@ from enum import Enum from typing import Any, Dict, List, Literal, Optional, Union -from pydantic.v1 import BaseModel, Extra, Field, confloat, conint +from pydantic.v1 import BaseModel, Extra, Field from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( BaseModelWithDeprecations, @@ -18,6 +18,12 @@ class AuthFlowType(Enum): oauth1_0 = "oauth1.0" +class ScopesJoinStrategy(Enum): + space = "space" + comma = "comma" + plus = "plus" + + class BasicHttpAuthenticator(BaseModel): type: Literal["BasicHttpAuthenticator"] username: str = Field( @@ -46,50 +52,15 @@ class BearerAuthenticator(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") -class HttpMethod(Enum): - GET = "GET" - POST = "POST" - - -class QuotaStatusSource(BaseModel): - type: Literal["QuotaStatusSource"] - url: str = Field( - ..., - description="The full URL of the quota status endpoint.", - examples=[ - "https://api.github.com/rate_limit", - "{{ config.get('api_url', 'https://api.github.com') }}/rate_limit", - ], - title="URL", - ) - http_method: Optional[HttpMethod] = Field( - HttpMethod.GET, - description="The HTTP method used to fetch the quota status.", - title="HTTP Method", - ) - request_headers: Optional[Dict[str, str]] = Field( - None, - description="Additional headers to send with the quota status request.", - title="Request Headers", - ) - unavailable_status_codes: Optional[List[int]] = Field( - None, - description="Status codes from the quota status endpoint that mean quota tracking is unavailable rather than broken, such as a self-hosted deployment with rate limiting turned off. Every pool of the token whose request returned that status is then treated as untracked, so the authenticator stops waiting for quota resets, stops throttling proactively and stops rotating on exhaustion for it, while still signing requests. A token untracked this way stays untracked for the rest of the sync, because the endpoint is never consulted for it again, so a status the endpoint can also return transiently costs quota tracking for the whole run. If only some tokens return that status the others stay tracked, but they are no longer refreshed either, because the authenticator stops waiting for quota resets as soon as one token is untracked; once their counters are locally spent all traffic moves onto the untracked tokens. Rate limiting reported by ordinary responses is still handled by the stream's error handler, so one that retries 429 or 403 keeps working, and a retry rotates onto the next token; it pays the backoff the response asks for rather than the shortened one a tracked pool would get, since an untracked pool has no counters with which to argue the rejection was about that credential. Any status not listed still fails the connection, and this field never excuses a quota path missing from a response the endpoint did answer, so list only the codes the endpoint uses to report that rate limiting is not enabled. Do not list authentication or authorization statuses, since a 401 or 403 from a revoked credential would then be read as quota tracking being unavailable rather than as a credentials failure.", - examples=[[404]], - title="Unavailable Status Codes", - unique_items=True, - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class DynamicStreamCheckConfig(BaseModel): type: Literal["DynamicStreamCheckConfig"] dynamic_stream_name: str = Field( ..., description="The dynamic stream name.", title="Dynamic Stream Name" ) - stream_count: Optional[conint(ge=1)] = Field( + stream_count: Optional[int] = Field( None, description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", + ge=1, title="Stream Count", ) @@ -133,16 +104,17 @@ class ConcurrencyLevel(BaseModel): class ConstantBackoffStrategy(BaseModel): type: Literal["ConstantBackoffStrategy"] - backoff_time_in_seconds: Union[confloat(ge=0.0), str] = Field( + backoff_time_in_seconds: Union[float, str] = Field( ..., description="Backoff time in seconds.", examples=[30, 30.5, "{{ config['backoff_time'] }}"], title="Backoff Time", ) - jitter_range_in_seconds: Optional[confloat(ge=0.0)] = Field( + jitter_range_in_seconds: Optional[float] = Field( None, description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", examples=[15], + ge=0, title="Jitter Range", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -531,7 +503,7 @@ class Config: ) weight: Optional[Union[int, str]] = Field( None, - description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.\n", + description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", title="Weight", ) @@ -551,32 +523,6 @@ class OnNoRecords(Enum): emit_parent = "emit_parent" -class RecordExpander(BaseModel): - type: Literal["RecordExpander"] - expand_records_from_field: List[str] = Field( - ..., - description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", - examples=[ - ["lines", "data"], - ["items"], - ["nested", "array"], - ["sections", "*", "items"], - ], - title="Expand Records From Field", - ) - remain_original_record: Optional[bool] = Field( - False, - description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', - title="Remain Original Record", - ) - on_no_records: Optional[OnNoRecords] = Field( - OnNoRecords.skip, - description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', - title="On No Records", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class ExponentialBackoffStrategy(BaseModel): type: Literal["ExponentialBackoffStrategy"] factor: Optional[Union[float, str]] = Field( @@ -585,10 +531,11 @@ class ExponentialBackoffStrategy(BaseModel): examples=[5, 5.5, "10"], title="Factor", ) - jitter_range_in_seconds: Optional[confloat(ge=0.0)] = Field( + jitter_range_in_seconds: Optional[float] = Field( None, description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", examples=[2], + ge=0, title="Jitter Range", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -609,6 +556,157 @@ class SessionTokenRequestBearerAuthenticator(BaseModel): type: Literal["Bearer"] +class HttpMethod(Enum): + GET = "GET" + POST = "POST" + + +class QuotaStatusSource(BaseModel): + type: Literal["QuotaStatusSource"] + url: str = Field( + ..., + description="The full URL of the quota status endpoint.", + examples=[ + "https://api.github.com/rate_limit", + "{{ config.get('api_url', 'https://api.github.com') }}/rate_limit", + ], + title="URL", + ) + http_method: Optional[HttpMethod] = Field( + HttpMethod.GET, + description="The HTTP method used to fetch the quota status.", + title="HTTP Method", + ) + request_headers: Optional[Dict[str, str]] = Field( + None, + description="Additional headers to send with the quota status request.", + title="Request Headers", + ) + unavailable_status_codes: Optional[List[int]] = Field( + None, + description="Status codes from the quota status endpoint that mean quota tracking is unavailable rather than broken, such as a self-hosted deployment with rate limiting turned off. Every pool of the token whose request returned that status is then treated as untracked, so the authenticator stops waiting for quota resets, stops throttling proactively and stops rotating on exhaustion for it, while still signing requests. A token untracked this way stays untracked for the rest of the sync, because the endpoint is never consulted for it again, so a status the endpoint can also return transiently costs quota tracking for the whole run. If only some tokens return that status the others stay tracked, but they are no longer refreshed either, because the authenticator stops waiting for quota resets as soon as one token is untracked; once their counters are locally spent all traffic moves onto the untracked tokens. Rate limiting reported by ordinary responses is still handled by the stream's error handler, so one that retries 429 or 403 keeps working, and a retry rotates onto the next token; it pays the backoff the response asks for rather than the shortened one a tracked pool would get, since an untracked pool has no counters with which to argue the rejection was about that credential. Any status not listed still fails the connection, and this field never excuses a quota path missing from a response the endpoint did answer, so list only the codes the endpoint uses to report that rate limiting is not enabled. Do not list authentication or authorization statuses, since a 401 or 403 from a revoked credential would then be read as quota tracking being unavailable rather than as a credentials failure.", + examples=[[404]], + title="Unavailable Status Codes", + unique_items=True, + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + +class TokenQuota(BaseModel): + type: Literal["TokenQuota"] + name: str = Field( + ..., + description="Name of the quota pool.", + examples=["rest", "graphql"], + title="Name", + ) + remaining_path: List[str] = Field( + ..., + description="Path to the remaining call count for this pool in the quota status response.", + examples=[["resources", "core", "remaining"]], + title="Remaining Path", + ) + reset_path: List[str] = Field( + ..., + description="Path to the quota reset timestamp for this pool in the quota status response.", + examples=[["resources", "core", "reset"]], + title="Reset Path", + ) + limit_path: Optional[List[str]] = Field( + None, + description="Optional path to the total call limit for this pool in the quota status response. Used to compute the proactive throttling reserve; falls back to the initially observed remaining count when not set. Setting it on every pool is recommended so the reserve does not shrink when a sync starts with the pool already partially consumed.", + examples=[["resources", "core", "limit"]], + title="Limit Path", + ) + matchers: Optional[List[HttpRequestRegexMatcher]] = Field( + None, + description="List of matchers that classify outgoing requests into this quota pool. The first pool whose matcher matches a request is used. A pool with no matchers acts as the default pool.", + title="Matchers", + ) + remaining_header: Optional[str] = Field( + None, + description="Optional response header carrying the remaining call count for this pool. When set, the pool's counter is reconciled against this header on every response, which corrects drift caused by sharing the token with other clients, by requests in flight concurrently, or by a sync running long enough for the initial quota status read to go stale. Without it the pool is only ever seeded from the quota status endpoint.", + examples=["X-RateLimit-Remaining"], + title="Remaining Header", + ) + reset_header: Optional[str] = Field( + None, + description="Optional response header carrying the quota reset timestamp for this pool. Parsed with the same rules as `reset_path`, so epoch seconds and ISO 8601 both work. Used to tell a rolled-over quota window from the current one; a response proving the window has rolled over restores the pool to its limit. Most useful alongside `remaining_header`.", + examples=["X-RateLimit-Reset"], + title="Reset Header", + ) + limit_header: Optional[str] = Field( + None, + description="Optional response header carrying the total call limit for this pool, used to keep the proactive throttling reserve accurate as the limit changes.", + examples=["X-RateLimit-Limit"], + title="Limit Header", + ) + exhaustion_status_codes: Optional[List[int]] = Field( + None, + description="Response status codes that mean this token's pool is spent. These have two effects. A response carrying one of them but no remaining count sets the pool to zero, so the next request rotates to another token instead of waiting out the reset window. They also mark which responses may report a zero for a quota window that has already elapsed, so a rate limit whose reset header trails the value being held still stops the token being used; a zero on any other response is treated as the last call of a finished window and ignored. Leaving this empty means such trailing rejections are ignored unless their reset is within the skew tolerance of the current window. Only list codes the API uses exclusively for rate limiting -- a code that also signals other failures would park a healthy token.", + examples=[[429]], + title="Exhaustion Status Codes", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + +class RateLimitedMultipleTokenAuthenticator(BaseModel): + type: Literal["RateLimitedMultipleTokenAuthenticator"] + tokens: Union[str, List[str]] = Field( + ..., + description="The tokens to rotate between. Either an explicit list of tokens, or a single string containing multiple tokens separated by `token_delimiter`.", + examples=[ + "{{ config['credentials']['personal_access_token'] }}", + ["{{ config['token_1'] }}", "{{ config['token_2'] }}"], + ], + title="Tokens", + ) + token_delimiter: Optional[str] = Field( + ",", + description="Delimiter used to split a single token string into multiple tokens.", + title="Token Delimiter", + ) + auth_method: Optional[str] = Field( + "Bearer", + description="The prefix to prepend to the token in the auth header value (e.g. `Authorization: Bearer `).", + examples=["Bearer", "token"], + title="Auth Method", + ) + header: Optional[str] = Field( + "Authorization", + description="The name of the HTTP header in which to inject the token.", + title="Header Name", + ) + quota_status_source: QuotaStatusSource = Field( + ..., + description="Defines where to fetch each token's current quota status. Called once per token at startup and after an exhaustion wait, not per data request.", + title="Quota Status Source", + ) + quotas: List[TokenQuota] = Field( + ..., + description="Quota pools tracked per token. Each outgoing request is classified into the first pool whose matchers match the request; a pool with no matchers acts as the default. The `remaining_path` and `reset_path` locate each pool's values in the quota status response.\n", + min_items=1, + title="Quota Pools", + ) + max_wait_time: Optional[str] = Field( + "PT2H", + description="ISO 8601 duration. When all tokens are exhausted, the maximum time to wait for a quota reset before raising a transient error.", + examples=["PT2H", "PT30M", "PT{{ config.get('max_waiting_time', 120) }}M"], + title="Maximum Wait Time", + ) + budget_reserve_fraction: Optional[float] = Field( + 0.1, + description="Fraction of each token's quota to keep in reserve. When every token drops below its reserve, requests are proactively throttled to spread the remaining calls until the quota reset. Set to 0 (along with `budget_min_reserve`) to disable throttling.", + title="Budget Reserve Fraction", + ) + budget_min_reserve: Optional[int] = Field( + 50, + description="Minimum number of calls to keep in reserve per token before proactive throttling kicks in.", + title="Budget Minimum Reserve", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class Action(Enum): SUCCESS = "SUCCESS" FAIL = "FAIL" @@ -740,13 +838,12 @@ class JsonItemsDecoder(BaseModel): type: Literal["JsonItemsDecoder"] items_path: str = Field( ..., - description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax — do not include leading `$.` or trailing `[*]`.", - examples=["dataByDepartmentAndSearchTerm", "dataByAsin", "data.users"], + description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax \u2014 do not include leading `$.` or trailing `[*]`.", title="Items Path", ) encoding: Optional[str] = Field( "utf-8", - description="Text encoding used to decode the streamed bytes before JSON parsing.", + description="The character encoding of the JSON data. Defaults to UTF-8.", title="Encoding", ) @@ -909,32 +1006,22 @@ class NoPagination(BaseModel): type: Literal["NoPagination"] -class Scope(BaseModel): - class Config: - extra = Extra.allow - - scope: str = Field(..., description="The OAuth scope string to request from the provider.") - - -class OptionalScope(BaseModel): +class State(BaseModel): class Config: extra = Extra.allow - scope: str = Field(..., description="The OAuth scope string to request from the provider.") - - -class ScopesJoinStrategy(Enum): - space = "space" - comma = "comma" - plus = "plus" + min: int + max: int -class State(BaseModel): +class OAuthScope(BaseModel): class Config: extra = Extra.allow - min: int - max: int + scope: str = Field( + ..., + description="The OAuth scope string to request from the provider.", + ) class OauthConnectorInputSpecification(BaseModel): @@ -956,13 +1043,17 @@ class Config: examples=["user:read user:read_orders workspaces:read"], title="Scopes", ) - scopes: Optional[List[Scope]] = Field( + # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the + # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. + # The CDK schema defines the manifest contract; the platform reads these fields + # during the OAuth consent flow to build the authorization URL. + scopes: Optional[List[OAuthScope]] = Field( None, description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], title="Scopes", ) - optional_scopes: Optional[List[OptionalScope]] = Field( + optional_scopes: Optional[List[OAuthScope]] = Field( None, description="Optional OAuth scope objects that may or may not be granted.", examples=[[{"scope": "admin:read"}]], @@ -1329,22 +1420,25 @@ class ResetPolicy(Enum): class PageSizeReduction(BaseModel): type: Literal["PageSizeReduction"] - reduction_factor: Optional[confloat(gt=1.0)] = Field( + reduction_factor: Optional[float] = Field( 2, description="Divisor applied to the page size on each reduction. The new page size is floor(current page size / reduction factor).", examples=[2, 4], + gt=1.0, title="Reduction Factor", ) - minimum_page_size: Optional[conint(ge=1)] = Field( + minimum_page_size: Optional[int] = Field( 1, description="Page size below which the connector stops reducing and fails the sync. It must be smaller than the page size configured on the pagination strategy, otherwise no reduction could ever be applied.", examples=[1, 10], + ge=1, title="Minimum Page Size", ) - max_attempts: Optional[conint(ge=1)] = Field( + max_attempts: Optional[int] = Field( 5, description="Maximum number of page size reductions made in a row without a single page succeeding, before the sync fails with a transient error. Every reduction follows a request that failed, so at most max_attempts + 1 failing requests are issued before giving up. With reset_policy NEVER this bounds the reductions for the whole partition, since the reduced page size is never restored; with AFTER_SUCCESSFUL_PAGE the budget restarts after every page that succeeds, so it bounds the reductions needed to get a single page through and not the number of pages a partition may have.", examples=[5, 10], + ge=1, title="Maximum Reduction Attempts", ) reset_policy: Optional[ResetPolicy] = Field( @@ -1367,14 +1461,7 @@ class AsyncJobStatusMap(BaseModel): completed: List[str] failed: List[str] timeout: List[str] - skipped: Optional[List[str]] = Field( - None, - description="Statuses that indicate the job was skipped because there is no data to return. Jobs with these statuses will not be retried and no records will be fetched.", - ) - - -class BlockSimultaneousSyncsAction(BaseModel): - type: Literal["BlockSimultaneousSyncsAction"] + skipped: Optional[List[str]] = None class ValueType(Enum): @@ -1742,64 +1829,6 @@ class AuthFlow(BaseModel): oauth_config_specification: Optional[OAuthConfigSpecification] = None -class TokenQuota(BaseModel): - type: Literal["TokenQuota"] - name: str = Field( - ..., - description="Name of the quota pool.", - examples=["rest", "graphql"], - title="Name", - ) - remaining_path: List[str] = Field( - ..., - description="Path to the remaining call count for this pool in the quota status response.", - examples=[["resources", "core", "remaining"]], - title="Remaining Path", - ) - reset_path: List[str] = Field( - ..., - description="Path to the quota reset timestamp for this pool in the quota status response.", - examples=[["resources", "core", "reset"]], - title="Reset Path", - ) - limit_path: Optional[List[str]] = Field( - None, - description="Optional path to the total call limit for this pool in the quota status response. Used to compute the proactive throttling reserve; falls back to the initially observed remaining count when not set. Setting it on every pool is recommended so the reserve does not shrink when a sync starts with the pool already partially consumed.", - examples=[["resources", "core", "limit"]], - title="Limit Path", - ) - matchers: Optional[List[HttpRequestRegexMatcher]] = Field( - None, - description="List of matchers that classify outgoing requests into this quota pool. The first pool whose matcher matches a request is used. A pool with no matchers acts as the default pool.", - title="Matchers", - ) - remaining_header: Optional[str] = Field( - None, - description="Optional response header carrying the remaining call count for this pool. When set, the pool's counter is reconciled against this header on every response, which corrects drift caused by sharing the token with other clients, by requests in flight concurrently, or by a sync running long enough for the initial quota status read to go stale. Without it the pool is only ever seeded from the quota status endpoint.", - examples=["X-RateLimit-Remaining"], - title="Remaining Header", - ) - reset_header: Optional[str] = Field( - None, - description="Optional response header carrying the quota reset timestamp for this pool. Parsed with the same rules as `reset_path`, so epoch seconds and ISO 8601 both work. Used to tell a rolled-over quota window from the current one; a response proving the window has rolled over restores the pool to its limit. Most useful alongside `remaining_header`.", - examples=["X-RateLimit-Reset"], - title="Reset Header", - ) - limit_header: Optional[str] = Field( - None, - description="Optional response header carrying the total call limit for this pool, used to keep the proactive throttling reserve accurate as the limit changes.", - examples=["X-RateLimit-Limit"], - title="Limit Header", - ) - exhaustion_status_codes: Optional[List[int]] = Field( - None, - description="Response status codes that mean this token's pool is spent. These have two effects. A response carrying one of them but no remaining count sets the pool to zero, so the next request rotates to another token instead of waiting out the reset window. They also mark which responses may report a zero for a quota window that has already elapsed, so a rate limit whose reset header trails the value being held still stops the token being used; a zero on any other response is treated as the last call of a finished window and ignored. Leaving this empty means such trailing rejections are ignored unless their reset is within the skew tolerance of the current window. Only list codes the API uses exclusively for rate limiting -- a code that also signals other failures would park a healthy token.", - examples=[[429]], - title="Exhaustion Status Codes", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class CheckStream(BaseModel): type: Literal["CheckStream"] stream_names: Optional[List[str]] = Field( @@ -2291,23 +2320,28 @@ class DefaultPaginator(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") -class DpathExtractor(BaseModel): - type: Literal["DpathExtractor"] - field_path: List[str] = Field( +class RecordExpander(BaseModel): + type: Literal["RecordExpander"] + expand_records_from_field: List[str] = Field( ..., - description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', + description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", examples=[ - ["data"], - ["data", "records"], - ["data", "{{ parameters.name }}"], - ["data", "*", "record"], + ["lines", "data"], + ["items"], + ["nested", "array"], + ["sections", "*", "items"], ], - title="Field Path", + title="Expand Records From Field", ) - record_expander: Optional[RecordExpander] = Field( - None, - description="Optional component to expand records by extracting items from nested array fields.", - title="Record Expander", + remain_original_record: Optional[bool] = Field( + False, + description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', + title="Remain Original Record", + ) + on_no_records: Optional[OnNoRecords] = Field( + OnNoRecords.skip, + description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', + title="On No Records", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -2380,27 +2414,6 @@ class ListPartitionRouter(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") -class RecordSelector(BaseModel): - type: Literal["RecordSelector"] - extractor: Union[DpathExtractor, CustomRecordExtractor] - record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( - None, - description="Responsible for filtering records to be emitted by the Source.", - title="Record Filter", - ) - schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( - None, - description="Responsible for normalization according to the schema.", - title="Schema Normalization", - ) - transform_before_filtering: Optional[bool] = Field( - None, - description="If true, transformation will be applied before record filtering.", - title="Transform Before Filtering", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class PaginationReset(BaseModel): type: Literal["PaginationReset"] action: Action1 @@ -2476,63 +2489,6 @@ class ConfigAddFields(BaseModel): ) -class RateLimitedMultipleTokenAuthenticator(BaseModel): - type: Literal["RateLimitedMultipleTokenAuthenticator"] - tokens: Union[str, List[str]] = Field( - ..., - description="The tokens to rotate between. Either an explicit list of tokens, or a single string containing multiple tokens separated by `token_delimiter`.", - examples=[ - "{{ config['credentials']['personal_access_token'] }}", - ["{{ config['token_1'] }}", "{{ config['token_2'] }}"], - ], - title="Tokens", - ) - token_delimiter: Optional[str] = Field( - ",", - description="Delimiter used to split a single token string into multiple tokens.", - title="Token Delimiter", - ) - auth_method: Optional[str] = Field( - "Bearer", - description="The prefix to prepend to the token in the auth header value (e.g. `Authorization: Bearer `).", - examples=["Bearer", "token"], - title="Auth Method", - ) - header: Optional[str] = Field( - "Authorization", - description="The name of the HTTP header in which to inject the token.", - title="Header Name", - ) - quota_status_source: QuotaStatusSource = Field( - ..., - description="Defines where to fetch each token's current quota status. Called once per token at startup and after an exhaustion wait, not per data request.", - title="Quota Status Source", - ) - quotas: List[TokenQuota] = Field( - ..., - description="Quota pools tracked per token. Each outgoing request is classified into the first pool whose matchers match the request; a pool with no matchers acts as the default. The `remaining_path` and `reset_path` locate each pool's values in the quota status response.\n", - min_items=1, - title="Quota Pools", - ) - max_wait_time: Optional[str] = Field( - "PT2H", - description="ISO 8601 duration. When all tokens are exhausted, the maximum time to wait for a quota reset before raising a transient error.", - examples=["PT2H", "PT30M", "PT{{ config.get('max_waiting_time', 120) }}M"], - title="Maximum Wait Time", - ) - budget_reserve_fraction: Optional[float] = Field( - 0.1, - description="Fraction of each token's quota to keep in reserve. When every token drops below its reserve, requests are proactively throttled to spread the remaining calls until the quota reset. Set to 0 (along with `budget_min_reserve`) to disable throttling.", - title="Budget Reserve Fraction", - ) - budget_min_reserve: Optional[int] = Field( - 50, - description="Minimum number of calls to keep in reserve per token before proactive throttling kicks in.", - title="Budget Minimum Reserve", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class CompositeErrorHandler(BaseModel): type: Literal["CompositeErrorHandler"] error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( @@ -2578,6 +2534,27 @@ class Config: ) +class DpathExtractor(BaseModel): + type: Literal["DpathExtractor"] + field_path: List[str] = Field( + ..., + description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', + examples=[ + ["data"], + ["data", "records"], + ["data", "{{ parameters.name }}"], + ["data", "*", "record"], + ], + title="Field Path", + ) + record_expander: Optional[RecordExpander] = Field( + None, + description="Optional component to expand records by extracting items from nested array fields.", + title="Record Expander", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class ZipfileDecoder(BaseModel): class Config: extra = Extra.allow @@ -2590,6 +2567,27 @@ class Config: ) +class RecordSelector(BaseModel): + type: Literal["RecordSelector"] + extractor: Union[DpathExtractor, CustomRecordExtractor] + record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( + None, + description="Responsible for filtering records to be emitted by the Source.", + title="Record Filter", + ) + schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( + None, + description="Responsible for normalization according to the schema.", + title="Schema Normalization", + ) + transform_before_filtering: Optional[bool] = Field( + None, + description="If true, transformation will be applied before record filtering.", + title="Transform Before Filtering", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class ConfigMigration(BaseModel): type: Literal["ConfigMigration"] description: Optional[str] = Field( @@ -2682,7 +2680,7 @@ class Config: api_budget: Optional[HTTPAPIBudget] = None stream_groups: Optional[Dict[str, StreamGroup]] = Field( None, - description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.\n", + description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", title="Stream Groups", ) max_concurrent_async_job_count: Optional[Union[int, str]] = Field( @@ -2722,7 +2720,7 @@ class Config: api_budget: Optional[HTTPAPIBudget] = None stream_groups: Optional[Dict[str, StreamGroup]] = Field( None, - description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.\n", + description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", title="Stream Groups", ) max_concurrent_async_job_count: Optional[Union[int, str]] = Field( @@ -3220,7 +3218,7 @@ class StateDelegatingStream(BaseModel): ) api_retention_period: Optional[str] = Field( None, - description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n* **PT1H**: 1 hour\n* **P1D**: 1 day\n* **P1W**: 1 week\n* **P1M**: 1 month\n* **P1Y**: 1 year\n* **P30D**: 30 days\n", + description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", examples=["P30D", "P90D", "P1Y"], title="API Retention Period", ) @@ -3326,9 +3324,10 @@ class AsyncRetriever(BaseModel): None, description="The time in minutes after which the single Async Job should be considered as Timed Out.", ) - failed_retry_wait_time_in_seconds: Optional[Union[conint(ge=1), str]] = Field( + failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( None, description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", + ge=1, ) download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( None, @@ -3409,14 +3408,20 @@ class AsyncRetriever(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") +class BlockSimultaneousSyncsAction(BaseModel): + type: Literal["BlockSimultaneousSyncsAction"] + + class StreamGroup(BaseModel): - streams: List[DeclarativeStream] = Field( + streams: List[str] = Field( ..., - description="List of references to streams that belong to this group.\n", + description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").', title="Streams", ) action: BlockSimultaneousSyncsAction = Field( - ..., description="The action to apply to streams in this group.", title="Action" + ..., + description="The action to apply to streams in this group.", + title="Action", ) @@ -3441,7 +3446,7 @@ class GroupingPartitionRouter(BaseModel): underlying_partition_router: Union[ ListPartitionRouter, SubstreamPartitionRouter, - UnionPartitionRouter, + "UnionPartitionRouter", CustomPartitionRouter, ] = Field( ..., @@ -3529,3 +3534,4 @@ class DynamicDeclarativeStream(BaseModel): SimpleRetriever.update_forward_refs() AsyncRetriever.update_forward_refs() GroupingPartitionRouter.update_forward_refs() +UnionPartitionRouter.update_forward_refs() From 28f76492742be71d3a005a18e8df5d1ff677a91a Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Tue, 15 Sep 2026 23:12:31 +0300 Subject: [PATCH 05/13] fix: stop classifying negated and arithmetic stop conditions as safe The stop-condition analysis read each comparison on its own, which is only sound while that comparison decides the condition in its own polarity. `{{ not (last_page_size >= 100) }}` and `{{ last_page_size - 100 < 0 }}` both mean `last_page_size < 100`, yet both were accepted as SAFE, so a full page at a reduced size read as a short page and the partition was truncated silently - the exact failure the check exists to prevent. A comparison is now classified only when it is reached from the root of the expression through `and`/`or` alone, and the `lt`/`lteq` readings require `last_page_size` to be compared bare rather than after arithmetic. Anything else is UNKNOWN, which warns instead of rejecting. Also brings the reducer's user-facing messages in line with the error-message guideline: every message names the stream, and the two `transient_error` messages drop the remediation the user cannot act on. The `config_error` messages keep theirs. Co-Authored-By: Claude Opus 5 (1M context) --- .../parsers/stop_condition_safety.py | 63 ++++++++++++++++--- .../retrievers/page_size_reducer.py | 18 ++++-- .../parsers/test_stop_condition_safety.py | 63 +++++++++++++++++++ .../retrievers/test_page_size_reducer.py | 47 ++++++++++++++ .../test_concurrent_declarative_source.py | 5 +- 5 files changed, 181 insertions(+), 15 deletions(-) diff --git a/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py b/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py index f270e3efde..3dd881011d 100644 --- a/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py +++ b/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py @@ -28,6 +28,13 @@ the reduction (it references the `page_size` variable) or when `x` is a literal at or below `minimum_page_size`, which makes the comparison a rewrite of "the page is empty". +Each verdict above reads the comparison on its own, which is only sound while the comparison decides the +condition in its own polarity. `{{ not (last_page_size >= 100) }}` means `last_page_size < 100`, and +`{{ last_page_size - 100 < 0 }}` means the same again, so the operator and the operands of the comparison no +longer say what the condition does. A comparison is therefore only classified when it is reached from the root +of the expression through `and`/`or` alone, and when `last_page_size` is compared bare rather than transformed +first; anything else is unclassifiable. + Anything else is reported as unclassifiable rather than as a truncation: this analysis rejects a manifest at stream construction, so a shape it does not understand must not be treated as a defect. """ @@ -93,11 +100,19 @@ def classify_stop_condition( unclassifiable: List[str] = [] understood = 0 - for left, operator, right in _comparisons(template): + for left, operator, right, decides_condition in _comparisons(template): if not _references(left, LAST_PAGE_SIZE_VARIABLE) and not _references( right, LAST_PAGE_SIZE_VARIABLE ): continue + if not decides_condition: + # The comparison is negated or is an operand of a larger expression, so its own operator no longer + # says what the condition does and neither verdict below would be about the right question. + unclassifiable.append( + f"it uses `{LAST_PAGE_SIZE_VARIABLE}` in a comparison that does not decide the condition on " + f"its own, such as one under a `not` or inside a larger expression" + ) + continue verdict, reason = _classify_comparison(left, operator, right, minimum_page_size) if verdict is StopConditionSafety.TRUNCATES: return verdict, reason @@ -119,13 +134,32 @@ def classify_stop_condition( ) -def _comparisons(template: nodes.Template) -> Iterator[Tuple[nodes.Node, str, nodes.Node]]: - """Flatten every comparison, including the chained ones, into (left, operator, right) triples.""" - for comparison in template.find_all(nodes.Compare): - left = comparison.expr - for operand in comparison.ops: - yield left, operand.op, operand.expr +def _comparisons( + node: nodes.Node, decides_condition: bool = True +) -> Iterator[Tuple[nodes.Node, str, nodes.Node, bool]]: + """ + Flatten every comparison, including the chained ones, into (left, operator, right, decides_condition). + + `decides_condition` is true only for a comparison the truth of the whole condition follows directly: + reached from the root through `and`/`or` alone. Under a `not`, inside a conditional expression, piped + through a filter or used as an operand of another expression, the comparison's own operator says nothing + about what the condition decides, so it is flagged and left unclassified. + """ + if isinstance(node, nodes.Compare): + left = node.expr + for operand in node.ops: + yield left, operand.op, operand.expr, decides_condition left = operand.expr + # A comparison nested inside an operand of this one is an ordinary sub-expression, not a decider. + for operand_node in [node.expr, *(operand.expr for operand in node.ops)]: + yield from _comparisons(operand_node, False) + return + + children_decide = decides_condition and isinstance( + node, (nodes.Template, nodes.Output, nodes.And, nodes.Or) + ) + for child in node.iter_child_nodes(): + yield from _comparisons(child, children_decide) def _classify_comparison( @@ -144,7 +178,12 @@ def _classify_comparison( is_bare = isinstance(left, nodes.Name) and left.name == LAST_PAGE_SIZE_VARIABLE if operator in ("eq", "ne"): - if is_bare and isinstance(right, nodes.Const) and right.value == 0: + if not is_bare: + return ( + StopConditionSafety.UNKNOWN, + f"`{LAST_PAGE_SIZE_VARIABLE}` is transformed before being compared", + ) + if isinstance(right, nodes.Const) and right.value == 0: # A page that is full at the requested size holds at least `minimum_page_size` records, which is # at least 1, so no reduction can make an emptiness test fire. return ( @@ -170,6 +209,14 @@ def _classify_comparison( ) if operator in ("lt", "lteq"): + if isinstance(left, nodes.BinExpr): + # `last_page_size - 100 < 0` is `last_page_size < 100` in disguise: with arithmetic on the left the + # threshold on the right is no longer the page size the condition really stops at, so neither the + # `page_size` reading nor the `minimum_page_size` reading below applies. + return ( + StopConditionSafety.UNKNOWN, + f"`{LAST_PAGE_SIZE_VARIABLE}` takes part in arithmetic before being compared", + ) if _references(right, REQUESTED_PAGE_SIZE_VARIABLE): return ( StopConditionSafety.SAFE, diff --git a/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py b/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py index 872d1755e9..31ec514f28 100644 --- a/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py +++ b/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py @@ -91,7 +91,8 @@ def reduce(self) -> None: if self._configured_page_size is None: raise AirbyteTracedException( internal_message=f"Stream {self._stream_name} received a REDUCE_PAGE_SIZE response action but its paginator does not inject a page size", - message="The connector is set up to reduce its page size on error but does not define one. Set `page_size` on the pagination strategy and `page_size_option` on the paginator.", + message=f"Stream {self._stream_name} is set up to reduce its page size on error but does not define " + f"one. Set `page_size` on the pagination strategy and `page_size_option` on the paginator.", failure_type=FailureType.config_error, ) @@ -106,8 +107,8 @@ def reduce(self) -> None: # the middle of a sync. raise AirbyteTracedException( internal_message=f"Stream {self._stream_name} has a page size of type {type(current_page_size).__name__}: {current_page_size!r}", - message="The connector is set up to reduce its page size on error but its page size is not a whole number. " - "Make sure the pagination strategy's `get_page_size` returns an integer.", + message=f"The page size of stream {self._stream_name} is not a whole number, so the connector cannot " + f"reduce it. Make sure the pagination strategy's `page_size` is a number.", failure_type=FailureType.config_error, ) @@ -120,7 +121,8 @@ def reduce(self) -> None: # has; a partition where nothing gets through burns the budget and fails here. raise AirbyteTracedException( internal_message=f"Stream {self._stream_name} reduced its page size {self._attempts - 1} times in a row without a single page succeeding, which is the configured maximum of {self._config.max_attempts} ({self._total_reductions - 1} reductions so far while reading this partition)", - message=f"The source kept failing while the connector requested smaller and smaller pages (down to {current_page_size} records per page). The API is likely unable to serve these requests. Try syncing fewer streams at once, or contact the API provider.", + # `transient_error`, so no remediation: the sync has nothing for the user to act on. + message=f"The source keeps rejecting pages of stream {self._stream_name} at every page size the connector requested, down to {current_page_size} records per page.", failure_type=FailureType.transient_error, ) @@ -135,12 +137,16 @@ def reduce(self) -> None: # rather than something the platform should retry the whole job for. raise AirbyteTracedException( internal_message=f"Stream {self._stream_name} has a configured page size of {current_page_size} which is not greater than the configured minimum page size of {self._config.minimum_page_size}, so it can never be reduced", - message=f"The connector is set up to reduce its page size on error but its page size ({current_page_size}) is already at or below the configured minimum of {self._config.minimum_page_size}. Lower `minimum_page_size` or raise the pagination strategy's `page_size`.", + message=f"The page size of stream {self._stream_name} ({current_page_size}) is already at or below " + f"the configured minimum of {self._config.minimum_page_size}, so the connector cannot reduce it. " + f"Raise the page size of the stream, or lower `minimum_page_size`.", failure_type=FailureType.config_error, ) raise AirbyteTracedException( internal_message=f"Stream {self._stream_name} still fails with a page size of {current_page_size}, which is the smallest page size allowed by the configured minimum of {self._config.minimum_page_size}", - message=f"The source is still failing with the smallest page the connector is allowed to request ({current_page_size} records per page). The API is likely unable to serve this request. Try syncing fewer streams at once, or contact the API provider.", + # `transient_error`, so no remediation: the sync has nothing for the user to act on. + message=f"The source keeps rejecting pages of stream {self._stream_name} at the smallest page size the " + f"connector is allowed to request ({current_page_size} records per page).", failure_type=FailureType.transient_error, ) diff --git a/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py b/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py index 3ed34e1cd5..431d0a2b4a 100644 --- a/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py +++ b/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py @@ -142,6 +142,53 @@ StopConditionSafety.TRUNCATES, id="upper_bound_through_a_filter", ), + # A negation flips what the comparison decides, so a lower bound that is safe on its own becomes the + # dangerous upper bound: `not (last_page_size >= 100)` is `last_page_size < 100`. The comparison alone + # no longer says what the condition does, so it cannot be classified. + pytest.param( + "{{ not (last_page_size >= 100) }}", + 1, + StopConditionSafety.UNKNOWN, + id="negated_lower_bound", + ), + pytest.param( + "{{ not last_page_size >= 100 }}", + 1, + StopConditionSafety.UNKNOWN, + id="negated_lower_bound_without_parentheses", + ), + pytest.param( + "{{ not (last_page_size == 0) }}", + 1, + StopConditionSafety.UNKNOWN, + id="negated_emptiness_test", + ), + pytest.param( + "{{ 1 if last_page_size == 0 else 0 }}", + 1, + StopConditionSafety.UNKNOWN, + id="comparison_inside_a_conditional_expression", + ), + # Arithmetic moves the threshold out of the comparison: `last_page_size - 100 < 0` is another way to + # write `last_page_size < 100`, so neither the literal on the right nor `minimum_page_size` bounds it. + pytest.param( + "{{ last_page_size - 100 < 0 }}", + 1, + StopConditionSafety.UNKNOWN, + id="upper_bound_with_arithmetic", + ), + pytest.param( + "{{ 0 > last_page_size - 100 }}", + 1, + StopConditionSafety.UNKNOWN, + id="upper_bound_with_arithmetic_reversed", + ), + pytest.param( + "{{ last_page_size * 2 < page_size }}", + 1, + StopConditionSafety.UNKNOWN, + id="requested_page_size_with_arithmetic", + ), ], ) def test_classify_stop_condition(stop_condition, minimum_page_size, expected): @@ -163,3 +210,19 @@ def test_reason_names_the_value_the_page_size_is_compared_against(): _, reason = classify_stop_condition("{{ last_page_size < config['page_size'] }}", 1) assert "config['page_size']" in reason + + +def test_given_negated_comparison_then_reason_points_at_the_shape(): + verdict, reason = classify_stop_condition("{{ not (last_page_size >= 100) }}", 1) + + assert verdict is StopConditionSafety.UNKNOWN + assert "does not decide the condition on its own" in reason + + +def test_given_truncating_comparison_next_to_a_negated_one_then_truncates(): + # The negated comparison is only warned about, so it must not mask a sibling that does truncate. + verdict, _ = classify_stop_condition( + "{{ not (last_page_size >= 100) or last_page_size < 500 }}", 1 + ) + + assert verdict is StopConditionSafety.TRUNCATES diff --git a/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py b/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py index 5db30fedeb..2df59d6fde 100644 --- a/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py +++ b/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py @@ -237,3 +237,50 @@ def test_when_reduce_then_wait_before_the_retry(): PageSizeReducer.BACKOFF_SECONDS * 2, ] assert all(wait > 0 for wait in sleeps) + + +@pytest.mark.parametrize( + "reducer_kwargs,reductions", + [ + pytest.param({"configured_page_size": 4, "minimum_page_size": 2}, 1, id="minimum_reached"), + pytest.param( + {"configured_page_size": 1000, "max_attempts": 2}, 2, id="max_attempts_exhausted" + ), + ], +) +def test_given_reduction_fails_then_message_names_the_stream_and_leaves_out_remediation( + reducer_kwargs, reductions +): + # The user cannot act on a `transient_error`, so the message states the failure alone. Naming the stream + # is what makes it actionable for whoever reads the sync, since a sync reduces per stream. + reducer = _reducer(**reducer_kwargs) + for _ in range(reductions): + reducer.reduce() + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.transient_error + assert A_STREAM_NAME in exception.value.message + assert "contact the API provider" not in exception.value.message + assert "Try syncing fewer streams" not in exception.value.message + + +@pytest.mark.parametrize( + "reducer_kwargs", + [ + pytest.param({"configured_page_size": 1, "minimum_page_size": 1}, id="never_reducible"), + pytest.param({"configured_page_size": None}, id="no_page_size_at_all"), + pytest.param({"configured_page_size": "100"}, id="page_size_is_not_a_number"), + ], +) +def test_given_misconfiguration_then_message_names_the_stream_and_keeps_remediation(reducer_kwargs): + # A `config_error` is the user's to fix, so the remediation stays in the message. + reducer = _reducer(**reducer_kwargs) + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.config_error + assert A_STREAM_NAME in exception.value.message + assert exception.value.message.rstrip().endswith(".") diff --git a/unit_tests/sources/declarative/test_concurrent_declarative_source.py b/unit_tests/sources/declarative/test_concurrent_declarative_source.py index 58db9e1c59..7716899b00 100644 --- a/unit_tests/sources/declarative/test_concurrent_declarative_source.py +++ b/unit_tests/sources/declarative/test_concurrent_declarative_source.py @@ -5162,7 +5162,10 @@ def test_given_reductions_exhausted_when_read_then_emit_a_transient_error(): ] assert errors assert all(error.failure_type == FailureType.transient_error for error in errors) - assert any("smaller and smaller pages" in error.message for error in errors) + assert any( + "keeps rejecting pages of stream" in error.message and "records per page" in error.message + for error in errors + ) def test_given_pagination_limit_reached_when_read_then_reset_pagination(): From caafc973a50cd9e8d05596096cb015228bd12ed6 Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Tue, 15 Sep 2026 23:28:24 +0300 Subject: [PATCH 06/13] fix: tighten the stop-condition analysis on two more shapes `{{ last_page_size < [page_size, 100] | max }}` was SAFE because the threshold only had to *mention* `page_size`, while `max(50, 100)` is the configured size again; `{{ last_page_size < page_size + 50 }}` had the same hole. The `lt` branch now requires the threshold to be the `page_size` variable itself, filters aside, so an expression built from it falls to UNKNOWN. `{% if last_page_size < 100 %}true{% endif %}` had dropped from TRUNCATES to UNKNOWN when the traversal started tracking whether a comparison decides the condition. A `{% if %}` renders truthy text exactly when its test holds, so the test is a decider - but only while the branch it guards is the whole story, so an `else`, an `elif`, or a body the CDK reads as false keeps it unclassified. Text rendered next to a comparison is unclassifiable for the same reason: the condition is then truthy whatever the comparison decided. Co-Authored-By: Claude Opus 5 (1M context) --- .../parsers/stop_condition_safety.py | 78 ++++++++++++++++--- .../parsers/test_stop_condition_safety.py | 54 +++++++++++++ 2 files changed, 122 insertions(+), 10 deletions(-) diff --git a/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py b/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py index 3dd881011d..6da834ee2b 100644 --- a/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py +++ b/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py @@ -24,28 +24,31 @@ `minimum_page_size` is at least 1. - `last_page_size > x` and `last_page_size >= x` can only stop being true as the page shrinks, so a reduction cannot introduce a stop that would not have happened anyway. -- `last_page_size < x` and `last_page_size <= x` are the dangerous shape. They are safe only when `x` follows - the reduction (it references the `page_size` variable) or when `x` is a literal at or below - `minimum_page_size`, which makes the comparison a rewrite of "the page is empty". +- `last_page_size < x` and `last_page_size <= x` are the dangerous shape. They are safe only when `x` is the + `page_size` variable, which follows the reduction, or when `x` is a literal at or below `minimum_page_size`, + which makes the comparison a rewrite of "the page is empty". An expression merely built from `page_size`, + such as `[page_size, 100] | max` or `page_size + 50`, does not count: it can hold the configured size again. Each verdict above reads the comparison on its own, which is only sound while the comparison decides the condition in its own polarity. `{{ not (last_page_size >= 100) }}` means `last_page_size < 100`, and `{{ last_page_size - 100 < 0 }}` means the same again, so the operator and the operands of the comparison no -longer say what the condition does. A comparison is therefore only classified when it is reached from the root -of the expression through `and`/`or` alone, and when `last_page_size` is compared bare rather than transformed -first; anything else is unclassifiable. +longer say what the condition does. A comparison is therefore only classified when the truth of the condition follows its own: reached from the +root through `and`/`or`, or as the test of a `{% if %}` that renders truthy text and nothing else, and with +`last_page_size` compared bare rather than transformed first. Anything else is unclassifiable. Anything else is reported as unclassifiable rather than as a truncation: this analysis rejects a manifest at stream construction, so a shape it does not understand must not be treated as a defect. """ from enum import Enum -from typing import Iterator, List, Tuple +from typing import Iterator, List, Optional, Tuple from jinja2 import nodes from jinja2.environment import Environment from jinja2.exceptions import TemplateSyntaxError +from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import FALSE_VALUES + LAST_PAGE_SIZE_VARIABLE = "last_page_size" REQUESTED_PAGE_SIZE_VARIABLE = "page_size" @@ -155,13 +158,51 @@ def _comparisons( yield from _comparisons(operand_node, False) return - children_decide = decides_condition and isinstance( - node, (nodes.Template, nodes.Output, nodes.And, nodes.Or) - ) + if isinstance(node, nodes.If): + # `{% if last_page_size < 100 %}true{% endif %}` renders truthy text exactly when its test holds, so + # the test decides the condition. That only follows while the branch it guards is the whole story: + # an `else`, an `elif`, or a body rendering something the CDK reads as false breaks the equivalence. + test_decides = decides_condition and _if_follows_its_test(node) + yield from _comparisons(node.test, test_decides) + for branch in [*node.body, *node.elif_, *node.else_]: + yield from _comparisons(branch, False) + return + + children_decide = decides_condition and _passes_truth_through(node) for child in node.iter_child_nodes(): yield from _comparisons(child, children_decide) +def _passes_truth_through(node: nodes.Node) -> bool: + """Whether the truth of the condition follows the truth of this node's children.""" + if isinstance(node, (nodes.And, nodes.Or)): + return True + if isinstance(node, nodes.Template): + # Anything rendered next to the comparison is text of its own, which makes the rendered condition + # truthy whatever the comparison decided. + return len(node.body) == 1 + if isinstance(node, nodes.Output): + return len(node.nodes) == 1 + return False + + +def _if_follows_its_test(node: nodes.If) -> bool: + if node.elif_ or node.else_: + return False + rendered = "" + for statement in node.body: + if not isinstance(statement, nodes.Output): + return False + for output in statement.nodes: + if isinstance(output, nodes.TemplateData): + rendered += output.data + elif isinstance(output, nodes.Const): + rendered += str(output.value) + else: + return False + return rendered not in FALSE_VALUES + + def _classify_comparison( left: nodes.Node, operator: str, right: nodes.Node, minimum_page_size: int ) -> Tuple[StopConditionSafety, str]: @@ -218,6 +259,15 @@ def _classify_comparison( f"`{LAST_PAGE_SIZE_VARIABLE}` takes part in arithmetic before being compared", ) if _references(right, REQUESTED_PAGE_SIZE_VARIABLE): + if not _is_requested_page_size(right): + # `[page_size, 100] | max` and `page_size + 50` both mention the variable while holding a + # value a reduction does not lower, so the threshold does not follow the reduction after all. + return ( + StopConditionSafety.UNKNOWN, + f"it compares `{LAST_PAGE_SIZE_VARIABLE}` against an expression built from " + f"`{REQUESTED_PAGE_SIZE_VARIABLE}` rather than against `{REQUESTED_PAGE_SIZE_VARIABLE}` " + f"itself, so the threshold may not follow the reduction", + ) return ( StopConditionSafety.SAFE, f"it compares `{LAST_PAGE_SIZE_VARIABLE}` against `{REQUESTED_PAGE_SIZE_VARIABLE}`, " @@ -244,6 +294,14 @@ def _classify_comparison( ) +def _is_requested_page_size(node: nodes.Node) -> bool: + """Whether the node is the `page_size` variable itself, filters aside.""" + unfiltered: Optional[nodes.Node] = node + while isinstance(unfiltered, nodes.Filter): + unfiltered = unfiltered.node + return isinstance(unfiltered, nodes.Name) and unfiltered.name == REQUESTED_PAGE_SIZE_VARIABLE + + def _references(node: nodes.Node, name: str) -> bool: if isinstance(node, nodes.Name): return bool(node.name == name) diff --git a/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py b/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py index 431d0a2b4a..70503b51ec 100644 --- a/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py +++ b/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py @@ -189,6 +189,60 @@ StopConditionSafety.UNKNOWN, id="requested_page_size_with_arithmetic", ), + # A threshold merely built from `page_size` can hold the configured size again - `max(50, 100)` is 100 - + # so mentioning the variable is not enough for the threshold to follow the reduction. + pytest.param( + "{{ last_page_size < [page_size, 100] | max }}", + 1, + StopConditionSafety.UNKNOWN, + id="requested_page_size_inside_a_list", + ), + pytest.param( + "{{ last_page_size < page_size + 50 }}", + 1, + StopConditionSafety.UNKNOWN, + id="requested_page_size_plus_a_literal", + ), + # A `{% if %}` renders truthy text exactly when its test holds, so the test decides the condition. + pytest.param( + "{% if last_page_size < 100 %}true{% endif %}", + 1, + StopConditionSafety.TRUNCATES, + id="literal_threshold_in_an_if_block", + ), + pytest.param( + "{% if last_page_size == 0 %}true{% endif %}", + 1, + StopConditionSafety.SAFE, + id="emptiness_test_in_an_if_block", + ), + # ... but only while the guarded branch is the whole story: an `else` branch, or a body the CDK reads + # as false, breaks the equivalence between the test and what the condition renders. + pytest.param( + "{% if last_page_size < 100 %}true{% else %}also true{% endif %}", + 1, + StopConditionSafety.UNKNOWN, + id="literal_threshold_in_an_if_block_with_an_else", + ), + pytest.param( + "{% if last_page_size < 100 %}false{% endif %}", + 1, + StopConditionSafety.UNKNOWN, + id="if_block_rendering_a_false_value", + ), + pytest.param( + "{% if last_page_size < 100 %}{% endif %}", + 1, + StopConditionSafety.UNKNOWN, + id="if_block_rendering_nothing", + ), + # Text rendered next to the comparison makes the condition truthy whatever the comparison decided. + pytest.param( + "{{ last_page_size < 100 }} records", + 1, + StopConditionSafety.UNKNOWN, + id="comparison_rendered_next_to_text", + ), ], ) def test_classify_stop_condition(stop_condition, minimum_page_size, expected): From b56e3e391ae2acab3b79307ad7df84769b97e89e Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Wed, 16 Sep 2026 17:37:15 +0300 Subject: [PATCH 07/13] feat: reject a RequestPath page token and let a connector supply the terminal failure message Two fixes from the architecture review of this PR. A `page_token_option` of type `RequestPath` makes the next page a URL the API built, and that URL already carries the page size the API echoed back. The reduced page size is injected as a request option on top of it, so the request goes out with the page size twice and which one the API honors is up to the API: every page after the first would keep asking for the size that just failed. `ModelToComponentFactory` now rejects that combination at stream construction and points at a `RequestOption` page token instead. When the reduction bottoms out, the CDK only knows that the API rejected every page size it asked for. What narrows a query down is API-specific, so `PageSizeReduction` now takes an optional `failure_message` that is appended to the two `transient_error` messages. It is deliberately not appended to the `config_error` ones: those are about the manifest, not about the API rejecting a page size, and they carry their own remediation. Co-Authored-By: Claude Opus 5 (1M context) --- .../declarative_component_schema.yaml | 19 +++++++- .../models/declarative_component_schema.py | 11 ++++- .../parsers/model_to_component_factory.py | 14 ++++++ .../retrievers/page_size_reducer.py | 28 +++++++++-- .../retrievers/simple_retriever.py | 6 ++- .../test_connector_builder_handler.py | 8 +++- .../test_model_to_component_factory.py | 43 +++++++++++++++++ .../retrievers/test_page_size_reducer.py | 47 +++++++++++++++++++ 8 files changed, 166 insertions(+), 10 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 4d1a176e3d..c8a4e101d8 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -4240,7 +4240,8 @@ definitions: Requires a DefaultPaginator that defines both page_size_option and a pagination strategy with a page_size. Cannot be combined with query properties, a file uploader, or a parent stream read lazily through lazy_read_pointer, because in those cases records of the failing page have already been emitted and - re-issuing the page would emit them twice. + re-issuing the page would emit them twice. A page_token_option of type RequestPath is rejected as + well, because the next page is then a URL built by the API which already carries the page size. "$ref": "#/definitions/PageSizeReduction" ignore_stream_slicer_parameters_on_paginated_requests: description: If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored. @@ -4307,7 +4308,9 @@ definitions: next_page_token accepts a page_size_override keyword argument. PageIncrement is not supported because a smaller page size moves every following page boundary and would skip records. It is also rejected when the stream uses query properties, a file uploader, or reads its parent stream lazily through lazy_read_pointer, - since re-issuing a page whose records were already emitted would duplicate them. Each reduction waits a + since re-issuing a page whose records were already emitted would duplicate them. A page_token_option of + type RequestPath is rejected too: the next page is then a URL built by the API which already carries the + page size, so the reduced page size would be sent next to the original one. Each reduction waits a short, growing amount of time before re-issuing the page so that an endpoint failing at every page size is not hit in a burst. type: object @@ -4352,6 +4355,18 @@ definitions: examples: - 5 - 10 + failure_message: + title: Failure Message + description: >- + Sentence appended to the error message shown to the user when the connector runs out of reductions, + either because max_attempts was reached or because the page size is already at minimum_page_size. Use + it to tell the user what they can do about it in terms of this specific API, for instance which + filter narrows the query down. Without it the message only states that the API kept rejecting every + page size the connector asked for. + type: string + examples: + - Narrow the sync down by selecting fewer fields on this stream. + - Set a more recent start date so that each page covers less data. reset_policy: title: Reset Policy description: >- diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 778c88673b..b91b70109d 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -1441,6 +1441,15 @@ class PageSizeReduction(BaseModel): ge=1, title="Maximum Reduction Attempts", ) + failure_message: Optional[str] = Field( + None, + description="Sentence appended to the error message shown to the user when the connector runs out of reductions, either because max_attempts was reached or because the page size is already at minimum_page_size. Use it to tell the user what they can do about it in terms of this specific API, for instance which filter narrows the query down. Without it the message only states that the API kept rejecting every page size the connector asked for.", + examples=[ + "Narrow the sync down by selecting fewer fields on this stream.", + "Set a more recent start date so that each page covers less data.", + ], + title="Failure Message", + ) reset_policy: Optional[ResetPolicy] = Field( ResetPolicy.NEVER, description="When to restore the page size configured on the pagination strategy. NEVER keeps the reduced page size for the rest of the partition. AFTER_SUCCESSFUL_PAGE restores it as soon as one page succeeds, which means hitting the same error again on every page - use it only when the reduction is worth one extra request per page, for instance because the configured page size usually works and only some pages are too heavy. AFTER_SUCCESSFUL_PAGE also restarts the max_attempts budget on every page that succeeds, so there is no limit on how many reductions a partition may make in total: a stream that needs one reduction per page reads to the end however many pages it has. What is bounded is the reductions that get no page through.", @@ -3262,7 +3271,7 @@ class SimpleRetriever(BaseModel): ) page_size_reduction: Optional[PageSizeReduction] = Field( None, - description="Describes how the page size is reduced when an error handler resolves to the REDUCE_PAGE_SIZE action. Requires a DefaultPaginator that defines both page_size_option and a pagination strategy with a page_size. Cannot be combined with query properties, a file uploader, or a parent stream read lazily through lazy_read_pointer, because in those cases records of the failing page have already been emitted and re-issuing the page would emit them twice.", + description="Describes how the page size is reduced when an error handler resolves to the REDUCE_PAGE_SIZE action. Requires a DefaultPaginator that defines both page_size_option and a pagination strategy with a page_size. Cannot be combined with query properties, a file uploader, or a parent stream read lazily through lazy_read_pointer, because in those cases records of the failing page have already been emitted and re-issuing the page would emit them twice. A page_token_option of type RequestPath is rejected as well, because the next page is then a URL built by the API which already carries the page size.", ) ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( False, diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 8d5aacdc83..8404eafbbb 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -3755,6 +3755,7 @@ def _create_page_size_reduction( reduction_factor=model.page_size_reduction.reduction_factor, # type: ignore[arg-type] # the schema defines a default minimum_page_size=model.page_size_reduction.minimum_page_size, # type: ignore[arg-type] # the schema defines a default max_attempts=model.page_size_reduction.max_attempts, # type: ignore[arg-type] # the schema defines a default + failure_message=model.page_size_reduction.failure_message, reset_policy=PageSizeResetPolicy(reset_policy.value) if reset_policy is not None else PageSizeResetPolicy.NEVER, @@ -3799,6 +3800,19 @@ def _validate_page_size_reduction_is_supported( f"the connector cannot tell the API to send a smaller page." ) + if isinstance(model.paginator.page_token_option, RequestPathModel): + # A RequestPath page token is a full URL built by the API, and it already carries the page size the + # API echoed back. The reduced page size is injected as a request option on top of that URL, so the + # request goes out with the page size twice - the original one from the URL and the reduced one - + # and which of the two the API honors is up to the API. Every page after the first would then keep + # asking for the page size that just failed. + raise ValueError( + f"`page_size_reduction` does not support a `page_token_option` of type RequestPath on stream " + f"{name}. The next page is then requested through a URL returned by the API, which already " + f"carries the page size, so the reduced page size would be sent alongside the original one. Use " + f"a CursorPagination strategy with a `page_token_option` of type RequestOption instead." + ) + strategy = model.paginator.pagination_strategy if isinstance(strategy, PageIncrementModel): raise ValueError( diff --git a/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py b/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py index 31ec514f28..b8305e6ba0 100644 --- a/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py +++ b/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py @@ -30,6 +30,10 @@ class PageSizeReduction: minimum_page_size: int = 1 max_attempts: int = 5 reset_policy: PageSizeResetPolicy = PageSizeResetPolicy.NEVER + # Appended to the two messages raised once the page size cannot be reduced any further. Those are + # `transient_error`s the CDK has no remediation for - it only knows that the API rejected every page size + # asked for - while the connector knows what narrows a query down on this particular API. + failure_message: Optional[str] = None def __post_init__(self) -> None: if self.reduction_factor <= 1: @@ -121,8 +125,11 @@ def reduce(self) -> None: # has; a partition where nothing gets through burns the budget and fails here. raise AirbyteTracedException( internal_message=f"Stream {self._stream_name} reduced its page size {self._attempts - 1} times in a row without a single page succeeding, which is the configured maximum of {self._config.max_attempts} ({self._total_reductions - 1} reductions so far while reading this partition)", - # `transient_error`, so no remediation: the sync has nothing for the user to act on. - message=f"The source keeps rejecting pages of stream {self._stream_name} at every page size the connector requested, down to {current_page_size} records per page.", + # `transient_error`, so the only remediation is the connector's own, if it defined one. + message=self._with_failure_message( + f"The source keeps rejecting pages of stream {self._stream_name} at every page size the " + f"connector requested, down to {current_page_size} records per page." + ), failure_type=FailureType.transient_error, ) @@ -144,9 +151,11 @@ def reduce(self) -> None: ) raise AirbyteTracedException( internal_message=f"Stream {self._stream_name} still fails with a page size of {current_page_size}, which is the smallest page size allowed by the configured minimum of {self._config.minimum_page_size}", - # `transient_error`, so no remediation: the sync has nothing for the user to act on. - message=f"The source keeps rejecting pages of stream {self._stream_name} at the smallest page size the " - f"connector is allowed to request ({current_page_size} records per page).", + # `transient_error`, so the only remediation is the connector's own, if it defined one. + message=self._with_failure_message( + f"The source keeps rejecting pages of stream {self._stream_name} at the smallest page size " + f"the connector is allowed to request ({current_page_size} records per page)." + ), failure_type=FailureType.transient_error, ) @@ -158,6 +167,15 @@ def reduce(self) -> None: self._current_page_size = reduced_page_size self._sleep(backoff) + def _with_failure_message(self, message: str) -> str: + """ + :return: the message followed by the connector's `failure_message`, when it defined one + """ + failure_message = (self._config.failure_message or "").strip() + if not failure_message: + return message + return f"{message} {failure_message}" + def on_successful_page(self) -> None: """ Called after each page that did not require a reduction. diff --git a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index adfa504867..83c53af199 100644 --- a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -91,7 +91,11 @@ class SimpleRetriever(Retriever): post_pagination_filter (Optional[ClientSideIncrementalRecordFilterDecorator]): Set for data feed streams only. Records the cursor considers already synced are dropped once pagination has observed them page_size_reduction (Optional[PageSizeReduction]): How much to shrink the page size when an error handler - resolves to `ResponseAction.REDUCE_PAGE_SIZE`. `None` disables page size reduction entirely + resolves to `ResponseAction.REDUCE_PAGE_SIZE`. `None` disables page size reduction entirely. + It is immutable configuration; the page size in effect lives in a `PageSizeReducer` that + `_read_pages` creates per call, so the retriever and its paginator - both shared by every + partition of the stream, read concurrently - stay stateless. When `page_size_reduction` is + `None` no reducer is created and `_read_pages` keeps its previous behaviour """ requester: Requester diff --git a/unit_tests/connector_builder/test_connector_builder_handler.py b/unit_tests/connector_builder/test_connector_builder_handler.py index 9223f77746..38c072208b 100644 --- a/unit_tests/connector_builder/test_connector_builder_handler.py +++ b/unit_tests/connector_builder/test_connector_builder_handler.py @@ -1961,7 +1961,13 @@ def test_full_resolve_manifest(valid_resolve_manifest_config_file): "inject_into": "request_parameter", "field_name": "first", }, - "page_token_option": {"type": "RequestPath"}, + # `page_size_reduction` rejects a RequestPath page token: the next-page URL built by the + # API already carries the page size, so the reduced one would be sent next to it. + "page_token_option": { + "type": "RequestOption", + "inject_into": "request_parameter", + "field_name": "after", + }, "pagination_strategy": { "type": "CursorPagination", "page_size": 100, diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index dc5bc092e1..bef12d45c1 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -6462,6 +6462,7 @@ def get_schema_loader(stream: DefaultStream): pagination_strategy: {pagination_strategy} {page_size_option} + {page_token_option} record_selector: type: RecordSelector extractor: @@ -6483,12 +6484,14 @@ def _page_size_reduction_stream( action="REDUCE_PAGE_SIZE", pagination_strategy=_CURSOR_PAGINATION_STRATEGY, page_size_option=_PAGE_SIZE_OPTION, + page_token_option="", ): content = _PAGE_SIZE_REDUCTION_STREAM.format( page_size_reduction=page_size_reduction, action=action, pagination_strategy=pagination_strategy, page_size_option=page_size_option, + page_token_option=page_token_option, ) stream_manifest = transformer.propagate_types_and_parameters( "", resolver.preprocess_manifest(YamlDeclarativeSource._parse(content)), {} @@ -6558,6 +6561,44 @@ def test_given_no_page_size_option_and_page_size_reduction_then_raise(): assert "page_size_option" in str(exception.value) +def test_given_request_path_page_token_option_and_page_size_reduction_then_raise(): + # The next page is then a URL built by the API which already carries the page size it echoed back, so the + # reduced page size would be sent next to the original one and the API picks which one it honors. + with pytest.raises(ValueError) as exception: + _page_size_reduction_stream(page_token_option="page_token_option:\n type: RequestPath") + + assert "RequestPath" in str(exception.value) + + +def test_given_request_option_page_token_option_and_page_size_reduction_then_create_retriever(): + retriever = get_retriever( + _page_size_reduction_stream( + page_token_option=( + "page_token_option:\n" + " type: RequestOption\n" + " inject_into: request_parameter\n" + " field_name: after" + ) + ) + ) + + assert retriever.page_size_reduction is not None + + +def test_given_failure_message_then_create_retriever_with_that_message(): + retriever = get_retriever( + _page_size_reduction_stream( + page_size_reduction=( + "page_size_reduction:\n" + " type: PageSizeReduction\n" + " failure_message: Select fewer fields on this stream." + ) + ) + ) + + assert retriever.page_size_reduction.failure_message == "Select fewer fields on this stream." + + class _StrategyHonoringOverride(PaginationStrategy): """A custom strategy that can be told the reduced page size.""" @@ -7007,6 +7048,7 @@ def test_given_query_properties_and_page_size_reduction_then_raise(): action="REDUCE_PAGE_SIZE", pagination_strategy=_CURSOR_PAGINATION_STRATEGY, page_size_option=_PAGE_SIZE_OPTION, + page_token_option="", ).replace( " http_method: POST\n", " http_method: POST\n" @@ -7169,6 +7211,7 @@ def test_given_file_uploader_and_page_size_reduction_then_raise(): action="REDUCE_PAGE_SIZE", pagination_strategy=_CURSOR_PAGINATION_STRATEGY, page_size_option=_PAGE_SIZE_OPTION, + page_token_option="", ) + """file_uploader: type: FileUploader diff --git a/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py b/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py index 2df59d6fde..605ebe97a9 100644 --- a/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py +++ b/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py @@ -284,3 +284,50 @@ def test_given_misconfiguration_then_message_names_the_stream_and_keeps_remediat assert exception.value.failure_type == FailureType.config_error assert A_STREAM_NAME in exception.value.message assert exception.value.message.rstrip().endswith(".") + + +@pytest.mark.parametrize( + "reducer_kwargs,reductions", + [ + pytest.param({"configured_page_size": 4, "minimum_page_size": 2}, 1, id="minimum_reached"), + pytest.param( + {"configured_page_size": 1000, "max_attempts": 2}, 2, id="max_attempts_exhausted" + ), + ], +) +def test_given_failure_message_when_reduction_fails_then_append_it(reducer_kwargs, reductions): + # The CDK only knows that the API rejected every page size it asked for; what narrows a query down is + # API-specific, so the connector supplies that sentence. + reducer = _reducer(failure_message="Select fewer fields on this stream.", **reducer_kwargs) + for _ in range(reductions): + reducer.reduce() + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.transient_error + assert exception.value.message.endswith(" Select fewer fields on this stream.") + + +def test_given_no_failure_message_when_reduction_fails_then_message_ends_with_the_cdk_sentence(): + reducer = _reducer(configured_page_size=4, minimum_page_size=2) + reducer.reduce() + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.message.endswith("records per page).") + + +def test_given_failure_message_when_misconfigured_then_do_not_append_it(): + # A `config_error` is about the manifest, not about the API rejecting a page size, so the connector's + # sentence about narrowing the query down would be misleading there. + reducer = _reducer( + configured_page_size=None, failure_message="Select fewer fields on this stream." + ) + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.config_error + assert "Select fewer fields" not in exception.value.message From e11e676a1024f4550f8d0e9c77c135c210dbc75c Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Thu, 17 Sep 2026 13:16:19 +0300 Subject: [PATCH 08/13] fix: stop assuming a condition without last_page_size is reduction-safe, and make the reduction budget mean what it says Seven findings from the code review of this PR. F1, the blocker. `classify_stop_condition` returned SAFE for every `stop_condition` that did not name `last_page_size`, but the response body carries the same count: `{{ response['data'] | length < 100 }}` is `{{ last_page_size < 100 }}` counted one layer out, and it truncates in exactly the same way. Which of a connector's own response fields holds a page length is not knowable at config time, so a condition that bounds any value from above is now reported as UNKNOWN - a warning, which is what this analysis already does for every shape it cannot read. A condition that makes no upper-bound comparison, the common `{{ not response.next }}` shape, cannot observe a page length by comparison and stays SAFE. Measured over the monorepo at `09124c5aabc`: of 1505 conditions, 1466 stay accepted, 3 rejected and 36 now warn - the 36 are the response-derived page lengths in source-pardot, source-zendesk-chat and source-zendesk-talk, which is the shape this is about. All three of source-github's conditions stay accepted. F2. `max_attempts` counts the reductions made in a row without a page getting through, which is what the terminal error claims happened - but `on_successful_page` only restarted the budget under `AFTER_SUCCESSFUL_PAGE`, so under the default `NEVER` a partition in which every reduction was followed by a successful page still failed at the `max_attempts + 1`-th reduction, blaming the API for rejecting every page size it had in fact served. That is the same defect as the `MAX_TOTAL_REDUCTIONS` cap this PR removed, with the cliff at the 6th reduction instead of the 1001st page, and it is reachable exactly when per-page cost varies - the GraphQL case the feature exists for. The budget now restarts under both policies; `reset_policy` only decides whether the page size is restored. Termination still holds: under `NEVER` the page size strictly decreases, so `minimum_page_size` bounds the partition on its own. F3. Each reduction divides the page size by `reduction_factor` and spends one attempt, so an unbroken run of failures bottoms out at `page_size / reduction_factor ** max_attempts` whatever `minimum_page_size` says: with the defaults a floor of 10 on a page size of 1000 is never reached and its error branch never fires. The factory now warns when the floor is out of reach of the budget, and says how large `max_attempts` would have to be. It only warns when the floor was set explicitly - the default of 1 is out of reach of the default budget on any page size above 32 - and it does not raise, because pages that succeed in between restart the budget, which makes the floor reachable over a partition. F4. The `PageSizeReduction` and `max_attempts` descriptions now say that the wait between reductions is the CDK's own and does not consult the error handler's `backoff_strategies` or a `Retry-After` header. F5. `bin/generate_component_manifest_files.py` now records why `declarative_component_schema.py` is edited by hand for bounded numeric fields: without `--field-constraints` the generator emits `conint`/`confloat` annotations that mypy rejects, on five fields that predate this PR. F6. A rejected page appeared in the Connector Builder auxiliary panel titled like an ordinary page fetch. `_as_auxiliary_request_log` now takes a title and description, and the reduction path passes its own. Co-Authored-By: Claude Opus 5 (1M context) --- .../declarative_component_schema.yaml | 27 +++++--- .../models/declarative_component_schema.py | 6 +- .../parsers/model_to_component_factory.py | 41 +++++++++++- .../parsers/stop_condition_safety.py | 64 ++++++++++++++++-- .../retrievers/page_size_reducer.py | 34 +++++----- .../sources/streams/http/http_client.py | 26 ++++++-- bin/generate_component_manifest_files.py | 8 +++ .../test_model_to_component_factory.py | 49 ++++++++++++++ .../parsers/test_stop_condition_safety.py | 66 ++++++++++++++++++- .../retrievers/test_page_size_reducer.py | 44 +++++++++++-- .../sources/streams/http/test_http_client.py | 6 ++ 11 files changed, 328 insertions(+), 43 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index c8a4e101d8..55f26fe907 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -4312,7 +4312,8 @@ definitions: type RequestPath is rejected too: the next page is then a URL built by the API which already carries the page size, so the reduced page size would be sent next to the original one. Each reduction waits a short, growing amount of time before re-issuing the page so that an endpoint failing at every page size - is not hit in a burst. + is not hit in a burst. That wait is the CDK's own: the error handler's backoff_strategies and any + Retry-After header are not consulted on this path, the same way they are not for RESET_PAGINATION. type: object required: - type @@ -4333,7 +4334,12 @@ definitions: title: Minimum Page Size description: >- Page size below which the connector stops reducing and fails the sync. It must be smaller than the page - size configured on the pagination strategy, otherwise no reduction could ever be applied. + size configured on the pagination strategy, otherwise no reduction could ever be applied. It is one of + two bounds on the reduction and whichever is tighter wins: an unbroken run of failing pages divides the + page size by reduction_factor at most max_attempts times, so reaching this floor in a single run needs + max_attempts of at least log(page_size / minimum_page_size) / log(reduction_factor) - with the defaults, + a page size of 1000 bottoms out at 31 records per page and a floor of 10 is never reached. Pages that + succeed in between restart the max_attempts budget, so the floor is still reachable over a partition. type: integer default: 1 minimum: 1 @@ -4345,10 +4351,13 @@ definitions: description: >- Maximum number of page size reductions made in a row without a single page succeeding, before the sync fails with a transient error. Every reduction follows a request that failed, so at most - max_attempts + 1 failing requests are issued before giving up. With reset_policy NEVER this bounds the - reductions for the whole partition, since the reduced page size is never restored; with - AFTER_SUCCESSFUL_PAGE the budget restarts after every page that succeeds, so it bounds the reductions - needed to get a single page through and not the number of pages a partition may have. + max_attempts + 1 failing requests are issued before giving up. The budget restarts after every page + that succeeds, under either reset_policy, so it bounds the reductions needed to get a single page + through and not the number of pages a partition may have: a stream that needs a reduction every now + and then reads to the end however long it is. Under NEVER the page size also strictly decreases, so + minimum_page_size bounds the reductions of the whole partition on its own. The wait between reduction + attempts is the CDK's own - it grows with each attempt - and does not consult the error handler's + backoff_strategies or a Retry-After header. type: integer default: 5 minimum: 1 @@ -4374,9 +4383,9 @@ definitions: the rest of the partition. AFTER_SUCCESSFUL_PAGE restores it as soon as one page succeeds, which means hitting the same error again on every page - use it only when the reduction is worth one extra request per page, for instance because the configured page size usually works and only some pages are too heavy. - AFTER_SUCCESSFUL_PAGE also restarts the max_attempts budget on every page that succeeds, so there is no - limit on how many reductions a partition may make in total: a stream that needs one reduction per page - reads to the end however many pages it has. What is bounded is the reductions that get no page through. + It only controls the page size: the max_attempts budget restarts on every page that succeeds under both + policies, so there is no limit on how many reductions a partition may make in total. What is bounded is + the reductions that get no page through. type: string enum: - NEVER diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index b91b70109d..d2f4b7a996 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -1429,14 +1429,14 @@ class PageSizeReduction(BaseModel): ) minimum_page_size: Optional[int] = Field( 1, - description="Page size below which the connector stops reducing and fails the sync. It must be smaller than the page size configured on the pagination strategy, otherwise no reduction could ever be applied.", + description="Page size below which the connector stops reducing and fails the sync. It must be smaller than the page size configured on the pagination strategy, otherwise no reduction could ever be applied. It is one of two bounds on the reduction and whichever is tighter wins: an unbroken run of failing pages divides the page size by reduction_factor at most max_attempts times, so reaching this floor in a single run needs max_attempts of at least log(page_size / minimum_page_size) / log(reduction_factor) - with the defaults, a page size of 1000 bottoms out at 31 records per page and a floor of 10 is never reached. Pages that succeed in between restart the max_attempts budget, so the floor is still reachable over a partition.", examples=[1, 10], ge=1, title="Minimum Page Size", ) max_attempts: Optional[int] = Field( 5, - description="Maximum number of page size reductions made in a row without a single page succeeding, before the sync fails with a transient error. Every reduction follows a request that failed, so at most max_attempts + 1 failing requests are issued before giving up. With reset_policy NEVER this bounds the reductions for the whole partition, since the reduced page size is never restored; with AFTER_SUCCESSFUL_PAGE the budget restarts after every page that succeeds, so it bounds the reductions needed to get a single page through and not the number of pages a partition may have.", + description="Maximum number of page size reductions made in a row without a single page succeeding, before the sync fails with a transient error. Every reduction follows a request that failed, so at most max_attempts + 1 failing requests are issued before giving up. The budget restarts after every page that succeeds, under either reset_policy, so it bounds the reductions needed to get a single page through and not the number of pages a partition may have: a stream that needs a reduction every now and then reads to the end however long it is. Under NEVER the page size also strictly decreases, so minimum_page_size bounds the reductions of the whole partition on its own. The wait between reduction attempts is the CDK's own - it grows with each attempt - and does not consult the error handler's backoff_strategies or a Retry-After header.", examples=[5, 10], ge=1, title="Maximum Reduction Attempts", @@ -1452,7 +1452,7 @@ class PageSizeReduction(BaseModel): ) reset_policy: Optional[ResetPolicy] = Field( ResetPolicy.NEVER, - description="When to restore the page size configured on the pagination strategy. NEVER keeps the reduced page size for the rest of the partition. AFTER_SUCCESSFUL_PAGE restores it as soon as one page succeeds, which means hitting the same error again on every page - use it only when the reduction is worth one extra request per page, for instance because the configured page size usually works and only some pages are too heavy. AFTER_SUCCESSFUL_PAGE also restarts the max_attempts budget on every page that succeeds, so there is no limit on how many reductions a partition may make in total: a stream that needs one reduction per page reads to the end however many pages it has. What is bounded is the reductions that get no page through.", + description="When to restore the page size configured on the pagination strategy. NEVER keeps the reduced page size for the rest of the partition. AFTER_SUCCESSFUL_PAGE restores it as soon as one page succeeds, which means hitting the same error again on every page - use it only when the reduction is worth one extra request per page, for instance because the configured page size usually works and only some pages are too heavy. It only controls the page size: the max_attempts budget restarts on every page that succeeds under both policies, so there is no limit on how many reductions a partition may make in total. What is bounded is the reductions that get no page through.", title="Reset Policy", ) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 8404eafbbb..6569359be8 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -9,6 +9,7 @@ import inspect import json import logging +import math import re from functools import partial from typing import ( @@ -3888,8 +3889,11 @@ def _validate_stop_condition_is_reduction_aware( The condition is parsed as a Jinja expression rather than matched as a string: only the AST tells `page_size`, which follows the reduction, apart from `config['page_size']`, which does not, and only the AST tells an inequality, which a reduction can invalidate, apart from `last_page_size == 0`, which it - cannot. A shape the analysis does not understand is warned about rather than rejected - this runs at - stream construction, so a false rejection takes `check`, `discover` and `read` down with it. + cannot. A condition that never names `last_page_size` is not waved through: a count read from the + response body - `{{ response['data'] | length < 100 }}` - truncates in exactly the same way, and which + response field counts the records of a page is not knowable here. A shape the analysis does not + understand is warned about rather than rejected - this runs at stream construction, so a false + rejection takes `check`, `discover` and `read` down with it. """ stop_condition = strategy.stop_condition if not stop_condition: @@ -3949,6 +3953,39 @@ def _validate_page_size_is_reducible( f"{minimum_page_size}. Lower `minimum_page_size` or raise `page_size`." ) + # Each reduction divides the page size by `reduction_factor` and spends one attempt, so an unbroken + # run of failures bottoms out at `page_size / reduction_factor ** max_attempts` whatever + # `minimum_page_size` says. Only warned about, and not raised: pages that succeed in between restart + # the budget while `NEVER` keeps the page size, so the floor is reachable over a partition even when + # it is out of reach of a single run - and a manifest that deliberately gives up earlier than its + # floor is not wrong, only worth pointing out. + reduction_factor = ( + page_size_reduction.reduction_factor if page_size_reduction else None + ) or 2.0 + max_attempts = (page_size_reduction.max_attempts if page_size_reduction else None) or 5 + # Only when the floor was asked for: the default of 1 is out of reach of the default budget on any page + # size above 32, so warning about a floor the author never set would fire on nearly every stream. + floor_was_set = bool( + page_size_reduction and "minimum_page_size" in page_size_reduction.__fields_set__ + ) + if ( + configured_page_size is not None + and floor_was_set + and reduction_factor**max_attempts < configured_page_size / minimum_page_size + ): + reachable_page_size = max( + minimum_page_size, int(configured_page_size // reduction_factor**max_attempts) + ) + LOGGER.warning( + f"Stream {name} sets `minimum_page_size` to {minimum_page_size}, which `page_size_reduction` " + f"cannot reach in one run of failing pages: `max_attempts` is {max_attempts} and " + f"`reduction_factor` is {reduction_factor}, so {max_attempts} reductions of a page size of " + f"{configured_page_size} stop at {reachable_page_size} records per page and the sync then " + f"fails with a transient error. Whichever of the two bounds is tighter wins; reaching " + f"{minimum_page_size} in one run needs `max_attempts` of at least " + f"{math.ceil(math.log(configured_page_size / minimum_page_size, reduction_factor))}." + ) + def _reject_reduce_page_size_action(self, requester: Any, description: str) -> None: """ `error_handler` is defined on `HttpRequester`, which is referenced by requesters that have no page of diff --git a/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py b/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py index 6da834ee2b..ad840247e0 100644 --- a/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py +++ b/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py @@ -36,6 +36,12 @@ root through `and`/`or`, or as the test of a `{% if %}` that renders truthy text and nothing else, and with `last_page_size` compared bare rather than transformed first. Anything else is unclassifiable. +A condition that never names `last_page_size` is not safe by default either: the response body carries the +same count, so `{{ response['data'] | length < 100 }}` truncates in exactly the same way. Which of a +connector's own response fields counts the records of a page is not knowable here, so any comparison that +bounds something from above is reported as unclassifiable. A condition with no such comparison, such as +`{{ not response.next }}`, cannot read a page length by comparison and is safe. + Anything else is reported as unclassifiable rather than as a truncation: this analysis rejects a manifest at stream construction, so a shape it does not understand must not be treated as a defect. """ @@ -96,10 +102,7 @@ def classify_stop_condition( ) if not _references(template, LAST_PAGE_SIZE_VARIABLE): - return ( - StopConditionSafety.SAFE, - f"it does not use `{LAST_PAGE_SIZE_VARIABLE}`, so the page size it was requested with is irrelevant", - ) + return _classify_without_last_page_size(template) unclassifiable: List[str] = [] understood = 0 @@ -137,6 +140,59 @@ def classify_stop_condition( ) +def _classify_without_last_page_size( + template: nodes.Template, +) -> Tuple[StopConditionSafety, str]: + """ + Classify a condition that never names `last_page_size`. + + `last_page_size` is not the only way to observe how many records a page held: the response body carries + the same number, and `{{ response['data'] | length < 100 }}` is the comparison `{{ last_page_size < 100 }}` + counted one layer out. The CDK cannot tell which of a connector's own response fields counts the records + of the page, so a condition that bounds *anything* from above is reported as unclassifiable and warned + about rather than assumed to be safe. A condition that makes no such comparison - the common + `{{ not response.next }}` shape - cannot read a page length by comparison at all, so it stays safe. + """ + for left, operator, right, decides_condition in _comparisons(template): + bounded = _bounded_from_above(left, operator, right, decides_condition) + if bounded is not None: + return ( + StopConditionSafety.UNKNOWN, + f"it stops when {_describe(bounded)} stays below a threshold. `{LAST_PAGE_SIZE_VARIABLE}` is " + f"not the only way to count the records of a page - a count read from the response body is " + f"another - and the connector cannot tell whether this one does, so a full page at a reduced " + f"size may read as a short page here", + ) + return ( + StopConditionSafety.SAFE, + f"it does not use `{LAST_PAGE_SIZE_VARIABLE}` and makes no comparison that could be bounding the " + f"number of records in a page from above", + ) + + +def _bounded_from_above( + left: nodes.Node, operator: str, right: nodes.Node, decides_condition: bool +) -> Optional[nodes.Node]: + """ + :return: the operand the comparison bounds from above, or None when it bounds nothing from above + + An upper bound is the shape a reduction can invalidate, for the same reason it can invalidate + `last_page_size < 100`: the reduction lowers what a full page holds, so the bound starts holding for pages + that are not short at all. A lower bound can only stop being satisfied as pages get smaller. When the + comparison does not decide the condition in its own polarity, which of the two it is cannot be read off + the operator, so it counts as an upper bound. + """ + if operator not in ("lt", "lteq", "gt", "gteq"): + return None + if not decides_condition: + return left if not isinstance(left, nodes.Const) else right + if operator in ("lt", "lteq"): + # `x < 100` bounds x from above; `100 < x` is a lower bound on x. + return None if isinstance(left, nodes.Const) else left + # `100 > x` is the mirror of `x < 100`; `x > 100` is a lower bound on x. + return right if isinstance(left, nodes.Const) else None + + def _comparisons( node: nodes.Node, decides_condition: bool = True ) -> Iterator[Tuple[nodes.Node, str, nodes.Node, bool]]: diff --git a/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py b/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py index b8305e6ba0..92dd7794fd 100644 --- a/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py +++ b/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py @@ -120,9 +120,9 @@ def reduce(self) -> None: self._total_reductions += 1 if self._attempts > self._config.max_attempts: # The budget counts the reductions that did *not* get a page through, which is what separates a - # partition that is stuck from one that is merely expensive. A partition where every page succeeds - # after a reduction resets this counter on each page and reads to the end, however many pages it - # has; a partition where nothing gets through burns the budget and fails here. + # partition that is stuck from one that is merely expensive. A partition where pages keep + # succeeding restarts this counter on each of them, under either reset policy, and reads to the + # end however many pages it has; a partition where nothing gets through burns the budget here. raise AirbyteTracedException( internal_message=f"Stream {self._stream_name} reduced its page size {self._attempts - 1} times in a row without a single page succeeding, which is the configured maximum of {self._config.max_attempts} ({self._total_reductions - 1} reductions so far while reading this partition)", # `transient_error`, so the only remediation is the connector's own, if it defined one. @@ -180,25 +180,29 @@ def on_successful_page(self) -> None: """ Called after each page that did not require a reduction. - Under `NEVER` nothing happens: the reduced page size stays in effect and `max_attempts` keeps bounding - the reductions for the whole partition, which is the right budget when reductions are one-off. + The `max_attempts` budget restarts under both policies. It counts the reductions made *in a row* + without a single page succeeding, which is what separates a partition that is stuck from one that is + merely expensive: on a stream whose per-page cost varies - the GraphQL case this feature exists for - + a handful of heavy pages spread over a long partition is a healthy read, and a budget spanning the + whole partition would fail it at the `max_attempts + 1`-th heavy page while every reduction so far had + been followed by a successful page. The terminal message says the source rejected every page size the + connector asked for, so the budget has to mean exactly that. - Under `AFTER_SUCCESSFUL_PAGE` the page size is restored and the `max_attempts` budget restarts. This - policy exists for an API that rejects the configured page size on every page, so every page legitimately - costs one reduction, and a budget spanning the whole partition would fail the sync at page - `max_attempts + 1` no matter how healthy the reads are. There is deliberately no partition-wide cap on - top of it: a stream where every page gets through is healthy and has to sync to completion, and a cap - would only move the same cliff further out. - - The budget still terminates the read, because only a page that succeeded can restart it and only + The budget still terminates the read, because only a page that succeeded restarts it and only `_read_pages` calls this, once per page it consumed. So between any two restarts the partition made one page of progress, and the reductions that make no progress are bounded by `max_attempts`. Under `NEVER` - the reduced page size is also never restored, so `minimum_page_size` bounds the reductions on its own. + the reduced page size is never restored either, so it strictly decreases and `minimum_page_size` bounds + the reductions of the whole partition on its own. + + Only `AFTER_SUCCESSFUL_PAGE` restores the page size. That policy is for an API that rejects the + configured page size on every page, so every page legitimately costs one reduction; `NEVER` keeps the + reduced size for the rest of the partition, which is the right behaviour when reductions are one-off. """ + self._attempts = 0 + if self._config.reset_policy != PageSizeResetPolicy.AFTER_SUCCESSFUL_PAGE: return - self._attempts = 0 if self._current_page_size is not None: LOGGER.info( f"Restoring the page size of stream {self._stream_name} from {self._current_page_size} to {self._configured_page_size}." diff --git a/airbyte_cdk/sources/streams/http/http_client.py b/airbyte_cdk/sources/streams/http/http_client.py index 825a59d4d2..caacdd503a 100644 --- a/airbyte_cdk/sources/streams/http/http_client.py +++ b/airbyte_cdk/sources/streams/http/http_client.py @@ -98,19 +98,27 @@ def monkey_patched_get_item(self, key): # type: ignore # this interface is a co requests_cache.SQLiteDict.__getitem__ = monkey_patched_get_item # type: ignore # see the method doc for more information -def _as_auxiliary_request_log(log_message: Any) -> Any: +def _as_auxiliary_request_log( + log_message: Any, title: Optional[str] = None, description: Optional[str] = None +) -> Any: """ Flag an already-formatted request/response log as an auxiliary request. The Connector Builder builds one page per non-auxiliary HTTP log and bounds a slice by the number of those - pages, so a request that will not produce a page has to be marked here or it inflates that count. The log - formatter is connector-supplied and only the CDK's own one is guaranteed to have an `http` object, hence - the defensive check. + pages, so a request that will not produce a page has to be marked here or it inflates that count. The + Builder also labels its side panel from the log's `title` and `description`, which the formatter filled + with the wording of an ordinary page, so a caller that knows why the request is auxiliary passes its own + and the panel does not read as a successful page fetch. The log formatter is connector-supplied and only + the CDK's own one is guaranteed to have an `http` object, hence the defensive check. """ if isinstance(log_message, dict): http = log_message.get("http") if isinstance(http, dict): http["is_auxiliary"] = True + if title is not None: + http["title"] = title + if description is not None: + http["description"] = description return log_message @@ -473,7 +481,15 @@ def _send( log_as_auxiliary = error_resolution.response_action == ResponseAction.REDUCE_PAGE_SIZE self._message_repository.log_message( Level.DEBUG, - lambda: _as_auxiliary_request_log(formatter(response)) + lambda: _as_auxiliary_request_log( + formatter(response), + title=f"Stream '{self._name}' page rejected, retrying with a smaller page size", + description=( + f"Request for stream '{self._name}' whose response asked for a smaller page. The " + f"same page is requested again with a reduced page size, so this request produced " + f"no records." + ), + ) if log_as_auxiliary else formatter(response), ) diff --git a/bin/generate_component_manifest_files.py b/bin/generate_component_manifest_files.py index 51b3d8efbf..098c108d3f 100755 --- a/bin/generate_component_manifest_files.py +++ b/bin/generate_component_manifest_files.py @@ -138,6 +138,14 @@ async def main(): "--set-default-enum-member", "--use-double-quotes", "--remove-special-field-name-prefix", + # NOTE: without `--field-constraints`, a numeric `minimum`/`exclusiveMinimum` in the YAML + # becomes a `conint(...)`/`confloat(...)` annotation, which mypy rejects - so regenerating + # today produces a file that does not type check, on fields that predate this comment + # (DynamicStreamCheckConfig.stream_count, both backoff strategies, AsyncRetriever, and + # PageSizeReduction). Adding the flag rewrites those fields across the generated module + # and is its own change; until then `declarative_component_schema.py` is edited by hand + # when a bounded numeric field is added, with the bound expressed as `Field(ge=...)`. + # The YAML bounds are what manifests are validated against either way. # allow usage of the extra key such as `deprecated`, etc. "--field-extra-keys", # account the `deprecated` flag provided for the field. diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index bef12d45c1..08ad233dfb 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -7038,6 +7038,55 @@ def test_given_page_size_reduction_without_reduce_page_size_action_then_warn(cap assert "REDUCE_PAGE_SIZE" in caplog.text +def test_given_minimum_page_size_out_of_reach_of_max_attempts_then_warn(caplog): + """ + Each reduction divides the page size by `reduction_factor` and spends one attempt, so the two settings are + two bounds and the tighter one wins. A floor the budget cannot reach in one run of failing pages is inert + there, and the error branch written for hitting it never fires on that run. + """ + with caplog.at_level(logging.WARNING, logger="airbyte.model_to_component_factory"): + retriever = get_retriever( + _page_size_reduction_stream( + page_size_reduction=( + "page_size_reduction:\n" + " type: PageSizeReduction\n" + " minimum_page_size: 10\n" + " max_attempts: 2" + ) + ) + ) + + assert retriever.page_size_reduction.minimum_page_size == 10 + assert "minimum_page_size" in caplog.text + assert "25 records per page" in caplog.text + assert "`max_attempts` of at least 4" in caplog.text + + +def test_given_minimum_page_size_within_reach_of_max_attempts_then_do_not_warn(caplog): + with caplog.at_level(logging.WARNING, logger="airbyte.model_to_component_factory"): + get_retriever( + _page_size_reduction_stream( + page_size_reduction=( + "page_size_reduction:\n" + " type: PageSizeReduction\n" + " minimum_page_size: 10\n" + " max_attempts: 5" + ) + ) + ) + + assert "minimum_page_size" not in caplog.text + + +def test_given_default_minimum_page_size_then_do_not_warn_about_its_reachability(caplog): + # The default floor of 1 is out of reach of the default budget on any page size above 32, so warning about + # a floor the author never set would fire on nearly every stream that opts in. + with caplog.at_level(logging.WARNING, logger="airbyte.model_to_component_factory"): + get_retriever(_page_size_reduction_stream()) + + assert "minimum_page_size" not in caplog.text + + def test_given_query_properties_and_page_size_reduction_then_raise(): """ Records of the earlier property chunks were already emitted when a later chunk asks for a smaller page, so diff --git a/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py b/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py index 70503b51ec..166d237006 100644 --- a/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py +++ b/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py @@ -5,6 +5,7 @@ import pytest from airbyte_cdk.sources.declarative.parsers.stop_condition_safety import ( + LAST_PAGE_SIZE_VARIABLE, StopConditionSafety, classify_stop_condition, ) @@ -95,10 +96,66 @@ pytest.param( "{{ last_page_size >= 1000 }}", 1, StopConditionSafety.SAFE, id="inclusive_lower_bound" ), - # A condition that never looks at the page size is unaffected by the reduction. + # A condition that cannot observe the length of a page is unaffected by the reduction, whatever else + # it reads from the response. pytest.param( "{{ not response.next }}", 1, StopConditionSafety.SAFE, id="no_last_page_size" ), + pytest.param( + "{{ response.next is none }}", 1, StopConditionSafety.SAFE, id="a_test_on_the_response" + ), + pytest.param( + "{{ response.page >= response.total_pages }}", + 1, + StopConditionSafety.SAFE, + id="a_lower_bound_on_a_response_value", + ), + pytest.param( + "{{ 100 < response.count }}", + 1, + StopConditionSafety.SAFE, + id="a_lower_bound_on_a_response_value_reversed", + ), + # `last_page_size` is not the only way to count the records of a page: the response body carries the + # same number, and these are `{{ last_page_size < 100 }}` counted one layer out. Which response field + # holds a page length is not knowable here, so an upper bound on any value is warned about rather than + # assumed to be safe. All four shapes below are live in the fleet today. + pytest.param( + "{{ response['values']|length < 1000 }}", + 1, + StopConditionSafety.UNKNOWN, + id="a_page_length_read_from_the_response", + ), + pytest.param( + '{{ response.get("count", 0) < 1000 }}', + 1, + StopConditionSafety.UNKNOWN, + id="a_count_read_from_the_response", + ), + pytest.param( + "{{ response.data | length < config['page_size'] }}", + 1, + StopConditionSafety.UNKNOWN, + id="a_page_length_against_a_config_value", + ), + pytest.param( + "{{ not response.result.emailClick or response.result.emailClick|length < 200 }}", + 1, + StopConditionSafety.UNKNOWN, + id="a_page_length_in_a_larger_expression", + ), + pytest.param( + "{{ 100 > response.data | length }}", + 1, + StopConditionSafety.UNKNOWN, + id="a_page_length_read_from_the_response_reversed", + ), + pytest.param( + "{{ not (response.data | length >= 100) }}", + 1, + StopConditionSafety.UNKNOWN, + id="a_negated_lower_bound_on_a_response_value", + ), # Shapes the analysis cannot reason about are reported as unknown so the caller can warn rather than # reject: this runs at stream construction, where a false rejection also breaks `check` and `discover`. pytest.param( @@ -280,3 +337,10 @@ def test_given_truncating_comparison_next_to_a_negated_one_then_truncates(): ) assert verdict is StopConditionSafety.TRUNCATES + + +def test_given_page_length_read_from_the_response_then_reason_names_the_expression(): + verdict, reason = classify_stop_condition("{{ response['values']|length < 1000 }}", 1) + + assert verdict is StopConditionSafety.UNKNOWN + assert LAST_PAGE_SIZE_VARIABLE in reason diff --git a/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py b/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py index 605ebe97a9..113dcb5969 100644 --- a/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py +++ b/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py @@ -162,16 +162,52 @@ def test_given_reset_policy_after_successful_page_when_no_page_succeeds_then_max assert "2 times in a row without a single page succeeding" in exception.value.internal_message -def test_given_reset_policy_never_when_pages_succeed_then_attempts_are_not_reset(): +def test_given_reset_policy_never_when_pages_succeed_then_attempts_are_reset(): + """ + `max_attempts` counts the reductions made in a row without a page getting through, which is what the + terminal error claims happened, so a page that succeeded has to restart it under this policy too. A stream + whose per-page cost varies - the GraphQL case this feature exists for - would otherwise fail at the + `max_attempts + 1`-th heavy page of a long partition in which every reduction was followed by a page. + """ + reducer = _reducer(configured_page_size=1000, max_attempts=2) + + for expected_page_size in [500, 250, 125, 62, 31, 15, 7, 3, 1]: + reducer.reduce() + assert reducer.page_size_override == expected_page_size + # the page size is not restored under NEVER, only the budget is + reducer.on_successful_page() + assert reducer.page_size_override == expected_page_size + + +def test_given_reset_policy_never_when_pages_succeed_then_minimum_page_size_still_ends_the_read(): + """ + With the budget restarting, `minimum_page_size` is what bounds a NEVER partition: the page size strictly + decreases, so the read cannot go on forever. + """ + reducer = _reducer(configured_page_size=100, minimum_page_size=10, max_attempts=2) + + for expected_page_size in [50, 25, 12, 10]: + reducer.reduce() + assert reducer.page_size_override == expected_page_size + reducer.on_successful_page() + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.transient_error + assert "smallest page size" in exception.value.message + + +def test_given_no_page_succeeds_then_attempts_are_not_reset(): reducer = _reducer(max_attempts=2) reducer.reduce() - reducer.on_successful_page() reducer.reduce() - reducer.on_successful_page() - with pytest.raises(AirbyteTracedException): + with pytest.raises(AirbyteTracedException) as exception: reducer.reduce() + assert "2 times in a row without a single page succeeding" in exception.value.internal_message + def test_given_reset_policy_after_successful_page_when_every_page_succeeds_then_never_fail(): """ diff --git a/unit_tests/sources/streams/http/test_http_client.py b/unit_tests/sources/streams/http/test_http_client.py index bf2832093b..14ddcaf7a6 100644 --- a/unit_tests/sources/streams/http/test_http_client.py +++ b/unit_tests/sources/streams/http/test_http_client.py @@ -1531,6 +1531,12 @@ def test_given_reduce_page_size_action_then_log_the_response_as_an_auxiliary_req logged = [json.loads(message.log.message) for message in message_repository.consume_queue()] assert [entry["http"]["is_auxiliary"] for entry in logged] == [True] + # The Builder labels its auxiliary panel from these two, and the formatter filled them with the wording of + # an ordinary page, so a rejected request would otherwise be indistinguishable from a successful fetch. + assert logged[0]["http"]["title"] == ( + "Stream 'test' page rejected, retrying with a smaller page size" + ) + assert "no records" in logged[0]["http"]["description"] def test_given_no_reduce_page_size_action_then_log_the_response_as_a_page(): From f1a23bc80dc3975b2bd5fa686a2aa0301e74767a Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Thu, 17 Sep 2026 13:56:58 +0300 Subject: [PATCH 09/13] fix: mirror both operator directions in the stop-condition gate, and only warn about a floor worth reaching Two findings from the second round of review. R2-F1. `_bounded_from_above` read `lt`/`lteq` in full - any non-literal left operand counts as bounded - but read `gt`/`gteq` only when the threshold was a literal, so `{{ config['page_size'] > response['data'] | length }}` stayed classified as safe while `{{ response['data'] | length < config['page_size'] }}` was warned about. Same comparison, same silent truncation, and the module docstring promises that any comparison bounding a value from above is reported. Every one of the four ordering operators bounds one of its operands from above and only a literal is certain not to be a page length, so both directions are now mirrored, as `_MIRRORED_OPERATORS` already does on the `last_page_size` path. `{{ response.page >= response.total_pages }}` warns as a result: in `a >= b` there is an upper bound on `b`, and the earlier reading of that shape as "no upper bound, so nothing here can be a page length" was wrong. Measured over the monorepo at `09124c5aabc`: of 1505 conditions, 1459 stay accepted, 3 rejected and 43 warn, up from 36. The 7 newly warned are the two shapes the mirror adds, source-serpstat x6 and source-jira x1; neither bounds a page length, and neither connector opts into `page_size_reduction`. All three of source-github's conditions, on the adopter branch, stay accepted. R2-F2. The floor-reachability warning gated on `__fields_set__`, which records that a value was supplied and not that it differs from the default, so spelling `minimum_page_size: 1` out longhand earned a warning advising `max_attempts: 10` on any page size above 32. It now gates on the floor being above 1, which is the condition that makes the floor worth reaching in the first place. Co-Authored-By: Claude Opus 5 (1M context) --- .../parsers/model_to_component_factory.py | 11 ++++--- .../parsers/stop_condition_safety.py | 13 +++++++-- .../test_model_to_component_factory.py | 20 ++++++++++--- .../parsers/test_stop_condition_safety.py | 29 ++++++++++++++++--- 4 files changed, 56 insertions(+), 17 deletions(-) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 6569359be8..4fd41a4159 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -3963,14 +3963,13 @@ def _validate_page_size_is_reducible( page_size_reduction.reduction_factor if page_size_reduction else None ) or 2.0 max_attempts = (page_size_reduction.max_attempts if page_size_reduction else None) or 5 - # Only when the floor was asked for: the default of 1 is out of reach of the default budget on any page - # size above 32, so warning about a floor the author never set would fire on nearly every stream. - floor_was_set = bool( - page_size_reduction and "minimum_page_size" in page_size_reduction.__fields_set__ - ) + # Only when there is a floor worth reaching. The default of 1 is out of reach of the default budget on + # any page size above 32, so this would otherwise fire on nearly every stream that opts in - and + # `__fields_set__` would not help, since it records that a value was supplied and not that it differs + # from the default, so spelling `minimum_page_size: 1` out longhand would earn the warning. if ( configured_page_size is not None - and floor_was_set + and minimum_page_size > 1 and reduction_factor**max_attempts < configured_page_size / minimum_page_size ): reachable_page_size = max( diff --git a/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py b/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py index ad840247e0..ed0e47f681 100644 --- a/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py +++ b/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py @@ -181,16 +181,23 @@ def _bounded_from_above( that are not short at all. A lower bound can only stop being satisfied as pages get smaller. When the comparison does not decide the condition in its own polarity, which of the two it is cannot be read off the operator, so it counts as an upper bound. + + Every comparison with one of these four operators bounds one of its operands from above - `a < b` bounds + `a`, `b > a` bounds `a` too - and a literal cannot be a page length, so the bounded side only escapes when + it is one. The two operator directions are therefore mirrored, the way `_MIRRORED_OPERATORS` mirrors them + on the `last_page_size` path: reading only `lt`/`lteq` in full left + `{{ config['page_size'] > response['data'] | length }}` classified as safe while + `{{ response['data'] | length < config['page_size'] }}` was warned about. """ if operator not in ("lt", "lteq", "gt", "gteq"): return None if not decides_condition: return left if not isinstance(left, nodes.Const) else right if operator in ("lt", "lteq"): - # `x < 100` bounds x from above; `100 < x` is a lower bound on x. + # `x < 100` bounds x from above; `100 < x` bounds the literal, which is not a page length. return None if isinstance(left, nodes.Const) else left - # `100 > x` is the mirror of `x < 100`; `x > 100` is a lower bound on x. - return right if isinstance(left, nodes.Const) else None + # `100 > x` is the mirror of `x < 100`; `x > 100` bounds the literal again. + return None if isinstance(right, nodes.Const) else right def _comparisons( diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index 08ad233dfb..0b03204513 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -7078,11 +7078,23 @@ def test_given_minimum_page_size_within_reach_of_max_attempts_then_do_not_warn(c assert "minimum_page_size" not in caplog.text -def test_given_default_minimum_page_size_then_do_not_warn_about_its_reachability(caplog): - # The default floor of 1 is out of reach of the default budget on any page size above 32, so warning about - # a floor the author never set would fire on nearly every stream that opts in. +@pytest.mark.parametrize( + "page_size_reduction", + [ + pytest.param("page_size_reduction:\n type: PageSizeReduction", id="floor_left_default"), + pytest.param( + "page_size_reduction:\n type: PageSizeReduction\n minimum_page_size: 1", + id="floor_set_to_the_default_value", + ), + ], +) +def test_given_no_floor_worth_reaching_then_do_not_warn_about_its_reachability( + page_size_reduction, caplog +): + # A floor of 1 is out of reach of the default budget on any page size above 32, so warning about it would + # fire on nearly every stream that opts in - including one that only spells the default out longhand. with caplog.at_level(logging.WARNING, logger="airbyte.model_to_component_factory"): - get_retriever(_page_size_reduction_stream()) + get_retriever(_page_size_reduction_stream(page_size_reduction=page_size_reduction)) assert "minimum_page_size" not in caplog.text diff --git a/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py b/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py index 166d237006..988509ac83 100644 --- a/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py +++ b/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py @@ -105,16 +105,16 @@ "{{ response.next is none }}", 1, StopConditionSafety.SAFE, id="a_test_on_the_response" ), pytest.param( - "{{ response.page >= response.total_pages }}", + "{{ 100 < response.count }}", 1, StopConditionSafety.SAFE, - id="a_lower_bound_on_a_response_value", + id="a_literal_bounded_from_above", ), pytest.param( - "{{ 100 < response.count }}", + "{{ response.count > 100 }}", 1, StopConditionSafety.SAFE, - id="a_lower_bound_on_a_response_value_reversed", + id="a_literal_bounded_from_above_reversed", ), # `last_page_size` is not the only way to count the records of a page: the response body carries the # same number, and these are `{{ last_page_size < 100 }}` counted one layer out. Which response field @@ -156,6 +156,27 @@ StopConditionSafety.UNKNOWN, id="a_negated_lower_bound_on_a_response_value", ), + # Both operator directions are mirrored: every one of the four ordering operators bounds one of its + # operands from above, and only a literal is certain not to be a page length. Reading `lt`/`lteq` in + # full while `gt`/`gteq` only counted a literal threshold left the first two of these classified safe. + pytest.param( + "{{ config['page_size'] > response['data'] | length }}", + 1, + StopConditionSafety.UNKNOWN, + id="a_page_length_under_a_config_threshold_on_the_left", + ), + pytest.param( + "{{ response.total > response['data'] | length }}", + 1, + StopConditionSafety.UNKNOWN, + id="a_page_length_under_a_response_threshold_on_the_left", + ), + pytest.param( + "{{ response.page >= response.total_pages }}", + 1, + StopConditionSafety.UNKNOWN, + id="two_response_values_compared", + ), # Shapes the analysis cannot reason about are reported as unknown so the caller can warn rather than # reject: this runs at stream construction, where a false rejection also breaks `check` and `discover`. pytest.param( From 4b3cff67ce82f00bb24fed81e8b2e36155b4a8d6 Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Thu, 17 Sep 2026 14:34:55 +0300 Subject: [PATCH 10/13] fix(page size reduction): make the backoff configurable and allow retries at the minimum page size Two gaps found while adopting `page_size_reduction` in source-github, where GitHub answers 502 both to a GraphQL page that is too expensive and to a transient hiccup: - The wait between attempts was a hard-coded 0.5s times the attempt count, so a run of five reductions spread six requests over ~7s. `backoff_seconds` makes it a knob; the default keeps today's behaviour. - Once the page size reached `minimum_page_size`, the first response asking for a smaller page ended the stream. `REDUCE_PAGE_SIZE` bypasses the HTTP retry budget, so at the floor a stream got fewer attempts than it had before adopting the reduction. `retries_at_minimum_page_size` re-issues the same page unchanged, after the backoff, in a budget of its own, separate from `max_attempts` (which only counts reductions) and restarted by every successful page. The default of 0 keeps today's behaviour. The `config_error` raised when the page size can never be reduced is now limited to the case where `minimum_page_size` is what blocks it, which is the only one the user can act on. A stream whose configured page size is already 1 gets the transient error instead: one record per page is as small as a page gets, so the API rejecting it says nothing about the configuration. Co-Authored-By: Claude Opus 5 (1M context) --- .../declarative_component_schema.yaml | 32 ++++- .../models/declarative_component_schema.py | 14 +++ .../parsers/model_to_component_factory.py | 2 + .../retrievers/page_size_reducer.py | 114 +++++++++++++----- .../retrievers/simple_retriever.py | 7 +- .../test_connector_builder_handler.py | 5 +- .../retrievers/test_page_size_reducer.py | 86 ++++++++++++- .../retrievers/test_simple_retriever.py | 59 ++++++++- .../test_concurrent_declarative_source.py | 3 +- 9 files changed, 273 insertions(+), 49 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 55f26fe907..d13a3a48c7 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -4313,7 +4313,8 @@ definitions: page size, so the reduced page size would be sent next to the original one. Each reduction waits a short, growing amount of time before re-issuing the page so that an endpoint failing at every page size is not hit in a burst. That wait is the CDK's own: the error handler's backoff_strategies and any - Retry-After header are not consulted on this path, the same way they are not for RESET_PAGINATION. + Retry-After header are not consulted on this path, the same way they are not for RESET_PAGINATION, and + backoff_seconds is what sets its length. type: object required: - type @@ -4364,6 +4365,35 @@ definitions: examples: - 5 - 10 + backoff_seconds: + title: Backoff Seconds + description: >- + Base number of seconds to wait before the page is re-issued, multiplied by the number of attempts made in + a row, so the second attempt waits twice as long as the first. A REDUCE_PAGE_SIZE response never reaches + the error handler's retry budget, backoff_strategies or a Retry-After header, so this is the only thing + spacing those requests out. Raise it on an API whose error also means "we are briefly unwell" rather than + only "your page is too big", since the default spaces the whole run of attempts over a few seconds. + type: number + default: 0.5 + minimum: 0 + examples: + - 0.5 + - 5 + retries_at_minimum_page_size: + title: Retries At Minimum Page Size + description: >- + Number of times the same page is re-issued unchanged, each after the backoff wait, once the page size + cannot be shrunk any further, before the sync fails with a transient error. The default of 0 fails on the + first response received at minimum_page_size. Raise it when the API returns the same error for a page that + is too big and for a server-side hiccup: at the floor, reducing is no longer an option but waiting still + is, and without this budget those responses end the stream on the first one. This budget is separate from + max_attempts, which only counts reductions, and it restarts on every page that succeeds. + type: integer + default: 0 + minimum: 0 + examples: + - 0 + - 3 failure_message: title: Failure Message description: >- diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index d2f4b7a996..3468d8b7e2 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -1441,6 +1441,20 @@ class PageSizeReduction(BaseModel): ge=1, title="Maximum Reduction Attempts", ) + backoff_seconds: Optional[float] = Field( + 0.5, + description='Base number of seconds to wait before the page is re-issued, multiplied by the number of attempts made in a row, so the second attempt waits twice as long as the first. A REDUCE_PAGE_SIZE response never reaches the error handler\'s retry budget, backoff_strategies or a Retry-After header, so this is the only thing spacing those requests out. Raise it on an API whose error also means "we are briefly unwell" rather than only "your page is too big", since the default spaces the whole run of attempts over a few seconds.', + examples=[0.5, 5], + ge=0.0, + title="Backoff Seconds", + ) + retries_at_minimum_page_size: Optional[int] = Field( + 0, + description="Number of times the same page is re-issued unchanged, each after the backoff wait, once the page size cannot be shrunk any further, before the sync fails with a transient error. The default of 0 fails on the first response received at minimum_page_size. Raise it when the API returns the same error for a page that is too big and for a server-side hiccup: at the floor, reducing is no longer an option but waiting still is, and without this budget those responses end the stream on the first one. This budget is separate from max_attempts, which only counts reductions, and it restarts on every page that succeeds.", + examples=[0, 3], + ge=0, + title="Retries At Minimum Page Size", + ) failure_message: Optional[str] = Field( None, description="Sentence appended to the error message shown to the user when the connector runs out of reductions, either because max_attempts was reached or because the page size is already at minimum_page_size. Use it to tell the user what they can do about it in terms of this specific API, for instance which filter narrows the query down. Without it the message only states that the API kept rejecting every page size the connector asked for.", diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 4fd41a4159..6587b77ff0 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -3756,6 +3756,8 @@ def _create_page_size_reduction( reduction_factor=model.page_size_reduction.reduction_factor, # type: ignore[arg-type] # the schema defines a default minimum_page_size=model.page_size_reduction.minimum_page_size, # type: ignore[arg-type] # the schema defines a default max_attempts=model.page_size_reduction.max_attempts, # type: ignore[arg-type] # the schema defines a default + backoff_seconds=model.page_size_reduction.backoff_seconds, # type: ignore[arg-type] # the schema defines a default + retries_at_minimum_page_size=model.page_size_reduction.retries_at_minimum_page_size, # type: ignore[arg-type] # the schema defines a default failure_message=model.page_size_reduction.failure_message, reset_policy=PageSizeResetPolicy(reset_policy.value) if reset_policy is not None diff --git a/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py b/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py index 92dd7794fd..e4f841025b 100644 --- a/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py +++ b/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py @@ -30,6 +30,17 @@ class PageSizeReduction: minimum_page_size: int = 1 max_attempts: int = 5 reset_policy: PageSizeResetPolicy = PageSizeResetPolicy.NEVER + # Base wait before a page is re-issued, multiplied by the number of attempts made in a row. A + # `REDUCE_PAGE_SIZE` response never reaches the HTTP retry budget - the exception raised for it is + # deliberately not a backoff exception - so this is the only thing spacing these requests out, and an + # API whose 502 means "we are briefly unwell" rather than "your page is too big" needs it to be more + # than a token pause. + backoff_seconds: float = 0.5 + # How many times the same page is re-issued unchanged once the page size cannot be shrunk any further, + # before the read gives up. Zero keeps the strict behaviour: the first response that cannot be answered + # with a smaller page fails the stream. It is what gives an API whose error is transient a budget at the + # floor, where reducing is no longer an option but waiting still is. + retries_at_minimum_page_size: int = 0 # Appended to the two messages raised once the page size cannot be reduced any further. Those are # `transient_error`s the CDK has no remediation for - it only knows that the API rejected every page size # asked for - while the connector knows what narrows a query down on this particular API. @@ -48,6 +59,14 @@ def __post_init__(self) -> None: raise ValueError( f"The maximum number of page size reductions needs to be strictly positive. Got {self.max_attempts}" ) + if self.backoff_seconds < 0: + raise ValueError( + f"The wait between page size reductions cannot be negative. Got {self.backoff_seconds}" + ) + if self.retries_at_minimum_page_size < 0: + raise ValueError( + f"The number of retries at the minimum page size cannot be negative. Got {self.retries_at_minimum_page_size}" + ) class PageSizeReducer: @@ -59,12 +78,6 @@ class PageSizeReducer: page size must not be stored on the paginator or on the retriever. """ - # `PageSizeReductionRequiredException` is deliberately neither a `BaseBackoffException` nor a transient - # exception, so the reduced page is re-issued outside of the HTTP retry budget and nothing else spaces - # those requests out. The wait is kept non-zero and grows with the number of reductions so an endpoint - # that fails whatever page size we ask for degrades to a slow retry instead of a burst of requests. - BACKOFF_SECONDS: float = 0.5 - def __init__( self, config: PageSizeReduction, @@ -79,6 +92,7 @@ def __init__( self._current_page_size: Optional[int] = None self._attempts = 0 self._total_reductions = 0 + self._retries_at_minimum_page_size = 0 @property def page_size_override(self) -> Optional[int]: @@ -116,6 +130,16 @@ def reduce(self) -> None: failure_type=FailureType.config_error, ) + reduced_page_size = max( + self._config.minimum_page_size, + int(current_page_size // self._config.reduction_factor), + ) + if reduced_page_size >= current_page_size: + # Nothing left to give up on the page size. Whether that is the end of the read is + # `retries_at_minimum_page_size`'s call, not this branch's: the response may still be transient. + self._retry_at_minimum_page_size(current_page_size) + return + self._attempts += 1 self._total_reductions += 1 if self._attempts > self._config.max_attempts: @@ -133,33 +157,7 @@ def reduce(self) -> None: failure_type=FailureType.transient_error, ) - reduced_page_size = max( - self._config.minimum_page_size, - int(current_page_size // self._config.reduction_factor), - ) - if reduced_page_size >= current_page_size: - if self._current_page_size is None: - # No reduction was ever applied, so the configured page size is already at or below the - # minimum. Nothing about the response can fix that, which makes it a configuration error - # rather than something the platform should retry the whole job for. - raise AirbyteTracedException( - internal_message=f"Stream {self._stream_name} has a configured page size of {current_page_size} which is not greater than the configured minimum page size of {self._config.minimum_page_size}, so it can never be reduced", - message=f"The page size of stream {self._stream_name} ({current_page_size}) is already at or below " - f"the configured minimum of {self._config.minimum_page_size}, so the connector cannot reduce it. " - f"Raise the page size of the stream, or lower `minimum_page_size`.", - failure_type=FailureType.config_error, - ) - raise AirbyteTracedException( - internal_message=f"Stream {self._stream_name} still fails with a page size of {current_page_size}, which is the smallest page size allowed by the configured minimum of {self._config.minimum_page_size}", - # `transient_error`, so the only remediation is the connector's own, if it defined one. - message=self._with_failure_message( - f"The source keeps rejecting pages of stream {self._stream_name} at the smallest page size " - f"the connector is allowed to request ({current_page_size} records per page)." - ), - failure_type=FailureType.transient_error, - ) - - backoff = self.BACKOFF_SECONDS * self._attempts + backoff = self._config.backoff_seconds * self._attempts LOGGER.info( f"Reducing the page size of stream {self._stream_name} from {current_page_size} to {reduced_page_size} " f"and retrying the same page in {backoff}s." @@ -167,6 +165,55 @@ def reduce(self) -> None: self._current_page_size = reduced_page_size self._sleep(backoff) + def _retry_at_minimum_page_size(self, current_page_size: int) -> None: + """ + Handle a `REDUCE_PAGE_SIZE` response that arrives when the page size is already as small as the + connector is allowed to request. + + Reducing is out of options here, but re-issuing the page is not: an API that answers 502 to a page it + considers too heavy answers the same 502 when it is briefly unwell, and the error handler cannot tell + the two apart. `REDUCE_PAGE_SIZE` bypasses the HTTP retry budget, so without this budget the second + kind of 502 ends the stream on the first response once the floor is reached - fewer attempts than the + same connector got before it adopted the reduction. + """ + if self._current_page_size is None and self._config.minimum_page_size > 1: + # No reduction was ever applied and the connector's own floor is what blocks it, so the page size + # can never be reduced on this stream however the API behaves. That is a configuration error, and + # it is actionable: both numbers in the message are the connector's to change. + raise AirbyteTracedException( + internal_message=f"Stream {self._stream_name} has a configured page size of {current_page_size} which is not greater than the configured minimum page size of {self._config.minimum_page_size}, so it can never be reduced", + message=f"The page size of stream {self._stream_name} ({current_page_size}) is already at or below " + f"the configured minimum of {self._config.minimum_page_size}, so the connector cannot reduce it. " + f"Raise the page size of the stream, or lower `minimum_page_size`.", + failure_type=FailureType.config_error, + ) + + if self._retries_at_minimum_page_size < self._config.retries_at_minimum_page_size: + self._retries_at_minimum_page_size += 1 + backoff = self._config.backoff_seconds * self._retries_at_minimum_page_size + LOGGER.info( + f"Stream {self._stream_name} cannot request a page smaller than {current_page_size} records, " + f"so the same page is retried unchanged in {backoff}s " + f"({self._retries_at_minimum_page_size} of {self._config.retries_at_minimum_page_size})." + ) + self._sleep(backoff) + return + + raise AirbyteTracedException( + internal_message=f"Stream {self._stream_name} still fails with a page size of {current_page_size}, which is the smallest page size allowed by the configured minimum of {self._config.minimum_page_size}" + + ( + f", after {self._retries_at_minimum_page_size} retries at that size" + if self._retries_at_minimum_page_size + else "" + ), + # `transient_error`, so the only remediation is the connector's own, if it defined one. + message=self._with_failure_message( + f"The source keeps rejecting pages of stream {self._stream_name} at the smallest page size " + f"the connector is allowed to request ({current_page_size} records per page)." + ), + failure_type=FailureType.transient_error, + ) + def _with_failure_message(self, message: str) -> str: """ :return: the message followed by the connector's `failure_message`, when it defined one @@ -199,6 +246,7 @@ def on_successful_page(self) -> None: reduced size for the rest of the partition, which is the right behaviour when reductions are one-off. """ self._attempts = 0 + self._retries_at_minimum_page_size = 0 if self._config.reset_policy != PageSizeResetPolicy.AFTER_SUCCESSFUL_PAGE: return diff --git a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index 83c53af199..85871d0eb7 100644 --- a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -482,8 +482,8 @@ def _read_pages( message=f"Stream {self.name} asked for a smaller page size in the middle of a page. The page cannot be requested again without duplicating the records already read from it. Move the REDUCE_PAGE_SIZE action to the error handler of the stream's main requester.", failure_type=FailureType.config_error, ) - # Raises once the page size cannot be reduced any further, which is what stops the loop when - # the API keeps failing. + # Raises once the page size cannot be reduced any further and the retries allowed at that + # floor are spent, which is what stops the loop when the API keeps failing. page_size_reducer.reduce() reduce_page_size = True else: @@ -493,7 +493,8 @@ def _read_pages( break if reduce_page_size: - # Retry the very same page: neither the token nor the slice change, only the page size does. + # Retry the very same page: neither the token nor the slice change, only the page size does - + # and not even that once the reducer is at its floor and only waiting is left. reduce_page_size = False continue diff --git a/unit_tests/connector_builder/test_connector_builder_handler.py b/unit_tests/connector_builder/test_connector_builder_handler.py index 38c072208b..c28f0f1485 100644 --- a/unit_tests/connector_builder/test_connector_builder_handler.py +++ b/unit_tests/connector_builder/test_connector_builder_handler.py @@ -2000,10 +2000,7 @@ def _create_502_page_response(): return response -@patch( - "airbyte_cdk.sources.declarative.retrievers.page_size_reducer.PageSizeReducer.BACKOFF_SECONDS", - 0, -) +@patch("airbyte_cdk.sources.declarative.retrievers.page_size_reducer.time.sleep", lambda _: None) @patch.object( requests.Session, "send", diff --git a/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py b/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py index 113dcb5969..8d20561ed5 100644 --- a/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py +++ b/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py @@ -72,13 +72,18 @@ def test_given_minimum_page_size_above_configured_page_size_when_reduce_then_rai assert "already at or below the configured minimum" in exception.value.message -def test_given_page_size_cannot_be_reduced_when_reduce_then_raise_config_error(): +def test_given_page_size_of_one_when_reduce_then_raise_transient_error(): + """ + Unlike a `minimum_page_size` above the configured page size, there is nothing to configure differently + here: one record per page is as small as a page gets. The API rejecting it is about the API, not about + the connector's setup, so the platform gets a transient error rather than a config error. + """ reducer = _reducer(configured_page_size=1) with pytest.raises(AirbyteTracedException) as exception: reducer.reduce() - assert exception.value.failure_type == FailureType.config_error + assert exception.value.failure_type == FailureType.transient_error def test_given_already_at_minimum_when_reduce_then_raise_transient_error(): @@ -234,6 +239,10 @@ def test_given_reset_policy_after_successful_page_when_every_page_succeeds_then_ pytest.param({"reduction_factor": 1}, id="reduction_factor_does_not_reduce"), pytest.param({"minimum_page_size": 0}, id="minimum_page_size_is_not_positive"), pytest.param({"max_attempts": 0}, id="max_attempts_is_not_positive"), + pytest.param({"backoff_seconds": -1}, id="backoff_seconds_is_negative"), + pytest.param( + {"retries_at_minimum_page_size": -1}, id="retries_at_minimum_page_size_is_negative" + ), ], ) def test_given_invalid_configuration_then_raise_value_error(kwargs): @@ -269,12 +278,79 @@ def test_when_reduce_then_wait_before_the_retry(): reducer.reduce() assert sleeps == [ - PageSizeReducer.BACKOFF_SECONDS, - PageSizeReducer.BACKOFF_SECONDS * 2, + PageSizeReduction().backoff_seconds, + PageSizeReduction().backoff_seconds * 2, ] assert all(wait > 0 for wait in sleeps) +def test_given_backoff_seconds_when_reduce_then_wait_that_long(): + sleeps: list = [] + reducer = _reducer(configured_page_size=1000, sleeps=sleeps, backoff_seconds=10) + + reducer.reduce() + reducer.reduce() + + assert sleeps == [10, 20] + + +def test_given_retries_at_minimum_page_size_when_at_the_floor_then_retry_the_same_page(): + """ + At the floor the page size cannot answer the error any more, but an API that returns the same status for + "your page is too big" and for a passing hiccup still can. Without this budget the first such response + ends the stream, which is fewer attempts than the same error handler gave before the reduction existed. + """ + sleeps: list = [] + reducer = _reducer( + configured_page_size=2, + sleeps=sleeps, + minimum_page_size=1, + backoff_seconds=1, + retries_at_minimum_page_size=2, + ) + + reducer.reduce() + assert reducer.page_size_override == 1 + + reducer.reduce() + reducer.reduce() + assert reducer.page_size_override == 1, "the page size is already at the minimum" + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.transient_error + assert sleeps == [1, 1, 2], "the reduction, then the two retries at the floor" + + +def test_given_retries_at_minimum_page_size_do_not_count_against_max_attempts(): + """ + The two budgets measure different things: `max_attempts` bounds the reductions that got no page through, + while this one bounds the waiting done once there is nothing left to reduce. + """ + reducer = _reducer(configured_page_size=2, max_attempts=1, retries_at_minimum_page_size=3) + + reducer.reduce() + reducer.reduce() + reducer.reduce() + reducer.reduce() + + with pytest.raises(AirbyteTracedException): + reducer.reduce() + + +def test_given_successful_page_then_restart_the_retries_at_minimum_page_size(): + reducer = _reducer(configured_page_size=1, retries_at_minimum_page_size=1) + reducer.reduce() + + reducer.on_successful_page() + + reducer.reduce() # would raise if the budget had not restarted + + with pytest.raises(AirbyteTracedException): + reducer.reduce() + + @pytest.mark.parametrize( "reducer_kwargs,reductions", [ @@ -305,7 +381,7 @@ def test_given_reduction_fails_then_message_names_the_stream_and_leaves_out_reme @pytest.mark.parametrize( "reducer_kwargs", [ - pytest.param({"configured_page_size": 1, "minimum_page_size": 1}, id="never_reducible"), + pytest.param({"configured_page_size": 5, "minimum_page_size": 10}, id="never_reducible"), pytest.param({"configured_page_size": None}, id="no_page_size_at_all"), pytest.param({"configured_page_size": "100"}, id="page_size_is_not_a_number"), ], diff --git a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index 818f04642c..1afa3b5fc9 100644 --- a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -1445,7 +1445,9 @@ def test_given_reach_pagination_limit_after_two_pages_when_read_records_than_red @pytest.fixture(autouse=True) def _no_page_size_reduction_backoff(monkeypatch): """The reducer waits between reduction retries; taking those waits for real adds seconds to every CI run.""" - monkeypatch.setattr(PageSizeReducer, "BACKOFF_SECONDS", 0) + monkeypatch.setattr( + "airbyte_cdk.sources.declarative.retrievers.page_size_reducer.time.sleep", lambda _: None + ) def _page_size_reduction_retriever( @@ -1502,6 +1504,61 @@ def test_given_page_size_reduction_when_read_records_then_retry_same_page_with_r ] == [None, 50] +def test_given_retries_at_minimum_page_size_when_at_the_floor_then_re_issue_the_same_page(): + """The page size stays at the floor: what the retry buys is the wait, not a smaller request.""" + requester = Mock(spec=Requester) + requester.send_request.side_effect = [ + PageSizeReductionRequiredException(), + [{"id": 1}], + ] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.return_value = [{"id": 1}] + paginator = _mock_paginator() + paginator.get_page_size.return_value = 1 + paginator.get_initial_token.return_value = None + paginator.next_page_token.return_value = None + + retriever = _page_size_reduction_retriever( + requester, + paginator, + record_selector, + PageSizeReduction(retries_at_minimum_page_size=1), + ) + + records = list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + assert records == [{"id": 1}] + assert requester.send_request.call_count == 2, ( + "the page that could not be reduced was issued twice" + ) + assert [ + call.kwargs.get("page_size_override") + for call in paginator.get_request_params.call_args_list + ] == [None, None] + + +def test_given_retries_at_minimum_page_size_are_spent_then_raise_transient_error(): + requester = Mock(spec=Requester) + requester.send_request.side_effect = PageSizeReductionRequiredException() + record_selector = Mock(spec=HttpSelector) + paginator = _mock_paginator() + paginator.get_page_size.return_value = 1 + paginator.get_initial_token.return_value = None + + retriever = _page_size_reduction_retriever( + requester, + paginator, + record_selector, + PageSizeReduction(retries_at_minimum_page_size=1), + ) + + with pytest.raises(AirbyteTracedException) as exception: + list(retriever.read_records(A_RECORD_SCHEMA, A_STREAM_SLICE)) + + assert exception.value.failure_type == FailureType.transient_error + assert requester.send_request.call_count == 2 + + def test_given_page_size_reduction_when_read_records_then_next_page_token_not_computed_for_failed_page(): requester = Mock(spec=Requester) requester.send_request.side_effect = [ diff --git a/unit_tests/sources/declarative/test_concurrent_declarative_source.py b/unit_tests/sources/declarative/test_concurrent_declarative_source.py index 7716899b00..c8aaca355b 100644 --- a/unit_tests/sources/declarative/test_concurrent_declarative_source.py +++ b/unit_tests/sources/declarative/test_concurrent_declarative_source.py @@ -60,7 +60,6 @@ from airbyte_cdk.sources.declarative.resolvers.http_components_resolver import ( HttpComponentsResolver, ) -from airbyte_cdk.sources.declarative.retrievers.page_size_reducer import PageSizeReducer from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever from airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_generator import ( StreamSlicerPartitionGenerator, @@ -5052,7 +5051,7 @@ def _read_page_size_reduction_source(manifest): state=None, ) # the reducer waits before each reduction retry; taking those waits for real adds seconds to every CI run - with patch.object(PageSizeReducer, "BACKOFF_SECONDS", 0): + with patch("airbyte_cdk.sources.declarative.retrievers.page_size_reducer.time.sleep"): yield from source.read(logger=source.logger, config={}, catalog=catalog, state=[]) From 01add3fa039116036c414d84bb9eb5904ea3e3fb Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Thu, 17 Sep 2026 15:21:32 +0300 Subject: [PATCH 11/13] fix(page size reduction): resolve the sleep function at call time `PageSizeReducer` took `sleep: Callable = time.sleep`, which binds the function object when the module is imported. A connector test that patches `time.sleep` to keep a run of reductions from taking its backoff for real therefore had no effect, and a suite covering a reduction down to the minimum page size paid the whole wait. The override is now stored and `time.sleep` is looked up per call. Co-Authored-By: Claude Opus 5 (1M context) --- .../declarative/retrievers/page_size_reducer.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py b/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py index e4f841025b..1e79543010 100644 --- a/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py +++ b/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py @@ -83,17 +83,26 @@ def __init__( config: PageSizeReduction, configured_page_size: Optional[int], stream_name: str = "", - sleep: Callable[[float], None] = time.sleep, + sleep: Optional[Callable[[float], None]] = None, ) -> None: self._config = config self._configured_page_size = configured_page_size self._stream_name = stream_name - self._sleep = sleep + # Resolved on each call rather than bound here: a default of `time.sleep` would capture + # the function object, and a test that patches `time.sleep` to keep a run of reductions + # from taking its two minutes for real would have no effect on an already-bound default. + self._sleep_override = sleep self._current_page_size: Optional[int] = None self._attempts = 0 self._total_reductions = 0 self._retries_at_minimum_page_size = 0 + def _sleep(self, seconds: float) -> None: + if self._sleep_override is not None: + self._sleep_override(seconds) + return + time.sleep(seconds) + @property def page_size_override(self) -> Optional[int]: """ From 54f86a82101dd7b439bc0918cbe05c6d99403af2 Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Thu, 17 Sep 2026 18:03:59 +0300 Subject: [PATCH 12/13] fix(page size reduction): apply the retries at the floor however the page size got there Two findings from the fourth round of review. R4-F1. `_retry_at_minimum_page_size` tested the `config_error` branch first, and that branch never consulted `retries_at_minimum_page_size`. So the same manifest gave a partition its retries when the reduction walked down to the floor and none at all when the user's `page_size` was already there: the first response ended the stream, telling the user to raise `page_size` or lower `minimum_page_size` - a manifest field they cannot reach, for a case the connector author had already answered by configuring retries. Whether a transient error was retried therefore depended on how the page size arrived at the floor rather than on anything about the error. The retries now come first and the misconfiguration is reported once they are spent, so it is still reported and still names both numbers to change. Measured over the eight configurations that reach the floor: `configured=20, min=10, retries=3` and `configured=10, min=10, retries=3` now take the same three waits and differ only in the failure type, which is the one thing that should differ - a floor that blocks every reduction is the connector's to fix, a floor the reduction reached means the API kept rejecting every size. R4-F2. The sleep-resolution fix of `01add3fa` was not pinned by any test: every test here passes its own `sleep`, so none exercised the default path. `test_given_no_sleep_override_when_reduce_then_wait_through_time_sleep` patches `page_size_reducer.time.sleep` and asserts the wait lands there. Verified against the defect: restoring the bound default makes that test fail, and the file's run takes 7.5s of real sleeping instead of 0.45s. Co-Authored-By: Claude Opus 5 (1M context) --- .../declarative_component_schema.yaml | 4 +- .../models/declarative_component_schema.py | 2 +- .../retrievers/page_size_reducer.py | 38 +++++++++---- .../retrievers/test_page_size_reducer.py | 55 +++++++++++++++++++ 4 files changed, 85 insertions(+), 14 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index d13a3a48c7..9c0d1aa603 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -4387,7 +4387,9 @@ definitions: first response received at minimum_page_size. Raise it when the API returns the same error for a page that is too big and for a server-side hiccup: at the floor, reducing is no longer an option but waiting still is, and without this budget those responses end the stream on the first one. This budget is separate from - max_attempts, which only counts reductions, and it restarts on every page that succeeds. + max_attempts, which only counts reductions, and it restarts on every page that succeeds. It applies however the page size arrived at the floor, whether by + reduction or because page_size was already there; a page size that minimum_page_size blocks from ever + being reduced is still reported as a configuration error, but only once this budget is spent. type: integer default: 0 minimum: 0 diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 3468d8b7e2..e1ef9e6593 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -1450,7 +1450,7 @@ class PageSizeReduction(BaseModel): ) retries_at_minimum_page_size: Optional[int] = Field( 0, - description="Number of times the same page is re-issued unchanged, each after the backoff wait, once the page size cannot be shrunk any further, before the sync fails with a transient error. The default of 0 fails on the first response received at minimum_page_size. Raise it when the API returns the same error for a page that is too big and for a server-side hiccup: at the floor, reducing is no longer an option but waiting still is, and without this budget those responses end the stream on the first one. This budget is separate from max_attempts, which only counts reductions, and it restarts on every page that succeeds.", + description="Number of times the same page is re-issued unchanged, each after the backoff wait, once the page size cannot be shrunk any further, before the sync fails with a transient error. The default of 0 fails on the first response received at minimum_page_size. Raise it when the API returns the same error for a page that is too big and for a server-side hiccup: at the floor, reducing is no longer an option but waiting still is, and without this budget those responses end the stream on the first one. This budget is separate from max_attempts, which only counts reductions, and it restarts on every page that succeeds. It applies however the page size arrived at the floor, whether by reduction or because page_size was already there; a page size that minimum_page_size blocks from ever being reduced is still reported as a configuration error, but only once this budget is spent.", examples=[0, 3], ge=0, title="Retries At Minimum Page Size", diff --git a/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py b/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py index 1e79543010..496313db2f 100644 --- a/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py +++ b/airbyte_cdk/sources/declarative/retrievers/page_size_reducer.py @@ -184,19 +184,15 @@ def _retry_at_minimum_page_size(self, current_page_size: int) -> None: the two apart. `REDUCE_PAGE_SIZE` bypasses the HTTP retry budget, so without this budget the second kind of 502 ends the stream on the first response once the floor is reached - fewer attempts than the same connector got before it adopted the reduction. - """ - if self._current_page_size is None and self._config.minimum_page_size > 1: - # No reduction was ever applied and the connector's own floor is what blocks it, so the page size - # can never be reduced on this stream however the API behaves. That is a configuration error, and - # it is actionable: both numbers in the message are the connector's to change. - raise AirbyteTracedException( - internal_message=f"Stream {self._stream_name} has a configured page size of {current_page_size} which is not greater than the configured minimum page size of {self._config.minimum_page_size}, so it can never be reduced", - message=f"The page size of stream {self._stream_name} ({current_page_size}) is already at or below " - f"the configured minimum of {self._config.minimum_page_size}, so the connector cannot reduce it. " - f"Raise the page size of the stream, or lower `minimum_page_size`.", - failure_type=FailureType.config_error, - ) + The budget applies however the page size arrived at the floor, whether by reduction or because the + configured page size was already there. Only once it is spent does the failure depend on that: a page + size the connector's own floor blocks from ever being reduced is a configuration error, while a floor + the reduction walked down to means the API kept rejecting every size, which is transient. + """ + # The retries come first, including on a stream whose page size started at the floor. How the page size + # arrived there says nothing about the response, so letting the misconfiguration branch below decide it + # would give the same manifest a retry budget or none depending on the user's `page_size`. if self._retries_at_minimum_page_size < self._config.retries_at_minimum_page_size: self._retries_at_minimum_page_size += 1 backoff = self._config.backoff_seconds * self._retries_at_minimum_page_size @@ -208,6 +204,24 @@ def _retry_at_minimum_page_size(self, current_page_size: int) -> None: self._sleep(backoff) return + if self._current_page_size is None and self._config.minimum_page_size > 1: + # No reduction was ever applied and the connector's own floor is what blocks it, so the page size + # can never be reduced on this stream however the API behaves. That is a configuration error, and + # it is actionable: both numbers in the message are the connector's to change. It is reported once + # the retries above are spent, so a manifest that asked for them still gets them. + raise AirbyteTracedException( + internal_message=f"Stream {self._stream_name} has a configured page size of {current_page_size} which is not greater than the configured minimum page size of {self._config.minimum_page_size}, so it can never be reduced" + + ( + f" ({self._retries_at_minimum_page_size} retries at that size were spent first)" + if self._retries_at_minimum_page_size + else "" + ), + message=f"The page size of stream {self._stream_name} ({current_page_size}) is already at or below " + f"the configured minimum of {self._config.minimum_page_size}, so the connector cannot reduce it. " + f"Raise the page size of the stream, or lower `minimum_page_size`.", + failure_type=FailureType.config_error, + ) + raise AirbyteTracedException( internal_message=f"Stream {self._stream_name} still fails with a page size of {current_page_size}, which is the smallest page size allowed by the configured minimum of {self._config.minimum_page_size}" + ( diff --git a/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py b/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py index 8d20561ed5..012ece3b9b 100644 --- a/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py +++ b/unit_tests/sources/declarative/retrievers/test_page_size_reducer.py @@ -323,6 +323,61 @@ def test_given_retries_at_minimum_page_size_when_at_the_floor_then_retry_the_sam assert sleeps == [1, 1, 2], "the reduction, then the two retries at the floor" +def test_given_page_size_configured_at_a_floor_above_one_then_retry_before_reporting_the_misconfiguration(): + """ + The retry budget cannot depend on how the page size arrived at the floor: the same manifest would then give + a partition three retries when the reduction walked down to the floor and none when the user's `page_size` + was already there. The misconfiguration is still reported, once the retries the manifest asked for are + spent. + """ + sleeps: list = [] + reducer = _reducer( + configured_page_size=10, + sleeps=sleeps, + minimum_page_size=10, + backoff_seconds=1, + retries_at_minimum_page_size=2, + ) + + reducer.reduce() + reducer.reduce() + assert reducer.page_size_override is None, "there was never a reduction to apply" + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert sleeps == [1, 2] + assert exception.value.failure_type == FailureType.config_error + assert "already at or below the configured minimum" in exception.value.message + assert "2 retries at that size were spent first" in exception.value.internal_message + + +def test_given_page_size_configured_at_a_floor_above_one_and_no_retries_then_report_the_misconfiguration(): + reducer = _reducer(configured_page_size=10, minimum_page_size=10) + + with pytest.raises(AirbyteTracedException) as exception: + reducer.reduce() + + assert exception.value.failure_type == FailureType.config_error + + +def test_given_no_sleep_override_when_reduce_then_wait_through_time_sleep(monkeypatch): + """ + The reducer resolves `time.sleep` per call rather than binding it as a default argument. Bound, a suite + that patches `time.sleep` to keep a run of reductions from taking its minutes for real has no effect - and + every other test here passes its own `sleep`, so this is the only one that exercises the default path. + """ + waits: list = [] + monkeypatch.setattr( + "airbyte_cdk.sources.declarative.retrievers.page_size_reducer.time.sleep", waits.append + ) + reducer = PageSizeReducer(PageSizeReduction(backoff_seconds=7), 100, stream_name=A_STREAM_NAME) + + reducer.reduce() + + assert waits == [7] + + def test_given_retries_at_minimum_page_size_do_not_count_against_max_attempts(): """ The two budgets measure different things: `max_attempts` bounds the reductions that got no page through, From a17b5a3dc120b11e59c6f2185fd31f088c5bf163 Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Fri, 18 Sep 2026 18:23:41 +0300 Subject: [PATCH 13/13] fix(page size reduction): reject a stop condition that reads an unbound page_size `page_size` is bound to the page size that was actually requested, which is what makes `{{ last_page_size < page_size }}` the one comparison that survives a reduction. But it is only a number when the CursorPagination strategy declares a `page_size`. Without one it binds to `None`, and the comparison does not fail: Jinja raises, `JinjaInterpolation._eval` treats the TypeError as "not a template" and returns the raw template string, and `InterpolatedBoolean` reads a non-empty string as `True`. The stop condition is then satisfied on page 1 and the rest of the partition is dropped without anything failing. This mattered beyond the streams that reduce: the example and the advice to prefer `page_size` over a hardcoded number sit on `CursorPagination.stop_condition`, a field 710 pagination blocks across 97 connectors already use without declaring a `page_size`. `CursorPaginationStrategy.__post_init__` now rejects a `stop_condition` or `cursor_value` that reads `page_size` while the strategy declares none. The reference is read off the Jinja AST, so the bare variable is told apart from `config['page_size']`, which is a lookup on the config and is bound either way. Rejecting at construction rather than failing per page keeps a sync from emitting a truncated partition first. The schema now says the variable needs a declared `page_size`, and points at `{{ last_page_size == 0 }}` for the case where there is none. No manifest in the monorepo is affected: 0 of 535 declare a CursorPagination without a `page_size` whose `stop_condition` or `cursor_value` reads the variable. Co-Authored-By: Claude Opus 5 --- .../declarative_component_schema.yaml | 8 +- .../models/declarative_component_schema.py | 2 +- .../parsers/stop_condition_safety.py | 19 +++++ .../strategies/cursor_pagination_strategy.py | 37 +++++++++ .../parsers/test_stop_condition_safety.py | 22 ++++++ .../test_cursor_pagination_strategy.py | 79 ++++++++++++++++++- 6 files changed, 162 insertions(+), 5 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 9c0d1aa603..0ac9715e43 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -776,9 +776,11 @@ definitions: description: >- Template string evaluating when to stop paginating. Compare last_page_size against page_size rather than against a hardcoded number: page_size is the page size that was actually requested, so the condition stays - correct when page_size_reduction shrinks it. Testing the page for emptiness with last_page_size == 0 is - equally safe. A stream that enables page_size_reduction is rejected when its stop condition compares - last_page_size against anything else, since a full page at a reduced size would then read as a short page. + correct when page_size_reduction shrinks it. It is only bound when this strategy declares page_size, and a + condition that reads it without one is rejected, so add page_size alongside the condition. Testing the page + for emptiness with last_page_size == 0 is equally safe and needs no page_size. A stream that enables + page_size_reduction is rejected when its stop condition compares last_page_size against anything else, + since a full page at a reduced size would then read as a short page. type: string interpolation_context: - config diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index e1ef9e6593..1608767606 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -140,7 +140,7 @@ class CursorPagination(BaseModel): ) stop_condition: Optional[str] = Field( None, - description="Template string evaluating when to stop paginating. Compare last_page_size against page_size rather than against a hardcoded number: page_size is the page size that was actually requested, so the condition stays correct when page_size_reduction shrinks it. Testing the page for emptiness with last_page_size == 0 is equally safe. A stream that enables page_size_reduction is rejected when its stop condition compares last_page_size against anything else, since a full page at a reduced size would then read as a short page.", + description="Template string evaluating when to stop paginating. Compare last_page_size against page_size rather than against a hardcoded number: page_size is the page size that was actually requested, so the condition stays correct when page_size_reduction shrinks it. It is only bound when this strategy declares page_size, and a condition that reads it without one is rejected, so add page_size alongside the condition. Testing the page for emptiness with last_page_size == 0 is equally safe and needs no page_size. A stream that enables page_size_reduction is rejected when its stop condition compares last_page_size against anything else, since a full page at a reduced size would then read as a short page.", examples=[ "{{ response.data.has_more is false }}", "{{ 'next' not in headers['link'] }}", diff --git a/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py b/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py index ed0e47f681..9ec61ce2b1 100644 --- a/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py +++ b/airbyte_cdk/sources/declarative/parsers/stop_condition_safety.py @@ -365,6 +365,25 @@ def _is_requested_page_size(node: nodes.Node) -> bool: return isinstance(unfiltered, nodes.Name) and unfiltered.name == REQUESTED_PAGE_SIZE_VARIABLE +def references_requested_page_size(template: str) -> bool: + """ + Whether an expression reads the `page_size` interpolation variable. + + The variable is only bound to a number when the strategy declares a `page_size`, and a comparison against + an unbound one does not fail: Jinja raises, the interpolation falls back to the raw template string, and a + non-empty string is truthy - so `{{ last_page_size < page_size }}` would stop the pagination after the + first page. Reading it off the AST is what tells the variable apart from `config['page_size']`, which is a + `Getitem` on `config` rather than a `Name` and is bound whatever the strategy declares. + """ + try: + parsed = _PARSING_ENVIRONMENT.parse(template) + except TemplateSyntaxError: + # An expression that does not parse is not this function's to report on. The interpolation layer + # renders it as the literal string it is, and it names no variable either way. + return False + return _references(parsed, REQUESTED_PAGE_SIZE_VARIABLE) + + def _references(node: nodes.Node, name: str) -> bool: if isinstance(node, nodes.Name): return bool(node.name == name) diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py index 1e600a7bdd..a4c43aed45 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py @@ -14,6 +14,10 @@ ) from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.parsers.stop_condition_safety import ( + REQUESTED_PAGE_SIZE_VARIABLE, + references_requested_page_size, +) from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import ( PaginationStrategy, ) @@ -62,6 +66,39 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None: raise Exception(f"{page_size} is of type {type(page_size)}. Expected {int}") self._page_size = page_size + if self._page_size is None: + self._reject_unbound_page_size_variable() + + def _reject_unbound_page_size_variable(self) -> None: + """ + Fail construction when an expression reads `page_size` while the strategy declares none. + + The variable holds the page size that was actually requested, so it is the one safe thing to compare + `last_page_size` against while a `page_size_reduction` is shrinking the page. Without a declared + `page_size` there is nothing to bind it to, and an unbound comparison does not raise: Jinja fails, the + interpolation falls back to the raw template string, and a non-empty string is truthy. A + `stop_condition` written that way stops after the first page and drops the rest of the partition + without failing, which is why this is a construction error rather than a warning. + """ + for field_name, template in ( + ("stop_condition", self.stop_condition), + ("cursor_value", self.cursor_value), + ): + expression = template.string if isinstance(template, InterpolatedString) else template + if isinstance(template, InterpolatedBoolean): + expression = template.condition + if not isinstance(expression, str) or not references_requested_page_size(expression): + continue + raise ValueError( + f"The `{field_name}` {expression!r} reads the `{REQUESTED_PAGE_SIZE_VARIABLE}` interpolation " + f"variable, but the CursorPagination strategy it belongs to declares no `page_size`, so there " + f"is nothing to bind it to. The comparison would not fail either - it renders as the template " + f"string itself, which is truthy - so the pagination would end after the first page and the " + f"rest of the partition would be dropped silently. Declare `page_size` on the pagination " + f"strategy, or compare against a value the manifest defines, such as " + f"`config['page_size']`." + ) + @property def initial_token(self) -> Optional[Any]: """ diff --git a/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py b/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py index 988509ac83..6d159cd354 100644 --- a/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py +++ b/unit_tests/sources/declarative/parsers/test_stop_condition_safety.py @@ -8,6 +8,7 @@ LAST_PAGE_SIZE_VARIABLE, StopConditionSafety, classify_stop_condition, + references_requested_page_size, ) @@ -365,3 +366,24 @@ def test_given_page_length_read_from_the_response_then_reason_names_the_expressi assert verdict is StopConditionSafety.UNKNOWN assert LAST_PAGE_SIZE_VARIABLE in reason + + +@pytest.mark.parametrize( + "template,expected", + [ + pytest.param("{{ last_page_size < page_size }}", True, id="bare_variable"), + pytest.param("{{ page_size }}", True, id="rendered_on_its_own"), + pytest.param("{{ [page_size, 100] | max }}", True, id="passed_through_a_filter"), + pytest.param("{{ last_page_size < config['page_size'] }}", False, id="config_lookup"), + pytest.param("{{ response['page_size'] }}", False, id="response_lookup"), + pytest.param("{{ last_page_size == 0 }}", False, id="unrelated_expression"), + pytest.param("page_size", False, id="plain_text_naming_it"), + pytest.param("{{ last_page_size < page_size", False, id="does_not_parse"), + ], +) +def test_references_requested_page_size(template, expected): + """ + The distinction that matters is the bare `page_size` variable, which only exists when the strategy declares + a page size, against `config['page_size']`, which is bound whatever the strategy declares. + """ + assert references_requested_page_size(template) is expected diff --git a/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py b/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py index 7527e0c736..4b25086fa1 100644 --- a/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +++ b/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py @@ -208,7 +208,6 @@ def test_given_stop_condition_uses_page_size_and_page_is_short_then_stop(): [ pytest.param(100, None, 100, id="test_configured_page_size_is_bound"), pytest.param(100, 50, 50, id="test_reduced_page_size_is_bound"), - pytest.param(None, None, None, id="test_no_page_size_interpolates_to_none"), ], ) def test_page_size_is_bound_in_the_cursor_value_interpolation_context( @@ -228,3 +227,81 @@ def test_page_size_is_bound_in_the_cursor_value_interpolation_context( strategy.next_page_token(response, 10, None, None, page_size_override=page_size_override) == expected_token ) + + +@pytest.mark.parametrize( + "field_name, kwargs", + [ + pytest.param( + "stop_condition", + { + "cursor_value": "{{ response.next }}", + "stop_condition": "{{ last_page_size < page_size }}", + }, + id="test_stop_condition_reads_page_size", + ), + pytest.param( + "cursor_value", + {"cursor_value": "{{ page_size }}"}, + id="test_cursor_value_reads_page_size", + ), + pytest.param( + "stop_condition", + { + "cursor_value": "{{ response.next }}", + "stop_condition": InterpolatedBoolean( + condition="{{ last_page_size < page_size }}", parameters={} + ), + }, + id="test_stop_condition_given_as_a_component", + ), + ], +) +def test_given_no_page_size_when_an_expression_reads_page_size_then_raise(field_name, kwargs): + """ + `page_size` is unbound without a declared page size, and an unbound comparison does not fail loudly: Jinja + raises, the interpolation falls back to the raw template string, and a non-empty string is truthy. A + `stop_condition` written that way would stop after the first page, so it has to be rejected here. + """ + with pytest.raises(ValueError) as error: + CursorPaginationStrategy(config={}, parameters={}, **kwargs) + + assert f"`{field_name}`" in str(error.value) + assert "declares no `page_size`" in str(error.value) + + +def test_given_no_page_size_when_the_expression_reads_config_page_size_then_accept(): + """`config['page_size']` is a lookup on the config, not the reduction-aware variable, and is always bound.""" + strategy = CursorPaginationStrategy( + cursor_value="{{ response.next }}", + stop_condition="{{ last_page_size < config['page_size'] }}", + config={"page_size": 100}, + parameters={}, + ) + response = requests.Response() + response._content = json.dumps({"next": "a token"}).encode("utf-8") + + assert strategy.next_page_token(response, 100, None, None) == "a token" + + +@pytest.mark.parametrize( + "page_size", + [ + pytest.param(100, id="test_literal_page_size"), + pytest.param("{{ config['page_size'] }}", id="test_interpolated_page_size"), + ], +) +def test_given_a_page_size_when_an_expression_reads_page_size_then_accept(page_size): + strategy = CursorPaginationStrategy( + cursor_value="{{ response.next }}", + stop_condition="{{ last_page_size < page_size }}", + page_size=page_size, + config={"page_size": 100}, + parameters={}, + ) + response = requests.Response() + response._content = json.dumps({"next": "a token"}).encode("utf-8") + + assert strategy.next_page_token(response, 100, None, None) == "a token" + assert strategy.next_page_token(response, 50, None, None, page_size_override=50) == "a token" + assert strategy.next_page_token(response, 49, None, None, page_size_override=50) is None