chore: lock file maintenance and fix new ty errors

docs/guides/code_examples/running_in_web_server/crawler.py

@@ -1,5 +1,5 @@
 import asyncio
-from collections.abc import AsyncIterator
+from collections.abc import AsyncGenerator
 from contextlib import asynccontextmanager
 from typing import TypedDict
 
@@ -16,7 +16,7 @@ class State(TypedDict):
 
 
 @asynccontextmanager
-async def lifespan(app: FastAPI) -> AsyncIterator[State]:
+async def lifespan(app: FastAPI) -> AsyncGenerator[State]:
     # Start up code that runs once when the app starts
 
     # Results will be stored in this dictionary

src/crawlee/_utils/time.py

@@ -9,7 +9,7 @@
 from async_timeout import Timeout, timeout
 
 if TYPE_CHECKING:
-    from collections.abc import Iterator
+    from collections.abc import Generator
     from types import TracebackType
 
 _SECONDS_PER_MINUTE = 60
@@ -23,7 +23,7 @@ class TimerResult:
 
 
 @contextmanager
-def measure_time() -> Iterator[TimerResult]:
+def measure_time() -> Generator[TimerResult]:
     """Measure the execution time (wall-clock and CPU) between the start and end of the with-block."""
     result = TimerResult()
     before_wall = time.monotonic()

src/crawlee/_utils/try_import.py

@@ -7,7 +7,7 @@
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from collections.abc import Iterator
+    from collections.abc import Generator
     from typing import Any
 
 
@@ -17,7 +17,7 @@ def try_import(
     *symbol_names: str,
     extra_name: str | list[str] | None = None,
     package_name: str = 'crawlee',
-) -> Iterator[None]:
+) -> Generator[None]:
     """Context manager to attempt importing symbols into a module.
 
     If an `ImportError` is raised during the import, the symbols are replaced with `FailedImport` objects. When the

src/crawlee/browsers/_browser_controller.py

@@ -8,7 +8,7 @@
 from crawlee._utils.docs import docs_group
 
 if TYPE_CHECKING:
-    from collections.abc import Mapping
+    from collections.abc import Mapping, Sequence
     from datetime import datetime, timedelta
 
     from playwright.async_api import Page
@@ -26,7 +26,7 @@ class BrowserController(ABC):
 
     @property
     @abstractmethod
-    def pages(self) -> list[Page]:
+    def pages(self) -> Sequence[Page]:
         """Return the list of opened pages."""
 
     @property

src/crawlee/crawlers/_pydantic_ai/_direct_extractor.py

@@ -2,7 +2,7 @@
 
 import asyncio
 from logging import getLogger
-from typing import TYPE_CHECKING, cast
+from typing import TYPE_CHECKING
 
 from pydantic_ai import Agent
 from pydantic_ai.usage import RunUsage
@@ -116,15 +116,11 @@ async def _run(
     ) -> TSchema:
         distilled_html = await asyncio.to_thread(self._distiller.distill, html)
 
-        # `cast` restores the static type pinned at runtime by `output_type`.
-        agent: Agent[None, TSchema] = cast(
-            'Agent[None, TSchema]',
-            Agent(
-                self._model,
-                output_type=schema,
-                instructions=self._base_instructions,
-                retries=self._retries,
-            ),
+        agent: Agent[None, TSchema] = Agent(
+            self._model,
+            output_type=schema,
+            instructions=self._base_instructions,
+            retries=self._retries,
         )
 
         # The task framing names the fields explicitly: the output tool schema alone is not enough for smaller

src/crawlee/crawlers/_pydantic_ai/_selector_extractor.py

@@ -511,14 +511,11 @@ async def _generate_selectors(
         schema: type[TSchema],
         additional_instructions: str | None,
     ) -> SelectorMap:
-        agent: Agent[None, SelectorMap] = cast(
-            'Agent[None, SelectorMap]',
-            Agent(
-                self._model,
-                output_type=SelectorMap,
-                instructions=self._base_instructions,
-                retries=self._retries,
-            ),
+        agent: Agent[None, SelectorMap] = Agent(
+            self._model,
+            output_type=SelectorMap,
+            instructions=self._base_instructions,
+            retries=self._retries,
         )
 
         @agent.output_validator

src/crawlee/storage_clients/_redis/_client_mixin.py

@@ -11,7 +11,7 @@
 from ._utils import await_redis_response, read_lua_script
 
 if TYPE_CHECKING:
-    from collections.abc import AsyncIterator
+    from collections.abc import AsyncGenerator
 
     from redis.asyncio import Redis
     from redis.asyncio.client import Pipeline
@@ -175,7 +175,7 @@ async def _ensure_scripts_loaded(self) -> None:
             self._scripts_loaded = True
 
     @asynccontextmanager
-    async def _get_pipeline(self, *, with_execute: bool = True) -> AsyncIterator[Pipeline]:
+    async def _get_pipeline(self, *, with_execute: bool = True) -> AsyncGenerator[Pipeline]:
         """Create a new Redis pipeline."""
         async with self._redis.pipeline() as pipe:
             try:

src/crawlee/storage_clients/_sql/_client_mixin.py

@@ -16,7 +16,7 @@
 from crawlee._utils.crypto import crypto_random_object_id
 
 if TYPE_CHECKING:
-    from collections.abc import AsyncIterator
+    from collections.abc import AsyncGenerator
 
     from sqlalchemy import Insert
     from sqlalchemy.ext.asyncio import AsyncSession
@@ -193,7 +193,7 @@ async def _safely_open(
         return client
 
     @asynccontextmanager
-    async def get_session(self, *, with_simple_commit: bool = False) -> AsyncIterator[AsyncSession]:
+    async def get_session(self, *, with_simple_commit: bool = False) -> AsyncGenerator[AsyncSession]:
         """Create a new SQLAlchemy session for this storage."""
         async with self._storage_client.create_session() as session:
             # For operations where a final commit is mandatory and does not require specific processing conditions

src/crawlee/storage_clients/_sql/_dataset_client.py

@@ -3,7 +3,7 @@
 from collections.abc import Sequence
 from datetime import datetime, timezone
 from logging import getLogger
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, ClassVar
 
 from sqlalchemy import Select, insert, select
 from sqlalchemy import func as sql_func
@@ -58,16 +58,16 @@ class SqlDatasetClient(DatasetClient, SqlClientMixin):
     _DEFAULT_NAME = 'default'
     """Default dataset name used when no name is provided."""
 
-    _METADATA_TABLE = DatasetMetadataDb
+    _METADATA_TABLE: ClassVar[type[DatasetMetadataDb]] = DatasetMetadataDb
     """SQLAlchemy model for dataset metadata."""
 
-    _ITEM_TABLE = DatasetItemDb
+    _ITEM_TABLE: ClassVar[type[DatasetItemDb]] = DatasetItemDb
     """SQLAlchemy model for dataset items."""
 
     _CLIENT_TYPE = 'Dataset'
     """Human-readable client type for error messages."""
 
-    _BUFFER_TABLE = DatasetMetadataBufferDb
+    _BUFFER_TABLE: ClassVar[type[DatasetMetadataBufferDb]] = DatasetMetadataBufferDb
     """SQLAlchemy model for metadata buffer."""
 
     def __init__(

src/crawlee/storage_clients/_sql/_key_value_store_client.py

@@ -3,7 +3,7 @@
 import json
 from datetime import datetime, timezone
 from logging import getLogger
-from typing import TYPE_CHECKING, Any, cast
+from typing import TYPE_CHECKING, Any, ClassVar, cast
 
 from sqlalchemy import CursorResult, delete, select
 from sqlalchemy import func as sql_func
@@ -69,16 +69,16 @@ class SqlKeyValueStoreClient(KeyValueStoreClient, SqlClientMixin):
     A single record larger than this is still read, but alone in its batch.
     """
 
-    _METADATA_TABLE = KeyValueStoreMetadataDb
+    _METADATA_TABLE: ClassVar[type[KeyValueStoreMetadataDb]] = KeyValueStoreMetadataDb
     """SQLAlchemy model for key-value store metadata."""
 
-    _ITEM_TABLE = KeyValueStoreRecordDb
+    _ITEM_TABLE: ClassVar[type[KeyValueStoreRecordDb]] = KeyValueStoreRecordDb
     """SQLAlchemy model for key-value store items."""
 
     _CLIENT_TYPE = 'Key-value store'
     """Human-readable client type for error messages."""
 
-    _BUFFER_TABLE = KeyValueStoreMetadataBufferDb
+    _BUFFER_TABLE: ClassVar[type[KeyValueStoreMetadataBufferDb]] = KeyValueStoreMetadataBufferDb
     """SQLAlchemy model for metadata buffer."""
 
     def __init__(

src/crawlee/storage_clients/_sql/_request_queue_client.py

@@ -5,7 +5,7 @@
 from functools import lru_cache
 from hashlib import sha256
 from logging import getLogger
-from typing import TYPE_CHECKING, Any, cast
+from typing import TYPE_CHECKING, Any, ClassVar, cast
 
 from sqlalchemy import CursorResult, exists, func, or_, select, update
 from sqlalchemy.exc import SQLAlchemyError
@@ -84,10 +84,10 @@ class SqlRequestQueueClient(RequestQueueClient, SqlClientMixin):
     database load).
     """
 
-    _METADATA_TABLE = RequestQueueMetadataDb
+    _METADATA_TABLE: ClassVar[type[RequestQueueMetadataDb]] = RequestQueueMetadataDb
     """SQLAlchemy model for request queue metadata."""
 
-    _ITEM_TABLE = RequestDb
+    _ITEM_TABLE: ClassVar[type[RequestDb]] = RequestDb
     """SQLAlchemy model for request items."""
 
     _CLIENT_TYPE = 'Request queue'
@@ -97,7 +97,7 @@ class SqlRequestQueueClient(RequestQueueClient, SqlClientMixin):
     """Number of seconds for which a request is considered blocked in the database after being fetched for processing.
     """
 
-    _BUFFER_TABLE = RequestQueueMetadataBufferDb
+    _BUFFER_TABLE: ClassVar[type[RequestQueueMetadataBufferDb]] = RequestQueueMetadataBufferDb
     """SQLAlchemy model for metadata buffer."""
 
     def __init__(

tests/unit/_utils/test_sitemap.py

@@ -31,7 +31,7 @@
 )
 
 if TYPE_CHECKING:
-    from collections.abc import AsyncIterator, Callable
+    from collections.abc import AsyncGenerator, AsyncIterator, Callable
 
 
 def _make_mock_client(url_map: dict[str, tuple[int, bytes]]) -> AsyncMock:
@@ -56,7 +56,7 @@ def _make_flaky_stream_client(body: bytes, *, fail_times: int) -> tuple[AsyncMoc
     attempts: list[int] = []
 
     @asynccontextmanager
-    async def stream(_url: str, **_kwargs: Any) -> 'AsyncIterator[HttpResponse]':
+    async def stream(_url: str, **_kwargs: Any) -> 'AsyncGenerator[HttpResponse]':
         attempt = len(attempts) + 1
         attempts.append(attempt)
         if attempt <= fail_times:
@@ -81,7 +81,7 @@ def _make_stream_client(body_for_url: 'Callable[[str], bytes]') -> tuple[AsyncMo
     fetched: list[str] = []
 
     @asynccontextmanager
-    async def stream(url: str, **_kwargs: Any) -> 'AsyncIterator[HttpResponse]':
+    async def stream(url: str, **_kwargs: Any) -> 'AsyncGenerator[HttpResponse]':
         fetched.append(url)
 
         async def read_stream() -> 'AsyncIterator[bytes]':
@@ -599,7 +599,7 @@ async def test_gzip_sitemap_stops_reading_after_member_end() -> None:
     chunks_read = 0
 
     @asynccontextmanager
-    async def stream(_url: str, **_kwargs: Any) -> 'AsyncIterator[HttpResponse]':
+    async def stream(_url: str, **_kwargs: Any) -> 'AsyncGenerator[HttpResponse]':
         async def read_stream() -> 'AsyncIterator[bytes]':
             nonlocal chunks_read
             chunks_read += 1
@@ -631,7 +631,7 @@ async def test_gzip_text_sitemap_parsed_by_extension_when_served_as_gzip() -> No
     body = compress_gzip('\n'.join(urls))
 
     @asynccontextmanager
-    async def stream(_url: str, **_kwargs: Any) -> 'AsyncIterator[HttpResponse]':
+    async def stream(_url: str, **_kwargs: Any) -> 'AsyncGenerator[HttpResponse]':
         async def read_stream() -> 'AsyncIterator[bytes]':
             yield body
 

tests/unit/request_loaders/test_sitemap_request_loader.py

@@ -23,7 +23,7 @@
 )
 
 if TYPE_CHECKING:
-    from collections.abc import AsyncIterator
+    from collections.abc import AsyncGenerator, AsyncIterator
 
     from crawlee._types import JsonSerializable
 
@@ -43,7 +43,7 @@ async def test_nested_sitemap_chain_bounded_by_max_depth() -> None:
     fetched: list[str] = []
 
     @asynccontextmanager
-    async def stream(url: str, **_kwargs: Any) -> 'AsyncIterator[HttpResponse]':
+    async def stream(url: str, **_kwargs: Any) -> 'AsyncGenerator[HttpResponse]':
         fetched.append(url)
         index = int(url.removesuffix('.xml').rsplit('-', 1)[-1])
         # Every sitemap is an index pointing at the next unique URL, so only the depth cap can stop the chain.

tests/unit/test_throttling_request_manager.py

@@ -20,7 +20,7 @@
 from crawlee.storages import RequestQueue
 
 if TYPE_CHECKING:
-    from collections.abc import Iterator
+    from collections.abc import Generator
 
 THROTTLED_DOMAIN = 'throttled.com'
 SECOND_THROTTLED_DOMAIN = 'slow.com'
@@ -86,7 +86,7 @@ def _make_request(url: str) -> Request:
 
 
 @contextmanager
-def _frozen_clock() -> Iterator[MagicMock]:
+def _frozen_clock() -> Generator[MagicMock]:
     """Freeze the manager's clock at `CLOCK_START`. Move it by reassigning `clock.now.return_value`."""
     with patch(f'{MANAGER_MODULE}.datetime') as clock:
         clock.now.return_value = CLOCK_START

tests/unit/utils.py

@@ -16,7 +16,7 @@
 from crawlee.storages import RequestQueue
 
 if TYPE_CHECKING:
-    from collections.abc import AsyncIterator, Awaitable, Callable
+    from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable
 
     from yarl import URL
 
@@ -40,7 +40,7 @@ def make_status_stream_client(
     indexes: dict[str, int] = {}
 
     @asynccontextmanager
-    async def stream(url: str, **_kwargs: Any) -> AsyncIterator[HttpResponse]:
+    async def stream(url: str, **_kwargs: Any) -> AsyncGenerator[HttpResponse]:
         sequence = responses[url]
         index = indexes.get(url, 0)
         indexes[url] = index + 1