Merge pull request #44 from modern-python/perf/can-decode-result-cache

perf(decoders): memoize can_decode verdict per model (0.9.2)

Author: lesnik512

planning/releases/0.9.2.md

@@ -0,0 +1,13 @@
+# httpware 0.9.2 — decoder `can_decode` verdicts are memoized
+
+**Patch release, no behavior change — a performance fix.** `MsgspecDecoder.can_decode` and `PydanticDecoder.can_decode` now cache their per-model verdict, so the expensive type probe runs once per `response_model` type instead of on every request.
+
+## The gap
+
+The client consults `decoder.can_decode(model)` on every dispatch to route a `response_model=` to the right decoder. After the 0.9.1 nested-`CustomType` fix, `MsgspecDecoder.can_decode` ran an uncached `msgspec.inspect.type_info()` call plus a recursive tree walk on each invocation — roughly **30 µs per call** for a type like `list[SomeStruct]`. `PydanticDecoder.can_decode` had the milder version of the same shape: it re-probed `TypeAdapter` construction every call and never cached a rejection. Since a client decodes the same handful of model types over and over, that cost was paid on every single request.
+
+## The fix
+
+Both decoders gain a per-instance `dict[type, bool]` verdict cache. `can_decode` returns the memoized result when present and only runs the probe on the first sighting of a model type. Repeat calls drop from ~30 µs to ~0.15 µs (msgspec) / ~0.07 µs (pydantic) — a ~200× reduction on the hot path. Unhashable models skip the cache and probe fresh, preserving the existing fallback behavior. The cache is bounded by the set of `response_model` types an application actually uses, mirroring the existing per-instance decoder/adapter caches.
+
+No public-API change: `can_decode`'s signature, the `ResponseDecoder` protocol, and every routing verdict are unchanged — only repeated computation is removed.

src/httpware/decoders/msgspec.py

@@ -54,11 +54,13 @@ class MsgspecDecoder:
     """
 
     _msgspec_decoders: dict[type, "msgspec.json.Decoder[typing.Any]"]
+    _can_decode_results: dict[type, bool]
 
     def __init__(self) -> None:
         if not import_checker.is_msgspec_installed:
             raise ImportError(MISSING_DEPENDENCY_MESSAGE)
         self._msgspec_decoders = {}
+        self._can_decode_results = {}
 
     def _get_msgspec_decoder(self, model: type[T]) -> "msgspec.json.Decoder[T]":
         decoder = self._msgspec_decoders.get(model)
@@ -70,6 +72,23 @@ def _get_msgspec_decoder(self, model: type[T]) -> "msgspec.json.Decoder[T]":
     def can_decode(self, model: type) -> bool:
         """Return True iff msgspec natively understands `model` end-to-end.
 
+        The verdict is memoized per `model`: the probe below (an uncached
+        `type_info` call plus a recursive tree walk) runs once per type, not on
+        every dispatch. Unhashable models skip the cache and probe fresh.
+        """
+        try:
+            cached = self._can_decode_results.get(model)
+        except TypeError:  # unhashable model — can't memoize, probe fresh
+            return self._probe_can_decode(model)
+        if cached is not None:
+            return cached
+        result = self._probe_can_decode(model)
+        self._can_decode_results[model] = result
+        return result
+
+    def _probe_can_decode(self, model: type) -> bool:
+        """Decide whether msgspec natively decodes `model` (the uncached path).
+
         msgspec builds a Decoder for almost any class via a generic CustomType
         fallback; the Decoder constructor does NOT raise on unsupported types
         (e.g. pydantic.BaseModel, or a container parameterized by one). We walk

src/httpware/decoders/pydantic.py

@@ -26,11 +26,13 @@ class PydanticDecoder:
     """Decode raw response bytes into `model` via a per-instance cached `pydantic.TypeAdapter`."""
 
     _adapters: dict[type, TypeAdapter[typing.Any]]
+    _can_decode_results: dict[type, bool]
 
     def __init__(self) -> None:
         if not import_checker.is_pydantic_installed:
             raise ImportError(MISSING_DEPENDENCY_MESSAGE)
         self._adapters = {}
+        self._can_decode_results = {}
 
     def _get_adapter(self, model: type[T]) -> "TypeAdapter[T]":
         adapter = self._adapters.get(model)
@@ -42,6 +44,23 @@ def _get_adapter(self, model: type[T]) -> "TypeAdapter[T]":
     def can_decode(self, model: type) -> bool:
         """Return True iff pydantic can build a schema for `model`.
 
+        The verdict is memoized per `model` so a rejection (which costs a
+        `PydanticSchemaGenerationError` round-trip) is not re-probed on every
+        dispatch. Unhashable models skip the cache and probe fresh.
+        """
+        try:
+            cached = self._can_decode_results.get(model)
+        except TypeError:  # unhashable model — can't memoize, probe fresh
+            return self._probe_can_decode(model)
+        if cached is not None:
+            return cached
+        result = self._probe_can_decode(model)
+        self._can_decode_results[model] = result
+        return result
+
+    def _probe_can_decode(self, model: type) -> bool:
+        """Decide whether pydantic can build a schema for `model` (uncached).
+
         Probes via `_get_adapter`; subsequent calls (including `decode`) reuse
         the cached `TypeAdapter`. Rejects `msgspec.Struct` subclasses —
         pydantic raises `PydanticSchemaGenerationError` (a `TypeError`) when

tests/test_decoders_msgspec.py

@@ -2,7 +2,7 @@
 
 import dataclasses
 from http import HTTPStatus
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
 
 import httpx2
 import msgspec
@@ -196,3 +196,37 @@ def test_msgspec_still_accepts_native_containers(model: type) -> None:
     plain builtin element types.
     """
     assert MsgspecDecoder().can_decode(model) is True
+
+
+def test_msgspec_can_decode_result_is_cached() -> None:
+    """Repeat can_decode calls reuse a cached verdict, not the per-dispatch probe."""
+    decoder = MsgspecDecoder()
+    with patch(
+        "httpware.decoders.msgspec.msgspec.inspect.type_info",
+        wraps=msgspec.inspect.type_info,
+    ) as spy:
+        assert decoder.can_decode(_Item) is True
+        assert decoder.can_decode(_Item) is True
+    assert spy.call_count == 1
+    assert decoder._can_decode_results[_Item] is True  # noqa: SLF001
+
+
+def test_msgspec_can_decode_caches_negative_verdict() -> None:
+    """A rejection is cached too, so repeat probes don't repeat the walk."""
+    decoder = MsgspecDecoder()
+    with patch(
+        "httpware.decoders.msgspec.msgspec.inspect.type_info",
+        wraps=msgspec.inspect.type_info,
+    ) as spy:
+        assert decoder.can_decode(_PydanticUser) is False
+        assert decoder.can_decode(_PydanticUser) is False
+    assert spy.call_count == 1
+    assert decoder._can_decode_results[_PydanticUser] is False  # noqa: SLF001
+
+
+def test_msgspec_can_decode_unhashable_model_does_not_raise() -> None:
+    """An unhashable model falls back to a fresh probe instead of raising from the cache."""
+    decoder = MsgspecDecoder()
+    decoder._can_decode_results = MagicMock()  # noqa: SLF001
+    decoder._can_decode_results.get.side_effect = TypeError("unhashable type")  # noqa: SLF001
+    assert decoder.can_decode(_Item) is True

tests/test_decoders_pydantic.py

@@ -3,7 +3,7 @@
 import asyncio
 import concurrent.futures
 import dataclasses
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
 
 import msgspec
 import pydantic
@@ -211,3 +211,31 @@ def test_pydantic_can_decode_uses_cache() -> None:
     decoder.can_decode(User)
     assert len(decoder._adapters) == 1  # noqa: SLF001
     assert User in decoder._adapters  # noqa: SLF001
+
+
+def test_pydantic_can_decode_result_is_cached() -> None:
+    """Repeat can_decode calls reuse a cached verdict, not the per-dispatch probe."""
+    decoder = PydanticDecoder()
+    with patch.object(decoder, "_get_adapter", wraps=decoder._get_adapter) as spy:  # noqa: SLF001
+        assert decoder.can_decode(User) is True
+        assert decoder.can_decode(User) is True
+    assert spy.call_count == 1
+    assert decoder._can_decode_results[User] is True  # noqa: SLF001
+
+
+def test_pydantic_can_decode_caches_negative_verdict() -> None:
+    """A rejection is cached too, so repeat probes skip the schema-build round-trip."""
+    decoder = PydanticDecoder()
+    with patch.object(decoder, "_get_adapter", wraps=decoder._get_adapter) as spy:  # noqa: SLF001
+        assert decoder.can_decode(_Struct) is False
+        assert decoder.can_decode(_Struct) is False
+    assert spy.call_count == 1
+    assert decoder._can_decode_results[_Struct] is False  # noqa: SLF001
+
+
+def test_pydantic_can_decode_unhashable_model_does_not_raise() -> None:
+    """An unhashable model falls back to a fresh probe instead of raising from the cache."""
+    decoder = PydanticDecoder()
+    decoder._can_decode_results = MagicMock()  # noqa: SLF001
+    decoder._can_decode_results.get.side_effect = TypeError("unhashable type")  # noqa: SLF001
+    assert decoder.can_decode(User) is True