From 5decbecc02da085c692cfae8e55cfa21aeea0d39 Mon Sep 17 00:00:00 2001 From: Nitesh Dhanpal Date: Thu, 6 Aug 2026 23:43:41 -0700 Subject: [PATCH 1/9] feat(tracing): per-step obs wrappers inside business Temporal activities (1:1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously _begin_obs skipped the obs wrapper for ANY Temporal activity (Option A) and only stamped the ambient RunActivity span, so all business spans in a turn collapsed onto ONE obs span (52:1). But inside a *business* activity, start_span and end_span run in the SAME process, so a wrapper is safe there. Option A is only required for the SDK's own dispatched START_SPAN/END_SPAN activities (the in_temporal_workflow path), where start and end are separate activities on possibly different workers. Discriminate on activity type: _in_tracing_dispatch_activity() is true only for the "start-span"/"end-span" activities. For everything else (sync, or a business activity) open a real per-step wrapper — it nests under the interceptor's ambient RunActivity span and closes in-process, giving each business span its own obs span (1:1), matching the sync path. The bounded _OBS_HANDLES registry backstops any mis-discrimination. --- src/agentex/lib/core/tracing/obs_span.py | 8 +-- src/agentex/lib/core/tracing/trace.py | 70 ++++++++++-------------- tests/test_temporal_obs_backend.py | 35 +++++++++++- 3 files changed, 68 insertions(+), 45 deletions(-) diff --git a/src/agentex/lib/core/tracing/obs_span.py b/src/agentex/lib/core/tracing/obs_span.py index 385507269..e213b13b2 100644 --- a/src/agentex/lib/core/tracing/obs_span.py +++ b/src/agentex/lib/core/tracing/obs_span.py @@ -241,10 +241,10 @@ def tag_ambient_obs_span( """Stamp the reverse tag onto the CURRENTLY ACTIVE obs span -- without opening a new one. - Used on the Temporal path (see ``trace._in_temporal_activity``): there we must - NOT open our own wrapper span, because start_span/end_span run as separate - activities on possibly different workers and the wrapper could never be - closed. Instead we lean on the span the Temporal OTel ``TracingInterceptor`` + Used inside the SDK's dispatched start-span/end-span activities (see + ``trace._in_tracing_dispatch_activity``): there we must NOT open our own + wrapper span, because start_span/end_span run as separate activities on + possibly different workers and the wrapper could never be closed. Instead we lean on the span the Temporal OTel ``TracingInterceptor`` already made active for this activity and just add ``agentex.business_span_id`` / ``agentex.business_trace_id`` so the obs -> business pivot still works. Best-effort; never raises. diff --git a/src/agentex/lib/core/tracing/trace.py b/src/agentex/lib/core/tracing/trace.py index d3decdb9b..447ed7d9b 100644 --- a/src/agentex/lib/core/tracing/trace.py +++ b/src/agentex/lib/core/tracing/trace.py @@ -106,39 +106,25 @@ def _run_on_span_end(processor: SyncTracingProcessor, span: Span) -> None: ) -def _in_temporal_activity() -> bool: - """True when executing inside a Temporal activity. - - On the Temporal path ``start_span`` and ``end_span`` run as SEPARATE - activities (START_SPAN / END_SPAN) that Temporal can route to DIFFERENT - worker processes. A wrapper obs span opened in the START_SPAN activity could - therefore never be closed by END_SPAN -- its handle lives in another - process's ``_OBS_HANDLES`` -- so it would leak (unbounded, OOM risk) and its - persisted ``obs_span_id`` would dangle (the span is never .end()ed, so never - exported to Tempo). - - So inside an activity we do NOT open our own wrapper. We lean on the span the - Temporal OTel ``TracingInterceptor`` (see ``core/tracing/temporal.py`` + - scale-agentex-python#485) already made active for this activity -- which is - rooted under the turn's propagated trace -- and merely stamp the reverse tag - onto it (``tag_ambient_obs_span``). That keeps trace-level correlation with - no cross-process handle to leak. - - Never raises; returns False when temporalio isn't importable. - - TODO(obs-followup): this intentionally drops the *named per-step* wrapper on - the Temporal path (obs_span_id becomes the ambient activity span, not a - step-named span) and does NOT add TurnTrace RETRY/ASYNC roll-up -- retried - turns still surface as N unlinked spans. Follow-up diff should (a) optionally - materialize a self-contained named wrapper inside a single activity using the - span's own start/end timestamps, and (b) build the TurnTrace roll-up. - Test-later: on a multi-replica worker fleet, assert _OBS_HANDLES stays - bounded (no leak / OOM) and that obs_trace_id resolves to the turn trace. - """ +def _in_tracing_dispatch_activity() -> bool: + """True only when running inside the SDK's OWN dispatched START_SPAN / END_SPAN + activity (the ``in_temporal_workflow()`` path, where a workflow runs span start + and end as SEPARATE activities that Temporal can route to different workers). + + That is the one case a per-step obs wrapper can't work: the wrapper opened in + the START_SPAN activity could never be closed by the END_SPAN activity. A span + created directly inside a *business* activity (an agent turn's own + ``adk.tracing.span``) runs start AND end in the same activity process, so a + wrapper there is safe -- it nests under the interceptor's ambient RunActivity + span and closes in-process. The tracing dispatch activities are named + ``start-span`` / ``end-span`` (``TracingActivityName``). Never raises; False + when temporalio isn't importable or we're not in an activity.""" try: from temporalio import activity - return activity.in_activity() + if not activity.in_activity(): + return False + return activity.info().activity_type in ("start-span", "end-span") except Exception: return False @@ -148,23 +134,27 @@ def _begin_obs( span_id: str, trace_id: str | None, ) -> tuple[ObsSpanHandle | None, dict[str, str]]: - """Open the obs wrapper for a business span (or, inside a Temporal activity, - tag the ambient interceptor span) and return ``(handle, correlation)``. + """Open the obs wrapper for a business span and return ``(handle, correlation)``. Shared by ``Trace.start_span`` and ``AsyncTrace.start_span`` so the two paths can't drift. The wrapper is named for the step so ``obs_span_id`` is stable/meaningful (not an arbitrary innermost httpx span), and it carries the reverse tag (business span/trace id) for the obs -> business pivot. - Temporal path: we do NOT open our own wrapper -- start_span / end_span run as - separate activities on possibly different workers, so the handle could never - be closed. Instead we tag the span the temporalio OTel ``TracingInterceptor`` - already made active. That span is OTel REGARDLESS of ``SGP_OBS_MODE``, so we - pass ``prefer_otel=True`` to both the tag and the correlation read -- otherwise - the default ``dd_only`` mode would tag/read an unrelated ddtrace span and the - ids would point at the wrong trace. See ``_in_temporal_activity``. + We open a real per-step wrapper on the sync path AND inside a *business* + Temporal activity -- there the wrapper nests under the interceptor's ambient + RunActivity span and start/end run in-process, so it closes cleanly and each + business step gets its own obs span (1:1), just like sync. + + The ONE exception is the SDK's own dispatched START_SPAN / END_SPAN activity + (a workflow calling ``adk.tracing`` -- see ``_in_tracing_dispatch_activity``): + there start and end are separate activities on possibly different workers, so + a wrapper could never be closed. We fall back to tagging the ambient + interceptor span instead, with ``prefer_otel=True`` (the interceptor span is + OTel regardless of ``SGP_OBS_MODE``, so a plain ``dd_only`` read would + otherwise point at an unrelated ddtrace span). """ - if _in_temporal_activity(): + if _in_tracing_dispatch_activity(): tag_ambient_obs_span(business_span_id=span_id, business_trace_id=trace_id, prefer_otel=True) return None, obs_correlation(prefer_otel=True) handle = open_obs_span(name, business_span_id=span_id, business_trace_id=trace_id) diff --git a/tests/test_temporal_obs_backend.py b/tests/test_temporal_obs_backend.py index 34daf1d11..05d3a6bfb 100644 --- a/tests/test_temporal_obs_backend.py +++ b/tests/test_temporal_obs_backend.py @@ -67,8 +67,10 @@ def _activate_otel_span(monkeypatch: pytest.MonkeyPatch) -> _RecordingOtelSpan: def test_temporal_path_tags_and_reads_otel_in_dd_only(monkeypatch: pytest.MonkeyPatch) -> None: # Default/dd_only mode is exactly where the old code went to ddtrace. + # Tagging the ambient interceptor span (no wrapper) now applies only inside + # the SDK's dispatched START_SPAN/END_SPAN activity, not any activity. monkeypatch.setenv("SGP_OBS_MODE", "dd_only") - monkeypatch.setattr(trace_mod, "_in_temporal_activity", lambda: True) + monkeypatch.setattr(trace_mod, "_in_tracing_dispatch_activity", lambda: True) activity_span = _activate_otel_span(monkeypatch) trace_obj = Trace(processors=[], client=cast(Any, object()), trace_id="trace-1") @@ -132,3 +134,34 @@ def current_span(self) -> _FakeDDSpan: assert tagged["agentex.business_trace_id"] == "bt" # The invalid OTel span was NOT tagged. assert invalid.attributes == {} + + +class _FakeHandle: + def __init__(self, corr): + self.correlation = corr + + +def test_begin_obs_opens_wrapper_outside_dispatch_activity(monkeypatch: pytest.MonkeyPatch) -> None: + """Sync path or inside a business Temporal activity: open a per-step wrapper + (1:1), not the ambient-span tag. Each business span gets its own obs span.""" + monkeypatch.setattr(trace_mod, "_in_tracing_dispatch_activity", lambda: False) + monkeypatch.setattr( + trace_mod, "open_obs_span", + lambda *a, **k: _FakeHandle({"obs_trace_id": "t1", "obs_span_id": "s1"}), + ) + handle, corr = trace_mod._begin_obs("mortgage.classify_intent", "bs", "bt") + assert handle is not None + assert corr == {"obs_trace_id": "t1", "obs_span_id": "s1"} + + +def test_begin_obs_tags_ambient_inside_dispatch_activity(monkeypatch: pytest.MonkeyPatch) -> None: + """Inside the dispatched START_SPAN/END_SPAN activity: no wrapper (would leak + across activities); tag the ambient interceptor span instead.""" + monkeypatch.setattr(trace_mod, "_in_tracing_dispatch_activity", lambda: True) + tagged: dict = {} + monkeypatch.setattr(trace_mod, "tag_ambient_obs_span", lambda **k: tagged.update(k)) + monkeypatch.setattr(trace_mod, "obs_correlation", lambda **k: {"obs_trace_id": "amb", "obs_span_id": "amb"}) + handle, corr = trace_mod._begin_obs("mortgage.advisor.turn", "bs", "bt") + assert handle is None + assert tagged.get("business_span_id") == "bs" and tagged.get("prefer_otel") is True + assert corr == {"obs_trace_id": "amb", "obs_span_id": "amb"} From 75dd451795cd51fba4718900ee0dea6c517126c9 Mon Sep 17 00:00:00 2001 From: Nitesh Dhanpal Date: Mon, 10 Aug 2026 15:50:49 -0700 Subject: [PATCH 2/9] fix(tracing): prefer OTel for the per-step wrapper inside business activities In dd_only (the default), the wrapper branch of _begin_obs read ddtrace for both open_obs_span and the obs_correlation fallback. Inside a Temporal worker there is no ddtrace request context, so open_obs_span returned None and the fallback obs_correlation() returned {} -- every business span in an async turn persisted with no obs_trace_id/obs_span_id. The ambient span in an activity is the temporalio OTel TracingInterceptor span regardless of SGP_OBS_MODE (same reasoning already applied to the dispatch/tag branch). Fix: thread prefer_otel through the wrapper branch, keyed on "in any Temporal activity" (restored _in_temporal_activity). open_obs_span gains a prefer_otel param that opens an OTel wrapper first (falling back to ddtrace). This also restores real per-step OTel wrapper spans in dd_only business activities, not just the ids. Adds a dd_only-inside-a-business-activity regression test (asserts OTel ids win over ddtrace, not empty); the earlier verification ran in lgtm so it missed this. Co-Authored-By: Claude Opus 4.8 --- src/agentex/lib/core/tracing/obs_span.py | 19 +++++++++++++-- src/agentex/lib/core/tracing/trace.py | 31 ++++++++++++++++++++++-- tests/lib/core/tracing/test_obs_span.py | 2 +- tests/test_temporal_obs_backend.py | 23 ++++++++++++++++++ 4 files changed, 70 insertions(+), 5 deletions(-) diff --git a/src/agentex/lib/core/tracing/obs_span.py b/src/agentex/lib/core/tracing/obs_span.py index e213b13b2..a8ecf38b1 100644 --- a/src/agentex/lib/core/tracing/obs_span.py +++ b/src/agentex/lib/core/tracing/obs_span.py @@ -175,6 +175,7 @@ def open_obs_span( name: str, business_span_id: Optional[str] = None, business_trace_id: Optional[str] = None, + prefer_otel: bool = False, ) -> Optional[ObsSpanHandle]: """Open an obs span named ``name`` in the active backend, make it the active span, and return a handle carrying its ``{"obs_trace_id","obs_span_id"}``. @@ -183,6 +184,14 @@ def open_obs_span( the reverse tag (``agentex.business_span_id`` / ``agentex.business_trace_id``) so you can pivot obs -> business by searching them in Tempo/DD. + ``prefer_otel``: open an OTel wrapper first, regardless of ``SGP_OBS_MODE``. + Set on the Temporal path, where the ambient span is the temporalio OTel + ``TracingInterceptor`` span regardless of mode -- an OTel wrapper nests under + it and yields valid ids, whereas the default ``dd_only`` path would open a + ddtrace wrapper, which finds no request context in a worker and returns None + (dropping the per-step span and its ids). Falls back to ddtrace if no OTel + span materializes. + Returns ``None`` (so the caller falls back to ambient behavior) when the backend tracer isn't available or, in ``dd_only``, no request trace is active. @@ -192,8 +201,14 @@ def open_obs_span( never fail an app call. """ try: - if get_obs_mode() == LGTM: - return _open_otel_span(name, business_span_id, business_trace_id) + if prefer_otel or get_obs_mode() == LGTM: + handle = _open_otel_span(name, business_span_id, business_trace_id) + if handle is not None or not prefer_otel: + # In lgtm mode a None handle means "no OTel span -> caller uses the + # ambient fallback". Only when prefer_otel is set (Temporal path) + # do we try ddtrace as a second choice. + return handle + return _open_ddtrace_span(name, business_span_id, business_trace_id) return _open_ddtrace_span(name, business_span_id, business_trace_id) except Exception: # pragma: no cover - backstop; obs must never break a call return None diff --git a/src/agentex/lib/core/tracing/trace.py b/src/agentex/lib/core/tracing/trace.py index 447ed7d9b..6decc6f70 100644 --- a/src/agentex/lib/core/tracing/trace.py +++ b/src/agentex/lib/core/tracing/trace.py @@ -129,6 +129,23 @@ def _in_tracing_dispatch_activity() -> bool: return False +def _in_temporal_activity() -> bool: + """True inside ANY Temporal activity. There the ambient span is the temporalio + OTel ``TracingInterceptor`` span REGARDLESS of ``SGP_OBS_MODE``, so callers + prefer OTel for both the wrapper backend and the correlation read: a plain + ``dd_only`` read would target ddtrace, which has no request context in a worker + (no inbound HTTP), so ``open_obs_span`` would return None and the fallback ids + would be empty -- the business span would persist with no obs_* ids at all. + Never raises; False when temporalio isn't importable or we're not in an + activity.""" + try: + from temporalio import activity + + return activity.in_activity() + except Exception: + return False + + def _begin_obs( name: str, span_id: str, @@ -153,12 +170,22 @@ def _begin_obs( interceptor span instead, with ``prefer_otel=True`` (the interceptor span is OTel regardless of ``SGP_OBS_MODE``, so a plain ``dd_only`` read would otherwise point at an unrelated ddtrace span). + + Inside ANY activity we also pass ``prefer_otel`` to the wrapper and the ambient + fallback: the ambient span is the interceptor's OTel span regardless of mode, + so a per-step OTel wrapper nests under it and yields valid ids, whereas the + default ``dd_only`` path would open a ddtrace wrapper -- which finds no request + context in a worker and returns None, leaving the business span with empty + obs_* ids. """ if _in_tracing_dispatch_activity(): tag_ambient_obs_span(business_span_id=span_id, business_trace_id=trace_id, prefer_otel=True) return None, obs_correlation(prefer_otel=True) - handle = open_obs_span(name, business_span_id=span_id, business_trace_id=trace_id) - correlation = handle.correlation if handle is not None else obs_correlation() + prefer_otel = _in_temporal_activity() + handle = open_obs_span( + name, business_span_id=span_id, business_trace_id=trace_id, prefer_otel=prefer_otel + ) + correlation = handle.correlation if handle is not None else obs_correlation(prefer_otel=prefer_otel) return handle, correlation diff --git a/tests/lib/core/tracing/test_obs_span.py b/tests/lib/core/tracing/test_obs_span.py index a7f40a511..a9c5739bc 100644 --- a/tests/lib/core/tracing/test_obs_span.py +++ b/tests/lib/core/tracing/test_obs_span.py @@ -351,7 +351,7 @@ def test_lgtm_wrapper_marked_error_when_business_step_raises(self, monkeypatch): def test_dd_only_no_ctx_falls_back_to_ambient(self, monkeypatch): monkeypatch.setenv("SGP_OBS_MODE", "dd_only") _install_fake_ddtrace(monkeypatch, active=False) - monkeypatch.setattr("agentex.lib.core.tracing.trace.obs_correlation", lambda: {}) + monkeypatch.setattr("agentex.lib.core.tracing.trace.obs_correlation", lambda **_k: {}) trace = Trace(processors=[], client=MagicMock(), trace_id="task-run-3") span = trace.start_span(name="get_state") diff --git a/tests/test_temporal_obs_backend.py b/tests/test_temporal_obs_backend.py index 05d3a6bfb..8de6f20bc 100644 --- a/tests/test_temporal_obs_backend.py +++ b/tests/test_temporal_obs_backend.py @@ -165,3 +165,26 @@ def test_begin_obs_tags_ambient_inside_dispatch_activity(monkeypatch: pytest.Mon assert handle is None assert tagged.get("business_span_id") == "bs" and tagged.get("prefer_otel") is True assert corr == {"obs_trace_id": "amb", "obs_span_id": "amb"} + + +def test_business_activity_dd_only_reads_otel_not_ddtrace(monkeypatch: pytest.MonkeyPatch) -> None: + """Regression: inside a BUSINESS activity (not the dispatched start/end-span) + with default dd_only mode, the per-step wrapper branch must prefer OTel. The + ambient span is the temporalio OTel interceptor span regardless of mode; a + dd_only ddtrace read finds no request context in a worker, so before the fix + the business span persisted with empty obs ids. Assert it carries the OTel + activity ids, not ddtrace's.""" + monkeypatch.setenv("SGP_OBS_MODE", "dd_only") + monkeypatch.setattr(trace_mod, "_in_tracing_dispatch_activity", lambda: False) + monkeypatch.setattr(trace_mod, "_in_temporal_activity", lambda: True) + # Make ddtrace resolve to DIFFERENT ids so we can prove which backend won. + monkeypatch.setattr(obs_ids_mod, "_ddtrace_ids", lambda: ("d" * 32, "e" * 16)) + _activate_otel_span(monkeypatch) # valid OTel span active (the interceptor span) + + trace_obj = Trace(processors=[], client=cast(Any, object()), trace_id="trace-1") + span = trace_obj.start_span(name="mortgage.classify_intent") + + assert isinstance(span.data, dict) + # OTel ids, not ddtrace's ("d"*32) and not empty. + assert span.data["obs_trace_id"] == _TRACE_HEX + assert span.data["obs_span_id"] == _SPAN_HEX From 2bc7a74b77c10ff09b5ef7d401b1b96c0cf0c4cb Mon Sep 17 00:00:00 2001 From: Nitesh Dhanpal Date: Mon, 10 Aug 2026 15:54:10 -0700 Subject: [PATCH 3/9] test(tracing): cover the dispatch discriminator directly The prior tests stubbed _in_tracing_dispatch_activity wholesale, so the actual `activity.info().activity_type in ("start-span", "end-span")` comparison -- the one line preventing a cross-worker handle leak inside START_SPAN -- had no coverage. Add tests that fake activity.info(): - start-span / end-span -> True (asserted against TracingActivityName.value, so this fails if the enum ever drifts from the strings hardcoded in trace.py), - a business activity type (process_mortgage_turn) -> False, - not in an activity -> False (in_activity guard short-circuits before info()), plus a small _in_temporal_activity() truth-table test. Co-Authored-By: Claude Opus 4.8 --- tests/test_temporal_obs_backend.py | 41 ++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/test_temporal_obs_backend.py b/tests/test_temporal_obs_backend.py index 8de6f20bc..8dc325994 100644 --- a/tests/test_temporal_obs_backend.py +++ b/tests/test_temporal_obs_backend.py @@ -11,17 +11,20 @@ from __future__ import annotations +from types import SimpleNamespace from typing import Any, cast import pytest from opentelemetry import trace as otel_trace from opentelemetry.trace import TraceFlags, SpanContext +from temporalio import activity as temporal_activity import agentex.lib.core.tracing.trace as trace_mod import agentex.lib.core.tracing.obs_ids as obs_ids_mod from agentex.lib.core.tracing.trace import _OBS_HANDLES, Trace from agentex.lib.core.tracing.obs_ids import obs_correlation from agentex.lib.core.tracing.obs_span import tag_ambient_obs_span +from agentex.lib.core.temporal.activities.adk.tracing_activities import TracingActivityName _TRACE_ID = 0x0123456789ABCDEF0123456789ABCDEF _SPAN_ID = 0x0123456789ABCDEF @@ -188,3 +191,41 @@ def test_business_activity_dd_only_reads_otel_not_ddtrace(monkeypatch: pytest.Mo # OTel ids, not ddtrace's ("d"*32) and not empty. assert span.data["obs_trace_id"] == _TRACE_HEX assert span.data["obs_span_id"] == _SPAN_HEX + + +# --------------------------------------------------------------------------- # +# The dispatch discriminator itself (trace.py:_in_tracing_dispatch_activity). +# This is the one line preventing a cross-worker handle leak inside START_SPAN, +# so it gets exercised directly with a faked activity.info() -- and against the +# enum's own .value, so it also fails if TracingActivityName ever drifts from +# the strings hardcoded in trace.py. +# --------------------------------------------------------------------------- # +def test_dispatch_discriminator_true_for_start_and_end_span(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(temporal_activity, "in_activity", lambda: True) + for name in (TracingActivityName.START_SPAN, TracingActivityName.END_SPAN): + # activity_type round-trips as the plain string value through protobuf. + monkeypatch.setattr(temporal_activity, "info", lambda n=name: SimpleNamespace(activity_type=n.value)) + assert trace_mod._in_tracing_dispatch_activity() is True, name + + +def test_dispatch_discriminator_false_for_business_activity(monkeypatch: pytest.MonkeyPatch) -> None: + # A real agent-turn activity (e.g. process_mortgage_turn) is NOT a dispatch + # activity -> it must take the per-step wrapper branch, not Option-A tagging. + monkeypatch.setattr(temporal_activity, "in_activity", lambda: True) + monkeypatch.setattr(temporal_activity, "info", lambda: SimpleNamespace(activity_type="process_mortgage_turn")) + assert trace_mod._in_tracing_dispatch_activity() is False + + +def test_dispatch_discriminator_false_when_not_in_activity(monkeypatch: pytest.MonkeyPatch) -> None: + # Guard short-circuits on in_activity()==False; info() (here a dispatch value) + # must never be consulted, else the sync path would be misclassified. + monkeypatch.setattr(temporal_activity, "in_activity", lambda: False) + monkeypatch.setattr(temporal_activity, "info", lambda: SimpleNamespace(activity_type="start-span")) + assert trace_mod._in_tracing_dispatch_activity() is False + + +def test_in_temporal_activity_tracks_in_activity(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(temporal_activity, "in_activity", lambda: True) + assert trace_mod._in_temporal_activity() is True + monkeypatch.setattr(temporal_activity, "in_activity", lambda: False) + assert trace_mod._in_temporal_activity() is False From d9717ef91e0c81e88ce0b828d4fbafab18e2f86a Mon Sep 17 00:00:00 2001 From: Nitesh Dhanpal Date: Mon, 10 Aug 2026 15:56:26 -0700 Subject: [PATCH 4/9] refactor(tracing): key the dispatch discriminator on TracingActivityName Compare activity.info().activity_type against TracingActivityName.START_SPAN / END_SPAN instead of the string literals "start-span" / "end-span", so the check can't silently drift from the enum that actually names the activities (@activity.defn(name=TracingActivityName.START_SPAN)). Uses a lazy import inside the function to avoid the activities -> TracingService -> AsyncTracer -> trace import cycle (the reason literals were used originally); at call time, inside an activity, the module graph is fully loaded so the import is safe. activity_type round-trips as the enum's str value, which a str-Enum member compares equal to. Co-Authored-By: Claude Opus 4.8 --- src/agentex/lib/core/tracing/trace.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/agentex/lib/core/tracing/trace.py b/src/agentex/lib/core/tracing/trace.py index 6decc6f70..0a1e52182 100644 --- a/src/agentex/lib/core/tracing/trace.py +++ b/src/agentex/lib/core/tracing/trace.py @@ -116,15 +116,29 @@ def _in_tracing_dispatch_activity() -> bool: created directly inside a *business* activity (an agent turn's own ``adk.tracing.span``) runs start AND end in the same activity process, so a wrapper there is safe -- it nests under the interceptor's ambient RunActivity - span and closes in-process. The tracing dispatch activities are named - ``start-span`` / ``end-span`` (``TracingActivityName``). Never raises; False + span and closes in-process. The tracing dispatch activities are named by + ``TracingActivityName`` (``start-span`` / ``end-span``). Never raises; False when temporalio isn't importable or we're not in an activity.""" try: from temporalio import activity + # Lazy import: TracingActivityName lives in the activities package whose + # module graph imports back into this one (activities -> TracingService -> + # AsyncTracer -> trace), so a top-level import would be circular. At call + # time (inside an activity) that graph is fully loaded, so this is safe -- + # and it keeps the discriminator keyed on the enum, not on drifting string + # literals. ``activity_type`` round-trips as the enum's str value, which a + # str-Enum member compares equal to. + from agentex.lib.core.temporal.activities.adk.tracing_activities import ( + TracingActivityName, + ) + if not activity.in_activity(): return False - return activity.info().activity_type in ("start-span", "end-span") + return activity.info().activity_type in ( + TracingActivityName.START_SPAN, + TracingActivityName.END_SPAN, + ) except Exception: return False From 35f127bafceb145765ef5a8f429179bf29ff2017 Mon Sep 17 00:00:00 2001 From: Nitesh Dhanpal Date: Mon, 10 Aug 2026 15:59:10 -0700 Subject: [PATCH 5/9] docs(tracing): re-home the follow-up TODOs orphaned by removing _in_temporal_activity Deleting _in_temporal_activity earlier dropped the TODO it carried. This PR does item (a) of it (the named per-step wrapper); the other two are still open, so re-home them on _begin_obs where the per-step wrapper decision now lives: (1) TurnTrace RETRY/ASYNC roll-up -- retried turns currently surface as N per-attempt span sets, not one PRIMARY + N RETRY view. (2) multi-replica bounded-_OBS_HANDLES + obs_trace_id-resolves-to-turn check. Co-Authored-By: Claude Opus 4.8 --- src/agentex/lib/core/tracing/trace.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/agentex/lib/core/tracing/trace.py b/src/agentex/lib/core/tracing/trace.py index 0a1e52182..717f16b55 100644 --- a/src/agentex/lib/core/tracing/trace.py +++ b/src/agentex/lib/core/tracing/trace.py @@ -195,6 +195,15 @@ def _begin_obs( if _in_tracing_dispatch_activity(): tag_ambient_obs_span(business_span_id=span_id, business_trace_id=trace_id, prefer_otel=True) return None, obs_correlation(prefer_otel=True) + # TODO(obs-followup): two items formerly tracked on the (now-deleted) + # _in_temporal_activity docstring, still open after this change: + # (1) TurnTrace RETRY/ASYNC roll-up. A retried business activity now emits a + # full per-step wrapper set PER ATTEMPT, each nested under that attempt's + # RunActivity. Each attempt correlates to the turn on its own, but they + # are not yet rolled up, so a retried turn surfaces as N per-attempt span + # sets rather than one PRIMARY + N RETRY view. + # (2) On a multi-replica worker fleet, assert _OBS_HANDLES stays bounded (no + # leak / OOM) and that obs_trace_id resolves to the turn trace. prefer_otel = _in_temporal_activity() handle = open_obs_span( name, business_span_id=span_id, business_trace_id=trace_id, prefer_otel=prefer_otel From 4a8b7e303527cfb61a82cfafdaa28dfd09bb9452 Mon Sep 17 00:00:00 2001 From: Nitesh Dhanpal Date: Mon, 10 Aug 2026 16:15:22 -0700 Subject: [PATCH 6/9] refactor(tracing): rename prefer_otel to expect_otel; warn on backend drift prefer_otel read like a per-call override of SGP_OBS_MODE, which undercut the mode as the source of truth. Rename to expect_otel: it is not a bypass but a context-derived expectation -- "the expected backend here is OTel" -- true on the Temporal path, where the interceptor span is OTel regardless of mode. Keep the mode authoritative and make config-vs-reality drift observable instead of silently absorbing it: warn_on_backend_drift() logs once (deduped per direction) when the expected backend has no active span but the other one does (e.g. dd_only configured but the live span is OTel). Fail-open -- it only warns; the caller still reads and falls back, so no correlation is lost. Wired into _begin_obs on both the dispatch and business-activity paths. Tests: the rename, plus two drift tests (warns once on mismatch; silent when the expected backend is the live one). Co-Authored-By: Claude Opus 4.8 --- src/agentex/lib/core/tracing/obs_ids.py | 56 ++++++++++++++++++++++-- src/agentex/lib/core/tracing/obs_span.py | 18 ++++---- src/agentex/lib/core/tracing/trace.py | 18 ++++---- tests/test_temporal_obs_backend.py | 53 +++++++++++++++++++--- 4 files changed, 117 insertions(+), 28 deletions(-) diff --git a/src/agentex/lib/core/tracing/obs_ids.py b/src/agentex/lib/core/tracing/obs_ids.py index 45fada783..474e8d84f 100644 --- a/src/agentex/lib/core/tracing/obs_ids.py +++ b/src/agentex/lib/core/tracing/obs_ids.py @@ -28,15 +28,22 @@ from __future__ import annotations import os +import logging from typing import Dict, Tuple, Optional -__all__ = ("get_obs_mode", "obs_correlation") +__all__ = ("get_obs_mode", "obs_correlation", "warn_on_backend_drift") DD_ONLY = "dd_only" LGTM = "lgtm" _DEFAULT_MODE = DD_ONLY _VALID_MODES = (DD_ONLY, LGTM) +_log = logging.getLogger(__name__) +# Deduped (expected, actual) drift directions already warned about, so a genuine +# mismatch logs once instead of once per span. Bounded by construction: at most +# the 2 direction pairs ("otel"/"ddtrace" either way). +_WARNED_DRIFT: set[Tuple[str, str]] = set() + def get_obs_mode() -> str: """Unset/empty/unrecognized -> ``dd_only`` (current behavior).""" @@ -69,7 +76,7 @@ def _ddtrace_ids() -> Optional[Tuple[str, str]]: return None -def obs_correlation(prefer_otel: bool = False) -> Dict[str, str]: +def obs_correlation(expect_otel: bool = False) -> Dict[str, str]: """Return ``{"obs_trace_id": ..., "obs_span_id": ...}`` for the active observability context, or ``{}`` if none is active. @@ -79,7 +86,7 @@ def obs_correlation(prefer_otel: bool = False) -> Dict[str, str]: dotted) keep them addressable via Postgres JSON paths (``operation_metadata->>'obs_trace_id'``). - ``prefer_otel``: on the Temporal path the active span is the temporalio OTel + ``expect_otel``: on the Temporal path the active span is the temporalio OTel ``TracingInterceptor`` span regardless of ``SGP_OBS_MODE``, so callers there read OTel first (falling back to ddtrace) -- otherwise the default ``dd_only`` mode would read ids for an unrelated ddtrace trace, not the activity span. @@ -87,7 +94,7 @@ def obs_correlation(prefer_otel: bool = False) -> Dict[str, str]: Never fabricates ids -- this is a correlation tag, not the span's id. """ try: - if prefer_otel: + if expect_otel: ids = _lgtm_ids() or _ddtrace_ids() else: ids = _lgtm_ids() if get_obs_mode() == LGTM else _ddtrace_ids() @@ -97,3 +104,44 @@ def obs_correlation(prefer_otel: bool = False) -> Dict[str, str]: if not ids: return {} return {"obs_trace_id": ids[0], "obs_span_id": ids[1]} + + +def warn_on_backend_drift(expect_otel: bool = False) -> None: + """Log once when the EXPECTED obs backend has no active span but the OTHER one + does. + + Expected backend = OTel when ``expect_otel`` (the Temporal path, where the + interceptor span is OTel regardless of ``SGP_OBS_MODE``), otherwise the backend + the mode implies. A mismatch means the mode does not match the tracer actually + running at this call site -- e.g. ``dd_only`` configured but the live span is + OTel -- which is a real config/instrumentation drift worth surfacing rather + than silently correlating against whatever happens to be live. + + Not a hard failure: obs stays fail-open (the caller still reads and falls back, + so no correlation is lost). The warning is deduped per direction, so a standing + mismatch logs once, not once per span. Never raises.""" + try: + otel = _lgtm_ids() + ddt = _ddtrace_ids() + if expect_otel or get_obs_mode() == LGTM: + expected, expected_live = "otel", otel + other_live = ddt + else: + expected, expected_live = "ddtrace", ddt + other_live = otel + if expected_live is None and other_live is not None: + actual = "ddtrace" if expected == "otel" else "otel" + if (expected, actual) not in _WARNED_DRIFT: + _WARNED_DRIFT.add((expected, actual)) + _log.warning( + "obs backend drift: expected %s here (SGP_OBS_MODE=%s%s) but the " + "active span is %s; correlating against %s. Check SGP_OBS_MODE and " + "the running instrumentation.", + expected, + get_obs_mode(), + ", temporal path" if expect_otel else "", + actual, + actual, + ) + except Exception: # obs must never fail an app call + pass diff --git a/src/agentex/lib/core/tracing/obs_span.py b/src/agentex/lib/core/tracing/obs_span.py index a8ecf38b1..9e0a24c4b 100644 --- a/src/agentex/lib/core/tracing/obs_span.py +++ b/src/agentex/lib/core/tracing/obs_span.py @@ -175,7 +175,7 @@ def open_obs_span( name: str, business_span_id: Optional[str] = None, business_trace_id: Optional[str] = None, - prefer_otel: bool = False, + expect_otel: bool = False, ) -> Optional[ObsSpanHandle]: """Open an obs span named ``name`` in the active backend, make it the active span, and return a handle carrying its ``{"obs_trace_id","obs_span_id"}``. @@ -184,7 +184,7 @@ def open_obs_span( the reverse tag (``agentex.business_span_id`` / ``agentex.business_trace_id``) so you can pivot obs -> business by searching them in Tempo/DD. - ``prefer_otel``: open an OTel wrapper first, regardless of ``SGP_OBS_MODE``. + ``expect_otel``: open an OTel wrapper first, regardless of ``SGP_OBS_MODE``. Set on the Temporal path, where the ambient span is the temporalio OTel ``TracingInterceptor`` span regardless of mode -- an OTel wrapper nests under it and yields valid ids, whereas the default ``dd_only`` path would open a @@ -201,11 +201,11 @@ def open_obs_span( never fail an app call. """ try: - if prefer_otel or get_obs_mode() == LGTM: + if expect_otel or get_obs_mode() == LGTM: handle = _open_otel_span(name, business_span_id, business_trace_id) - if handle is not None or not prefer_otel: + if handle is not None or not expect_otel: # In lgtm mode a None handle means "no OTel span -> caller uses the - # ambient fallback". Only when prefer_otel is set (Temporal path) + # ambient fallback". Only when expect_otel is set (Temporal path) # do we try ddtrace as a second choice. return handle return _open_ddtrace_span(name, business_span_id, business_trace_id) @@ -251,7 +251,7 @@ def _tag_ddtrace_ambient(business_span_id: Optional[str], business_trace_id: Opt def tag_ambient_obs_span( business_span_id: Optional[str] = None, business_trace_id: Optional[str] = None, - prefer_otel: bool = False, + expect_otel: bool = False, ) -> None: """Stamp the reverse tag onto the CURRENTLY ACTIVE obs span -- without opening a new one. @@ -264,13 +264,13 @@ def tag_ambient_obs_span( ``agentex.business_span_id`` / ``agentex.business_trace_id`` so the obs -> business pivot still works. Best-effort; never raises. - ``prefer_otel``: on the Temporal path the ambient span is the temporalio OTel + ``expect_otel``: on the Temporal path the ambient span is the temporalio OTel ``TracingInterceptor`` span REGARDLESS of ``SGP_OBS_MODE`` -- so callers there - pass ``prefer_otel=True`` to tag OTel first (falling back to ddtrace only if + pass ``expect_otel=True`` to tag OTel first (falling back to ddtrace only if no valid OTel span is active). Without this, the default ``dd_only`` mode would tag an unrelated ddtrace span (or nothing) instead of the real activity span.""" try: - if prefer_otel: + if expect_otel: if _tag_otel_ambient(business_span_id, business_trace_id): return _tag_ddtrace_ambient(business_span_id, business_trace_id) diff --git a/src/agentex/lib/core/tracing/trace.py b/src/agentex/lib/core/tracing/trace.py index 717f16b55..681b378d8 100644 --- a/src/agentex/lib/core/tracing/trace.py +++ b/src/agentex/lib/core/tracing/trace.py @@ -12,7 +12,7 @@ from agentex.types.span import Span from agentex.lib.utils.logging import make_logger from agentex.lib.utils.model_utils import recursive_model_dump -from agentex.lib.core.tracing.obs_ids import obs_correlation +from agentex.lib.core.tracing.obs_ids import obs_correlation, warn_on_backend_drift from agentex.lib.core.tracing.obs_span import ( ObsSpanHandle, open_obs_span, @@ -181,11 +181,11 @@ def _begin_obs( (a workflow calling ``adk.tracing`` -- see ``_in_tracing_dispatch_activity``): there start and end are separate activities on possibly different workers, so a wrapper could never be closed. We fall back to tagging the ambient - interceptor span instead, with ``prefer_otel=True`` (the interceptor span is + interceptor span instead, with ``expect_otel=True`` (the interceptor span is OTel regardless of ``SGP_OBS_MODE``, so a plain ``dd_only`` read would otherwise point at an unrelated ddtrace span). - Inside ANY activity we also pass ``prefer_otel`` to the wrapper and the ambient + Inside ANY activity we also pass ``expect_otel`` to the wrapper and the ambient fallback: the ambient span is the interceptor's OTel span regardless of mode, so a per-step OTel wrapper nests under it and yields valid ids, whereas the default ``dd_only`` path would open a ddtrace wrapper -- which finds no request @@ -193,8 +193,9 @@ def _begin_obs( obs_* ids. """ if _in_tracing_dispatch_activity(): - tag_ambient_obs_span(business_span_id=span_id, business_trace_id=trace_id, prefer_otel=True) - return None, obs_correlation(prefer_otel=True) + warn_on_backend_drift(expect_otel=True) + tag_ambient_obs_span(business_span_id=span_id, business_trace_id=trace_id, expect_otel=True) + return None, obs_correlation(expect_otel=True) # TODO(obs-followup): two items formerly tracked on the (now-deleted) # _in_temporal_activity docstring, still open after this change: # (1) TurnTrace RETRY/ASYNC roll-up. A retried business activity now emits a @@ -204,11 +205,12 @@ def _begin_obs( # sets rather than one PRIMARY + N RETRY view. # (2) On a multi-replica worker fleet, assert _OBS_HANDLES stays bounded (no # leak / OOM) and that obs_trace_id resolves to the turn trace. - prefer_otel = _in_temporal_activity() + expect_otel = _in_temporal_activity() + warn_on_backend_drift(expect_otel) handle = open_obs_span( - name, business_span_id=span_id, business_trace_id=trace_id, prefer_otel=prefer_otel + name, business_span_id=span_id, business_trace_id=trace_id, expect_otel=expect_otel ) - correlation = handle.correlation if handle is not None else obs_correlation(prefer_otel=prefer_otel) + correlation = handle.correlation if handle is not None else obs_correlation(expect_otel=expect_otel) return handle, correlation diff --git a/tests/test_temporal_obs_backend.py b/tests/test_temporal_obs_backend.py index 8dc325994..892199da4 100644 --- a/tests/test_temporal_obs_backend.py +++ b/tests/test_temporal_obs_backend.py @@ -92,20 +92,20 @@ def test_temporal_path_tags_and_reads_otel_in_dd_only(monkeypatch: pytest.Monkey assert span.id not in _OBS_HANDLES -def test_obs_correlation_prefer_otel_prefers_otel_over_ddtrace(monkeypatch: pytest.MonkeyPatch) -> None: +def test_obs_correlation_expect_otel_prefers_otel_over_ddtrace(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("SGP_OBS_MODE", "dd_only") _activate_otel_span(monkeypatch) # Make ddtrace resolve to DIFFERENT ids so we can prove which backend won. monkeypatch.setattr(obs_ids_mod, "_ddtrace_ids", lambda: ("d" * 32, "e" * 16)) - # prefer_otel (Temporal path): OTel wins even though mode is dd_only. - assert obs_correlation(prefer_otel=True) == {"obs_trace_id": _TRACE_HEX, "obs_span_id": _SPAN_HEX} + # expect_otel (Temporal path): OTel wins even though mode is dd_only. + assert obs_correlation(expect_otel=True) == {"obs_trace_id": _TRACE_HEX, "obs_span_id": _SPAN_HEX} # Default (in-process path): still honors mode -> ddtrace. assert obs_correlation() == {"obs_trace_id": "d" * 32, "obs_span_id": "e" * 16} -def test_tag_ambient_prefer_otel_falls_back_to_ddtrace(monkeypatch: pytest.MonkeyPatch) -> None: - """When no valid OTel span is active, prefer_otel falls back to ddtrace.""" +def test_tag_ambient_expect_otel_falls_back_to_ddtrace(monkeypatch: pytest.MonkeyPatch) -> None: + """When no valid OTel span is active, expect_otel falls back to ddtrace.""" monkeypatch.setenv("SGP_OBS_MODE", "dd_only") # No valid OTel span active. @@ -130,7 +130,7 @@ def current_span(self) -> _FakeDDSpan: ddtrace_trace.tracer = _FakeDDTracer() # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, "ddtrace.trace", ddtrace_trace) - tag_ambient_obs_span(business_span_id="bs", business_trace_id="bt", prefer_otel=True) + tag_ambient_obs_span(business_span_id="bs", business_trace_id="bt", expect_otel=True) # OTel was invalid -> fell back to ddtrace, which got the reverse tag. assert tagged["agentex.business_span_id"] == "bs" @@ -166,7 +166,7 @@ def test_begin_obs_tags_ambient_inside_dispatch_activity(monkeypatch: pytest.Mon monkeypatch.setattr(trace_mod, "obs_correlation", lambda **k: {"obs_trace_id": "amb", "obs_span_id": "amb"}) handle, corr = trace_mod._begin_obs("mortgage.advisor.turn", "bs", "bt") assert handle is None - assert tagged.get("business_span_id") == "bs" and tagged.get("prefer_otel") is True + assert tagged.get("business_span_id") == "bs" and tagged.get("expect_otel") is True assert corr == {"obs_trace_id": "amb", "obs_span_id": "amb"} @@ -229,3 +229,42 @@ def test_in_temporal_activity_tracks_in_activity(monkeypatch: pytest.MonkeyPatch assert trace_mod._in_temporal_activity() is True monkeypatch.setattr(temporal_activity, "in_activity", lambda: False) assert trace_mod._in_temporal_activity() is False + + +# --------------------------------------------------------------------------- # +# Backend-drift warning: the mode stays authoritative, and a mode-vs-live +# mismatch is surfaced once (not silently absorbed by the fallback). +# --------------------------------------------------------------------------- # +def test_warn_on_backend_drift_logs_once_on_mismatch(monkeypatch: pytest.MonkeyPatch, caplog) -> None: + """dd_only configured but the live span is OTel (config doesn't match the + running tracer) -> warn, and only once even across repeated spans.""" + import logging + + monkeypatch.setenv("SGP_OBS_MODE", "dd_only") + obs_ids_mod._WARNED_DRIFT.clear() + _activate_otel_span(monkeypatch) # OTel span live + monkeypatch.setattr(obs_ids_mod, "_ddtrace_ids", lambda: None) # ddtrace absent + + with caplog.at_level(logging.WARNING, logger="agentex.lib.core.tracing.obs_ids"): + obs_ids_mod.warn_on_backend_drift(expect_otel=False) + obs_ids_mod.warn_on_backend_drift(expect_otel=False) # deduped + + drift = [r for r in caplog.records if "backend drift" in r.getMessage()] + assert len(drift) == 1 + + +def test_warn_on_backend_drift_silent_when_expected_backend_is_live( + monkeypatch: pytest.MonkeyPatch, caplog +) -> None: + """Temporal path expects OTel and OTel IS the live span -> the by-design case, + no warning.""" + import logging + + monkeypatch.setenv("SGP_OBS_MODE", "dd_only") + obs_ids_mod._WARNED_DRIFT.clear() + _activate_otel_span(monkeypatch) # OTel live == what expect_otel expects + + with caplog.at_level(logging.WARNING, logger="agentex.lib.core.tracing.obs_ids"): + obs_ids_mod.warn_on_backend_drift(expect_otel=True) + + assert [r for r in caplog.records if "backend drift" in r.getMessage()] == [] From 84558a98602ce9e5a391cda3c5132ee1fce0d177 Mon Sep 17 00:00:00 2001 From: Nitesh Dhanpal Date: Mon, 10 Aug 2026 16:25:38 -0700 Subject: [PATCH 7/9] fix(tracing): import TracingActivityName only after the in_activity() guard The enum import sat above the in_activity() check, so it ran on every start_span -- including the pure-sync ACP path that never touches Temporal. That pulled the whole temporal activities module graph into workflow-less processes, made the docstring's "inside an activity" safety justification untrue for the path actually taken, and meant a broken import would silently return False (disabling dispatch discrimination) even on the sync path. Move it below the guard: sync never runs it, and inside an activity the graph is loaded so the lazy import stays safe. Co-Authored-By: Claude Opus 4.8 --- src/agentex/lib/core/tracing/trace.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/agentex/lib/core/tracing/trace.py b/src/agentex/lib/core/tracing/trace.py index 681b378d8..8f5260913 100644 --- a/src/agentex/lib/core/tracing/trace.py +++ b/src/agentex/lib/core/tracing/trace.py @@ -122,19 +122,21 @@ def _in_tracing_dispatch_activity() -> bool: try: from temporalio import activity - # Lazy import: TracingActivityName lives in the activities package whose - # module graph imports back into this one (activities -> TracingService -> - # AsyncTracer -> trace), so a top-level import would be circular. At call - # time (inside an activity) that graph is fully loaded, so this is safe -- - # and it keeps the discriminator keyed on the enum, not on drifting string - # literals. ``activity_type`` round-trips as the enum's str value, which a - # str-Enum member compares equal to. + if not activity.in_activity(): + return False + # Import only AFTER the in_activity() guard: the pure-sync ACP path never + # runs this, so it doesn't pull the temporal activities module graph + # (activities -> TracingService -> AsyncTracer -> trace, also circular at + # import time) into a process that never runs a workflow, and a broken + # import can't silently disable the guard on that path. Inside an activity + # the graph is fully loaded, so the lazy import is safe -- and it keeps the + # discriminator keyed on the enum, not on drifting string literals. + # ``activity_type`` round-trips as the enum's str value, which a str-Enum + # member compares equal to. from agentex.lib.core.temporal.activities.adk.tracing_activities import ( TracingActivityName, ) - if not activity.in_activity(): - return False return activity.info().activity_type in ( TracingActivityName.START_SPAN, TracingActivityName.END_SPAN, From 35b7701cfb7044b43c0654123925aa8100609173 Mon Sep 17 00:00:00 2001 From: Nitesh Dhanpal Date: Mon, 10 Aug 2026 16:29:40 -0700 Subject: [PATCH 8/9] style(tracing): sort test imports (ruff I001) Co-Authored-By: Claude Opus 4.8 --- tests/test_temporal_obs_backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_temporal_obs_backend.py b/tests/test_temporal_obs_backend.py index 892199da4..f69fde451 100644 --- a/tests/test_temporal_obs_backend.py +++ b/tests/test_temporal_obs_backend.py @@ -15,9 +15,9 @@ from typing import Any, cast import pytest +from temporalio import activity as temporal_activity from opentelemetry import trace as otel_trace from opentelemetry.trace import TraceFlags, SpanContext -from temporalio import activity as temporal_activity import agentex.lib.core.tracing.trace as trace_mod import agentex.lib.core.tracing.obs_ids as obs_ids_mod From 549d72642fae5250b8e240248d6f19e8dd07f14c Mon Sep 17 00:00:00 2001 From: Nitesh Dhanpal Date: Mon, 10 Aug 2026 22:14:29 -0700 Subject: [PATCH 9/9] refactor(tracing): probe expected backend first in warn_on_backend_drift; rewrap docstring - warn_on_backend_drift probes the expected backend first and returns early when it is live, so the healthy common path skips the second probe -- and avoids re-attempting the import of a backend that isn't installed (failed imports aren't cached in sys.modules, so the finder cost otherwise recurs every span). Behavior is unchanged. - Rewrap the tag_ambient_obs_span docstring paragraph a prior edit left with one overlong line. Addresses review nits. Co-Authored-By: Claude Opus 4.8 --- src/agentex/lib/core/tracing/obs_ids.py | 42 ++++++++++++------------ src/agentex/lib/core/tracing/obs_span.py | 5 +-- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/src/agentex/lib/core/tracing/obs_ids.py b/src/agentex/lib/core/tracing/obs_ids.py index 474e8d84f..3189b6df8 100644 --- a/src/agentex/lib/core/tracing/obs_ids.py +++ b/src/agentex/lib/core/tracing/obs_ids.py @@ -119,29 +119,29 @@ def warn_on_backend_drift(expect_otel: bool = False) -> None: Not a hard failure: obs stays fail-open (the caller still reads and falls back, so no correlation is lost). The warning is deduped per direction, so a standing - mismatch logs once, not once per span. Never raises.""" + mismatch logs once, not once per span. Probes the expected backend first and + returns early when it is live, so the healthy common path never touches the + other backend. Never raises.""" try: - otel = _lgtm_ids() - ddt = _ddtrace_ids() if expect_otel or get_obs_mode() == LGTM: - expected, expected_live = "otel", otel - other_live = ddt + expected, expected_probe, other_probe, actual = "otel", _lgtm_ids, _ddtrace_ids, "ddtrace" else: - expected, expected_live = "ddtrace", ddt - other_live = otel - if expected_live is None and other_live is not None: - actual = "ddtrace" if expected == "otel" else "otel" - if (expected, actual) not in _WARNED_DRIFT: - _WARNED_DRIFT.add((expected, actual)) - _log.warning( - "obs backend drift: expected %s here (SGP_OBS_MODE=%s%s) but the " - "active span is %s; correlating against %s. Check SGP_OBS_MODE and " - "the running instrumentation.", - expected, - get_obs_mode(), - ", temporal path" if expect_otel else "", - actual, - actual, - ) + expected, expected_probe, other_probe, actual = "ddtrace", _ddtrace_ids, _lgtm_ids, "otel" + if expected_probe() is not None: + return # expected backend is live -> healthy; skip the other probe + if other_probe() is None: + return # nothing live at all -> uninstrumented path, not drift + if (expected, actual) not in _WARNED_DRIFT: + _WARNED_DRIFT.add((expected, actual)) + _log.warning( + "obs backend drift: expected %s here (SGP_OBS_MODE=%s%s) but the " + "active span is %s; correlating against %s. Check SGP_OBS_MODE and " + "the running instrumentation.", + expected, + get_obs_mode(), + ", temporal path" if expect_otel else "", + actual, + actual, + ) except Exception: # obs must never fail an app call pass diff --git a/src/agentex/lib/core/tracing/obs_span.py b/src/agentex/lib/core/tracing/obs_span.py index 9e0a24c4b..4c53a600b 100644 --- a/src/agentex/lib/core/tracing/obs_span.py +++ b/src/agentex/lib/core/tracing/obs_span.py @@ -259,8 +259,9 @@ def tag_ambient_obs_span( Used inside the SDK's dispatched start-span/end-span activities (see ``trace._in_tracing_dispatch_activity``): there we must NOT open our own wrapper span, because start_span/end_span run as separate activities on - possibly different workers and the wrapper could never be closed. Instead we lean on the span the Temporal OTel ``TracingInterceptor`` - already made active for this activity and just add + possibly different workers and the wrapper could never be closed. Instead we + lean on the span the Temporal OTel ``TracingInterceptor`` already made active + for this activity and just add ``agentex.business_span_id`` / ``agentex.business_trace_id`` so the obs -> business pivot still works. Best-effort; never raises.