From 80c322a88f2e73dc0a444f1c68478ca7fbf340fc Mon Sep 17 00:00:00 2001 From: Ethan Jackson Date: Wed, 26 Aug 2026 13:26:53 -0400 Subject: [PATCH 1/3] feat(agentic): nest search_web search and verifier generations in Langfuse ADK OpenInference only records the search_web tool span, so the inner googleSearch and leakage-verifier LiteLLM calls were invisible. Wrap them with langfuse_generation so they nest under the agent trace, and skip the cutoff fence when as_of is today or later so live searches are not stripped. Co-authored-by: Cursor --- README.md | 2 +- aieng-forecasting/README.md | 2 +- .../aieng/forecasting/langfuse_tracing.py | 55 ++++ .../aieng/forecasting/methods/README.md | 2 +- .../methods/agentic/agent_factory.py | 267 +++++++++++++----- .../methods/agentic/test_agent_factory.py | 184 ++++++++++++ .../forecasting/test_langfuse_tracing.py | 57 ++++ guides/03-customize-agent-strategy.md | 2 +- .../skills/research-playbook/SKILL.md | 6 +- .../energy_oil_forecasting/README.md | 2 +- .../analyst_agent/agent.py | 10 +- .../skills/research-playbook/SKILL.md | 6 +- .../skills/research-playbook/SKILL.md | 6 +- .../skills/research-playbook/SKILL.md | 6 +- 14 files changed, 516 insertions(+), 91 deletions(-) diff --git a/README.md b/README.md index 94abc257..29ca306a 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Also in this README: [Setup](#setup) · [Core concepts](#core-concepts) · [Repo - **Core library** — `aieng-forecasting` (`aieng.forecasting`): data services, cutoff enforcement, forecasting tasks, prediction payloads, backtesting, evaluation, and artifacts. - **Reusable methods** — `aieng.forecasting.methods`: `Predictor` implementations including naive baselines (continuous, binary, and categorical), Darts numerical predictors, LLM-process predictors (continuous, binary-probability, and categorical-probability), and ADK-based agentic infrastructure (`build_adk_agent`, `AdkTextRunner`, `AgentPredictor`). - **Reference implementations** — `implementations//`: notebooks, helper modules, task-specific configuration, and co-located YAML specs. -- **Tracing** — Langfuse / OpenTelemetry bootstrap (`aieng.forecasting.langfuse_tracing`) for LiteLLM and Google ADK. +- **Tracing** — Langfuse / OpenTelemetry bootstrap (`aieng.forecasting.langfuse_tracing`) for LiteLLM and Google ADK. Agent `search_web` calls nest inner search and leakage-verifier generations in the same trace. - **Data scripts** — `scripts/`: one fetch script per data source, plus `build_e2b_template.py` for the agentic code-execution sandbox. ## Two ways to use a forecaster diff --git a/aieng-forecasting/README.md b/aieng-forecasting/README.md index 037c5735..c990e9e3 100644 --- a/aieng-forecasting/README.md +++ b/aieng-forecasting/README.md @@ -8,7 +8,7 @@ This package provides the stable infrastructure used across all reference implem - Forecasting task and prediction payload models. - Backtesting, evaluation, scoring, and artifact helpers. - Reusable reference predictors under `aieng.forecasting.methods`. -- Langfuse / OpenTelemetry tracing bootstrap in `aieng.forecasting.langfuse_tracing`. +- Langfuse / OpenTelemetry tracing bootstrap in `aieng.forecasting.langfuse_tracing` (including nested `search_web` search/verifier generations). Current data adapters cover StatCan tables, FRED series, and daily yfinance market series. diff --git a/aieng-forecasting/aieng/forecasting/langfuse_tracing.py b/aieng-forecasting/aieng/forecasting/langfuse_tracing.py index 0a94e5db..65245f3a 100644 --- a/aieng-forecasting/aieng/forecasting/langfuse_tracing.py +++ b/aieng-forecasting/aieng/forecasting/langfuse_tracing.py @@ -4,6 +4,10 @@ ``llm`` or ``agentic`` extras and Langfuse credentials are set in the environment. +Inner LiteLLM calls that ADK OpenInference does not see (``search_web``'s +grounded search and leakage verifier) should wrap with +:func:`langfuse_generation` so they nest under the active agent trace. + Call :func:`print_langfuse_trace_url` after a ``predict()`` call to flush pending spans and print a clickable Langfuse UI link. """ @@ -12,11 +16,20 @@ import logging import os +from contextlib import contextmanager +from typing import Any, Iterator logger = logging.getLogger(__name__) +class _NoOpObservation: + """Stand-in when Langfuse is unavailable so callers can always ``.update()``.""" + + def update(self, **_kwargs: Any) -> None: + return None + + def _langfuse_credentials_present() -> bool: pub = os.environ.get("LANGFUSE_PUBLIC_KEY", "").strip() sec = os.environ.get("LANGFUSE_SECRET_KEY", "").strip() @@ -105,6 +118,48 @@ def _instrument_google_adk(self) -> None: _bootstrap = _LangfuseTracingBootstrap() +@contextmanager +def langfuse_generation( + name: str, + *, + model: str | None = None, + input: Any = None, # noqa: A002 — matches Langfuse observation field name + metadata: dict[str, Any] | None = None, +) -> Iterator[Any]: + """Open a Langfuse generation nested under the current observation. + + Used for inner LiteLLM calls that ADK OpenInference does not see (the + ``search_web`` googleSearch completion and the independent leakage + verifier). When an ADK tool span is already active, the new generation + becomes its child, so verifier traces show up inside the agent tree + rather than as a separate root. + + No-op when credentials are absent or the SDK raises, so tool code can + wrap completions without a tracing extra. + """ + if not _langfuse_credentials_present(): + yield _NoOpObservation() + return + try: + from langfuse import get_client # noqa: PLC0415 + + kwargs: dict[str, Any] = {"name": name, "as_type": "generation"} + if model is not None: + kwargs["model"] = model + if input is not None: + kwargs["input"] = input + if metadata: + kwargs["metadata"] = metadata + observation_cm = get_client().start_as_current_observation(**kwargs) + except Exception: + logger.debug("langfuse_generation(%s) failed; continuing without a span.", name, exc_info=True) + yield _NoOpObservation() + return + + with observation_cm as generation: + yield generation + + def init_langfuse_tracing() -> None: """Wire LiteLLM and Google ADK to Langfuse. diff --git a/aieng-forecasting/aieng/forecasting/methods/README.md b/aieng-forecasting/aieng/forecasting/methods/README.md index d591f412..2c851763 100644 --- a/aieng-forecasting/aieng/forecasting/methods/README.md +++ b/aieng-forecasting/aieng/forecasting/methods/README.md @@ -104,7 +104,7 @@ from aieng.forecasting.methods.agentic import ( |---|---|---| | `agentic/adk_runner.py` | `AdkTextRunner` | Async text-in / text-out wrapper around ADK `InMemoryRunner`. Manages ADK sessions (fresh-per-message or sticky) and optionally traces each turn to Langfuse via `propagate_attributes`. | | `agentic/adk_runner.py` | `AdkTextRunnerConfig` | Pydantic configuration for `AdkTextRunner` (session mode, Langfuse fields). | -| `agentic/agent_factory.py` | `build_adk_agent` | Generic ADK `LlmAgent` factory with optional code execution, context retrieval, skills, generation controls, and structured output schema. | +| `agentic/agent_factory.py` | `build_adk_agent` | Generic ADK `LlmAgent` factory with optional code execution, context retrieval, skills, generation controls, and structured output schema. `search_web` runs a cutoff-aware googleSearch sub-call plus an independent leakage verifier on historical origins (skipped when `as_of` is today or later); both inner LLM calls emit nested Langfuse generations (`search_web.google_search`, `search_web.leakage_verifier`) under the agent trace. | | `agentic/agent_factory.py` | `AgentConfig` | Pydantic configuration for reusable ADK agents. `output_schema=None` supports interactive/free-form agents; a structured `AgentForecastOutput` schema supports Track 1 predictors. The `function_tools` field attaches conventional ADK tools (e.g. `ForecastTool`). Use-case-specific prompts and presets should live in `implementations//`. | | `agentic/forecast_tool.py` | `ForecastTool` | Conventional ADK `FunctionTool` that runs a pre-specified `Predictor` (AutoARIMA by default) on any registered series at a given cutoff/horizon, returning a structured JSON forecast. A controlled, reproducible alternative to open-ended code execution; series data never enters the LLM context. | | `agentic/outputs.py` | `AgentForecastOutput` | Abstract output adapter interface for converting structured agent JSON into evaluation `Prediction` objects. | diff --git a/aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py b/aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py index 5866997d..bd10364c 100644 --- a/aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py +++ b/aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py @@ -16,6 +16,7 @@ import logging import os import warnings +from datetime import date, datetime, timezone from pathlib import Path from typing import Any, Callable, Sequence @@ -124,19 +125,27 @@ class ContextRetrievalConfig(BaseModel): the calling agent can retrieve grounded, sourced web context without a direct Gemini API key. - Temporal cutoff enforcement has two layers. The first is soft - (LLM-judgment-based): when ``enforce_cutoff`` is ``True`` and the calling - agent passes a ``cutoff_date`` to the tool, the inner proxy prompt - explicitly asks the model to exclude post-cutoff sources. This alone is - not a hard guarantee — backtests have shown it leak real post-cutoff - information despite the instruction. The second, hard layer is an - independent verifier call (see ``verifier_model`` etc. below): a separate - LLM call extracts and judges each factual claim in the search result - against the cutoff, strips violations, and retries the search with - feedback when it cannot produce a sufficiently confident result — + Temporal cutoff enforcement has two layers, and both run only when the + effective cutoff is a *retrospective* date (strictly before UTC today). + Live origins (cutoff on or after today) skip the fence so current news + is not stripped; set ``enforce_cutoff=False`` to skip it even on + historical dates. The first layer is soft (LLM-judgment-based): the + inner proxy prompt asks the model to exclude post-cutoff sources. This + alone is not a hard guarantee — backtests have shown it leak real + post-cutoff information despite the instruction. The second, hard layer + is an independent verifier call (see ``verifier_model`` etc. below): a + separate LLM call extracts and judges each factual claim in the search + result against the cutoff, strips violations, and retries the search + with feedback when it cannot produce a sufficiently confident result — returning an explicit failure sentinel rather than silently risky content if verification never succeeds within the attempt budget. + Both inner LLM calls (grounded search and verifier) emit Langfuse + generations named ``search_web.google_search`` and + ``search_web.leakage_verifier`` when tracing is configured, nested + under the ADK ``search_web`` tool span so they appear in the same + agent trace. + Attributes ---------- enabled : bool, default=False @@ -150,11 +159,12 @@ class ContextRetrievalConfig(BaseModel): when ``enabled`` is ``True``. enforce_cutoff : bool, default=True When ``True``, the ``search_web`` tool appends a cutoff-date - constraint to the user prompt whenever ``cutoff_date`` is supplied by - the calling agent, and runs the independent leakage verifier - described above. Set to ``False`` for live (non-backtest) agents - where no temporal fence is needed — the verifier is skipped entirely - in that case, at zero extra cost. + constraint and runs the independent leakage verifier whenever the + effective cutoff (harness ``as_of``, else the LLM-supplied + ``cutoff_date``) is strictly before UTC today. Cutoffs on or after + today are treated as live and skip both layers automatically. Set + to ``False`` to skip the fence even on historical dates, at zero + extra verifier cost. temperature : float | None, default=None Sampling temperature for the inner search call. max_output_tokens : int | None, default=None @@ -270,6 +280,56 @@ def _build_leakage_verification_schema() -> dict[str, Any]: } +def _utc_today() -> date: + """Calendar date in UTC; used to decide whether a cutoff is retrospective.""" + return datetime.now(timezone.utc).date() + + +def _is_retrospective_cutoff(cutoff: str) -> bool: + """Return True when *cutoff* is a calendar date strictly before UTC today. + + Live origins (today or a future date) should not run the leakage fence: + there is nothing post-cutoff to leak, and the verifier would strip + current news. Unparseable values are treated as retrospective so a + malformed date cannot silently disable the guard. + """ + raw = cutoff.strip()[:10] + try: + cutoff_d = date.fromisoformat(raw) + except ValueError: + return True + return cutoff_d < _utc_today() + + +def _verification_skip_reason(effective_cutoff: str | None, *, enforce_cutoff: bool) -> str | None: + """Why cutoff enforcement is skipped, or ``None`` when the verifier should run.""" + if not effective_cutoff: + return "no_cutoff" + if not enforce_cutoff: + return "enforce_cutoff_disabled" + if not _is_retrospective_cutoff(effective_cutoff): + return "live_as_of" + return None + + +def _usage_from_litellm(resp: Any) -> dict[str, int]: + """Map a LiteLLM response's token counts into Langfuse ``usage_details``.""" + usage = getattr(resp, "usage", None) + if usage is None: + return {} + try: + in_tok = int(getattr(usage, "prompt_tokens", 0) or 0) + out_tok = int(getattr(usage, "completion_tokens", 0) or 0) + except (TypeError, ValueError): + return {} + details: dict[str, int] = {} + if in_tok: + details["input_tokens"] = in_tok + if out_tok: + details["output_tokens"] = out_tok + return details + + _LEAKAGE_VERIFIER_INSTRUCTION = """\ You are an independent fact-checker verifying that a web search result contains \ no information published on or after a given cutoff date. @@ -297,6 +357,7 @@ async def _verify_no_leakage( verifier_model: str, openai_base_url: str, openai_api_key: str | None, + trace_metadata: dict[str, Any] | None = None, ) -> _LeakageVerification: """Judge a search result for post-cutoff claims via an independent verifier call. @@ -307,39 +368,62 @@ async def _verify_no_leakage( it consumes a retry attempt like any other rejection. """ import litellm # noqa: PLC0415 + from aieng.forecasting.langfuse_tracing import langfuse_generation # noqa: PLC0415 from aieng.forecasting.methods.llm_processes._client import ( # noqa: PLC0415 make_json_schema_response_format, strip_markdown_fence, ) model = verifier_model if verifier_model.startswith("openai/") else f"openai/{verifier_model}" - resp = await litellm.acompletion( - model=model, - api_base=openai_base_url, - api_key=openai_api_key, - messages=[ - {"role": "system", "content": _LEAKAGE_VERIFIER_INSTRUCTION}, - { - "role": "user", - "content": f"Original query: {query}\nCutoff date: {cutoff_date}\n\nSearch result to verify:\n{text}", - }, - ], - response_format=make_json_schema_response_format("LeakageVerification", _build_leakage_verification_schema()), - temperature=0.0, - max_tokens=2048, - timeout=60.0, - ) - raw = resp.choices[0].message.content or "{}" - try: - return _LeakageVerification.model_validate(json.loads(strip_markdown_fence(raw))) - except (json.JSONDecodeError, ValidationError): - logger.warning("Leakage verifier returned unparseable output; treating as non-clean: %r", raw[:200]) - return _LeakageVerification( - flagged_claims=["verifier response could not be parsed"], - filtered_text=text, - confidence=1, - clean=False, + messages = [ + {"role": "system", "content": _LEAKAGE_VERIFIER_INSTRUCTION}, + { + "role": "user", + "content": f"Original query: {query}\nCutoff date: {cutoff_date}\n\nSearch result to verify:\n{text}", + }, + ] + with langfuse_generation( + name="search_web.leakage_verifier", + model=verifier_model, + input={"messages": messages}, + metadata=trace_metadata, + ) as generation: + resp = await litellm.acompletion( + model=model, + api_base=openai_base_url, + api_key=openai_api_key, + messages=messages, + response_format=make_json_schema_response_format( + "LeakageVerification", _build_leakage_verification_schema() + ), + temperature=0.0, + max_tokens=2048, + timeout=60.0, ) + raw = resp.choices[0].message.content or "{}" + try: + verdict = _LeakageVerification.model_validate(json.loads(strip_markdown_fence(raw))) + except (json.JSONDecodeError, ValidationError): + logger.warning("Leakage verifier returned unparseable output; treating as non-clean: %r", raw[:200]) + verdict = _LeakageVerification( + flagged_claims=["verifier response could not be parsed"], + filtered_text=text, + confidence=1, + clean=False, + ) + update: dict[str, Any] = { + "output": { + "clean": verdict.clean, + "confidence": verdict.confidence, + "flagged_claims": verdict.flagged_claims, + "filtered_text": verdict.filtered_text, + }, + } + usage = _usage_from_litellm(resp) + if usage: + update["usage_details"] = usage + generation.update(**update) + return verdict def _build_search_tool( @@ -355,14 +439,15 @@ def _build_search_tool( server-side grounding and returns a synthesised answer plus source URLs extracted from ``choices[0].provider_specific_fields["grounding_metadata"]``. - When a ``cutoff_date`` is supplied and ``config.enforce_cutoff`` is - ``True``, the raw result is passed through an independent leakage - verifier (:func:`_verify_no_leakage`) before being returned. On a flagged - result, the search is retried (up to ``config.verifier_max_attempts`` - times) with the previously flagged claims injected as explicit negative - feedback. If no attempt is verified clean, an explicit - ``[SEARCH_VERIFICATION_FAILED]`` sentinel is returned instead of - potentially-leaky content. + When a retrospective ``cutoff_date`` (or harness ``as_of``) is present + and ``config.enforce_cutoff`` is ``True``, the raw result is passed + through an independent leakage verifier (:func:`_verify_no_leakage`) + before being returned. Cutoffs on or after UTC today skip the fence + (live search). On a flagged result, the search is retried (up to + ``config.verifier_max_attempts`` times) with the previously flagged + claims injected as explicit negative feedback. If no attempt is verified + clean, an explicit ``[SEARCH_VERIFICATION_FAILED]`` sentinel is returned + instead of potentially-leaky content. """ def _format_result(content: str, sources: list[str]) -> str: @@ -370,32 +455,47 @@ def _format_result(content: str, sources: list[str]) -> str: content += "\n\nSources:\n" + "\n".join(sources[:5]) return content - async def _do_search(user_content: str) -> tuple[str, list[str]]: + async def _do_search(user_content: str, *, trace_metadata: dict[str, Any] | None = None) -> tuple[str, list[str]]: import litellm # noqa: PLC0415 + from aieng.forecasting.langfuse_tracing import langfuse_generation # noqa: PLC0415 search_model = config.search_model if not search_model.startswith("openai/"): search_model = f"openai/{search_model}" - resp = await litellm.acompletion( - model=search_model, - api_base=openai_base_url, - api_key=openai_api_key, - messages=[ - {"role": "system", "content": config.instruction}, - {"role": "user", "content": user_content}, - ], - tools=[{"googleSearch": {}}], - max_tokens=config.max_output_tokens or 4096, - temperature=config.temperature or 0.0, - timeout=60.0, - ) - content = resp.choices[0].message.content or "" - psf = getattr(resp.choices[0], "provider_specific_fields", {}) or {} - gm = psf.get("grounding_metadata") or {} - sources: list[str] = [ - uri for c in gm.get("groundingChunks", []) if (uri := (c.get("web") or {}).get("uri")) is not None + messages = [ + {"role": "system", "content": config.instruction}, + {"role": "user", "content": user_content}, ] - return content, sources + with langfuse_generation( + name="search_web.google_search", + model=config.search_model, + input={"messages": messages}, + metadata=trace_metadata, + ) as generation: + resp = await litellm.acompletion( + model=search_model, + api_base=openai_base_url, + api_key=openai_api_key, + messages=messages, + tools=[{"googleSearch": {}}], + max_tokens=config.max_output_tokens or 4096, + temperature=config.temperature or 0.0, + timeout=60.0, + ) + content = resp.choices[0].message.content or "" + psf = getattr(resp.choices[0], "provider_specific_fields", {}) or {} + gm = psf.get("grounding_metadata") or {} + sources: list[str] = [ + uri for c in gm.get("groundingChunks", []) if (uri := (c.get("web") or {}).get("uri")) is not None + ] + update: dict[str, Any] = {"output": content} + usage = _usage_from_litellm(resp) + if usage: + update["usage_details"] = usage + extra_meta = {**(trace_metadata or {}), "source_count": len(sources)} + update["metadata"] = extra_meta + generation.update(**update) + return content, sources async def search_web(query: str, cutoff_date: str | None = None, tool_context: ToolContext | None = None) -> str: """Search the web and return a grounded summary with source URLs. @@ -423,6 +523,11 @@ async def search_web(query: str, cutoff_date: str | None = None, tool_context: T the calling LLM cannot see, omit, or alter it. This closes a bypass where the model simply didn't pass ``cutoff_date`` and both the soft cutoff instruction and the verifier below were silently skipped. + + Cutoff enforcement (soft prompt + verifier) runs only when that + effective date is strictly before UTC today. An ``as_of`` of today + or later is treated as a live origin and searched without a + temporal fence, matching ``enforce_cutoff=False``. """ harness_as_of = tool_context.state.get(AS_OF_STATE_KEY) if tool_context is not None else None if harness_as_of and cutoff_date and harness_as_of != cutoff_date: @@ -432,20 +537,33 @@ async def search_web(query: str, cutoff_date: str | None = None, tool_context: T harness_as_of, ) effective_cutoff = harness_as_of or cutoff_date - - needs_verification = bool(effective_cutoff and config.enforce_cutoff) - if not needs_verification: - content, sources = await _do_search(query) + skip_reason = _verification_skip_reason(effective_cutoff, enforce_cutoff=config.enforce_cutoff) + base_meta: dict[str, Any] = { + "effective_cutoff": effective_cutoff, + "harness_as_of": harness_as_of, + "llm_cutoff_date": cutoff_date, + } + if skip_reason is not None: + logger.info( + "search_web: skipping cutoff enforcement (%s); cutoff=%s", + skip_reason, + effective_cutoff, + ) + content, sources = await _do_search( + query, + trace_metadata={**base_meta, "verification_skipped": skip_reason}, + ) return _format_result(content, sources) negative_guidance = "" for attempt in range(1, config.verifier_max_attempts + 1): + attempt_meta = {**base_meta, "attempt": attempt, "verifier_max_attempts": config.verifier_max_attempts} user_content = ( query + f"\n\nOnly include and cite information published strictly before {effective_cutoff}." ) if negative_guidance: user_content += f"\n\n{negative_guidance}" - content, sources = await _do_search(user_content) + content, sources = await _do_search(user_content, trace_metadata=attempt_meta) verdict = await _verify_no_leakage( text=content, query=query, @@ -453,6 +571,7 @@ async def search_web(query: str, cutoff_date: str | None = None, tool_context: T verifier_model=config.verifier_model, openai_base_url=openai_base_url, openai_api_key=openai_api_key, + trace_metadata=attempt_meta, ) logger.info( "search_web verification attempt %d/%d: clean=%s confidence=%d flagged=%d", diff --git a/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_agent_factory.py b/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_agent_factory.py index c49b098f..5bbb6f31 100644 --- a/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_agent_factory.py +++ b/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_agent_factory.py @@ -3,8 +3,11 @@ import inspect import json import logging +from contextlib import contextmanager +from datetime import date from pathlib import Path from types import SimpleNamespace +from typing import Any, Iterator from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -14,6 +17,8 @@ CodeExecutionConfig, ContextRetrievalConfig, _build_search_tool, + _is_retrospective_cutoff, + _verification_skip_reason, build_adk_agent, ) from aieng.forecasting.methods.agentic.outputs import ContinuousAgentForecastOutput @@ -634,3 +639,182 @@ async def _fake_acompletion(**kwargs): # type: ignore[override] assert len(calls) == 2 assert result == "Clean summary." + + +class TestRetrospectiveCutoff: + """Live origins skip the verifier; historical dates still fence.""" + + def test_yesterday_is_retrospective(self) -> None: + """A cutoff strictly before UTC today is a backtest fence.""" + with patch( + "aieng.forecasting.methods.agentic.agent_factory._utc_today", + return_value=date(2026, 8, 26), + ): + assert _is_retrospective_cutoff("2026-08-25") is True + assert _is_retrospective_cutoff("2024-01-15") is True + + def test_today_and_future_are_live(self) -> None: + """Today or later is a live origin — no leakage fence.""" + with patch( + "aieng.forecasting.methods.agentic.agent_factory._utc_today", + return_value=date(2026, 8, 26), + ): + assert _is_retrospective_cutoff("2026-08-26") is False + assert _is_retrospective_cutoff("2026-08-27") is False + + def test_unparseable_cutoff_is_treated_as_retrospective(self) -> None: + """Malformed dates must not silently disable the guard.""" + assert _is_retrospective_cutoff("not-a-date") is True + + def test_skip_reason_live_as_of(self) -> None: + """enforce_cutoff=True still skips when the cutoff is today.""" + with patch( + "aieng.forecasting.methods.agentic.agent_factory._utc_today", + return_value=date(2026, 8, 26), + ): + assert _verification_skip_reason("2026-08-26", enforce_cutoff=True) == "live_as_of" + assert _verification_skip_reason("2026-08-25", enforce_cutoff=True) is None + + @pytest.mark.asyncio + async def test_verifier_skipped_when_cutoff_is_today(self) -> None: + """A live as_of (today) is a single search call, no verifier.""" + config = ContextRetrievalConfig(enabled=True, instruction="Search assistant.") + tool = _build_search_tool(config, openai_base_url="https://proxy.example.com/v1", openai_api_key="test-key") + calls: list[dict] = [] + + async def _fake_acompletion(**kwargs): # type: ignore[override] + calls.append(kwargs) + resp = MagicMock() + resp.choices[0].message.content = "Live news." + resp.choices[0].provider_specific_fields = {} + resp.usage = None + return resp + + with ( + patch("aieng.forecasting.methods.agentic.agent_factory._utc_today", return_value=date(2026, 8, 26)), + patch("litellm.acompletion", new=AsyncMock(side_effect=_fake_acompletion)), + ): + result = await tool(query="WTI price", cutoff_date="2026-08-26") + + assert len(calls) == 1 + assert result == "Live news." + user_msg = next(m for m in calls[0]["messages"] if m["role"] == "user") + assert "2026-08-26" not in user_msg["content"] + + @pytest.mark.asyncio + async def test_harness_today_skips_even_when_llm_passes_past_cutoff(self) -> None: + """Harness as_of of today wins; verifier does not run.""" + config = ContextRetrievalConfig(enabled=True, instruction="Search assistant.") + tool = _build_search_tool(config, openai_base_url="https://proxy.example.com/v1", openai_api_key="test-key") + fake_tool_context = SimpleNamespace(state={AS_OF_STATE_KEY: "2026-08-26"}) + calls: list[dict] = [] + + async def _fake_acompletion(**kwargs): # type: ignore[override] + calls.append(kwargs) + resp = MagicMock() + resp.choices[0].message.content = "Live news." + resp.choices[0].provider_specific_fields = {} + resp.usage = None + return resp + + with ( + patch("aieng.forecasting.methods.agentic.agent_factory._utc_today", return_value=date(2026, 8, 26)), + patch("litellm.acompletion", new=AsyncMock(side_effect=_fake_acompletion)), + ): + result = await tool( + query="WTI price", + cutoff_date="2024-01-15", + tool_context=fake_tool_context, + ) + + assert len(calls) == 1 + assert result == "Live news." + + +class TestSearchToolLangfuseTracing: + """Inner search and verifier emit nested Langfuse generations.""" + + @staticmethod + def _search_response(content: str) -> MagicMock: + resp = MagicMock() + resp.choices[0].message.content = content + resp.choices[0].provider_specific_fields = {} + resp.usage = None + return resp + + @staticmethod + def _verify_response() -> MagicMock: + payload = { + "flagged_claims": [], + "filtered_text": "Clean summary.", + "confidence": 9, + "clean": True, + } + resp = MagicMock() + resp.choices[0].message.content = json.dumps(payload) + resp.choices[0].provider_specific_fields = {} + resp.usage = None + return resp + + @pytest.mark.asyncio + async def test_search_and_verifier_generations_nested_on_historical_cutoff(self) -> None: + """A past cutoff records google_search then leakage_verifier generations.""" + config = ContextRetrievalConfig(enabled=True, instruction="Search assistant.") + tool = _build_search_tool(config, openai_base_url="https://proxy.example.com/v1", openai_api_key="test-key") + gen_names: list[str] = [] + gen_metadata: list[dict[str, Any]] = [] + + @contextmanager + def _fake_generation( + name: str, + *, + model: str | None = None, + input: Any = None, # noqa: A002 + metadata: dict[str, Any] | None = None, + ) -> Iterator[MagicMock]: + gen_names.append(name) + gen_metadata.append(dict(metadata or {})) + yield MagicMock() + + async def _fake_acompletion(**kwargs): # type: ignore[override] + if kwargs["model"] == f"openai/{config.verifier_model}": + return self._verify_response() + return self._search_response("Raw summary.") + + with ( + patch("aieng.forecasting.langfuse_tracing.langfuse_generation", _fake_generation), + patch("litellm.acompletion", new=AsyncMock(side_effect=_fake_acompletion)), + ): + result = await tool(query="WTI price", cutoff_date="2024-01-15") + + assert result == "Clean summary." + assert gen_names == ["search_web.google_search", "search_web.leakage_verifier"] + assert gen_metadata[0]["effective_cutoff"] == "2024-01-15" + assert gen_metadata[1]["effective_cutoff"] == "2024-01-15" + assert gen_metadata[0]["attempt"] == 1 + assert gen_metadata[1]["attempt"] == 1 + + @pytest.mark.asyncio + async def test_live_origin_records_search_generation_without_verifier(self) -> None: + """Live as_of still traces the search call; verifier span is absent.""" + config = ContextRetrievalConfig(enabled=True, instruction="Search assistant.") + tool = _build_search_tool(config, openai_base_url="https://proxy.example.com/v1", openai_api_key="test-key") + gen_names: list[str] = [] + + @contextmanager + def _fake_generation(name: str, **kwargs: Any) -> Iterator[MagicMock]: + gen_names.append(name) + yield MagicMock() + + async def _fake_acompletion(**kwargs): # type: ignore[override] + return self._search_response("Live news.") + + with ( + patch("aieng.forecasting.methods.agentic.agent_factory._utc_today", return_value=date(2026, 8, 26)), + patch("aieng.forecasting.langfuse_tracing.langfuse_generation", _fake_generation), + patch("litellm.acompletion", new=AsyncMock(side_effect=_fake_acompletion)), + ): + result = await tool(query="WTI price", cutoff_date="2026-08-26") + + assert result == "Live news." + assert gen_names == ["search_web.google_search"] diff --git a/aieng-forecasting/tests/aieng/forecasting/test_langfuse_tracing.py b/aieng-forecasting/tests/aieng/forecasting/test_langfuse_tracing.py index 7d2c2d0b..b4d08d9f 100644 --- a/aieng-forecasting/tests/aieng/forecasting/test_langfuse_tracing.py +++ b/aieng-forecasting/tests/aieng/forecasting/test_langfuse_tracing.py @@ -12,6 +12,7 @@ _langfuse_credentials_present, _LangfuseTracingBootstrap, init_langfuse_tracing, + langfuse_generation, ) @@ -198,3 +199,59 @@ def test_init_langfuse_tracing_is_a_no_op_without_credentials( monkeypatch.delenv("LANGFUSE_SECRET_KEY", raising=False) init_langfuse_tracing() assert not fresh._langfuse_client_initialized + + +# --------------------------------------------------------------------------- +# langfuse_generation — nested generations for inner LiteLLM calls +# --------------------------------------------------------------------------- + + +class TestLangfuseGeneration: + """``langfuse_generation`` no-ops without credentials and nests when present.""" + + def test_no_op_without_credentials(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Missing keys yield an object that accepts ``update`` without raising.""" + monkeypatch.delenv("LANGFUSE_PUBLIC_KEY", raising=False) + monkeypatch.delenv("LANGFUSE_SECRET_KEY", raising=False) + with langfuse_generation(name="search_web.leakage_verifier") as generation: + generation.update(output="ok") + + def test_opens_generation_when_credentials_present(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Credentials plus SDK open a nested generation observation.""" + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-test") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-test") + mock_client = MagicMock() + mock_generation = MagicMock() + mock_client.start_as_current_observation.return_value.__enter__.return_value = mock_generation + mock_client.start_as_current_observation.return_value.__exit__.return_value = False + langfuse_mod = MagicMock() + langfuse_mod.get_client.return_value = mock_client + with ( + patch.dict(sys.modules, {"langfuse": langfuse_mod}), + langfuse_generation( + name="search_web.leakage_verifier", + model="gemini-3.5-flash", + input={"query": "WTI"}, + metadata={"attempt": 1}, + ) as generation, + ): + assert generation is mock_generation + mock_client.start_as_current_observation.assert_called_once() + kwargs = mock_client.start_as_current_observation.call_args.kwargs + assert kwargs["name"] == "search_web.leakage_verifier" + assert kwargs["as_type"] == "generation" + assert kwargs["model"] == "gemini-3.5-flash" + assert kwargs["input"] == {"query": "WTI"} + assert kwargs["metadata"] == {"attempt": 1} + + def test_sdk_failure_is_swallowed(self, monkeypatch: pytest.MonkeyPatch) -> None: + """A raising SDK must not break the wrapped LiteLLM call.""" + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-test") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-test") + langfuse_mod = MagicMock() + langfuse_mod.get_client.side_effect = RuntimeError("connection refused") + with ( + patch.dict(sys.modules, {"langfuse": langfuse_mod}), + langfuse_generation(name="search_web.google_search") as generation, + ): + generation.update(output="still works") diff --git a/guides/03-customize-agent-strategy.md b/guides/03-customize-agent-strategy.md index a1bd2926..982d095d 100644 --- a/guides/03-customize-agent-strategy.md +++ b/guides/03-customize-agent-strategy.md @@ -84,7 +84,7 @@ When the agent calls `search_web`, a bounded **sub-agent** — one grounded-sear 1. **What the analyst asks for** — query guidance in the `research-playbook` skill (lever 4) or an instruction supplement. 2. **What the search sub-agent looks for and reports** — `ContextRetrievalConfig.instruction`. *This is the big one*: the analyst never sees raw search results, only this sub-agent's brief. Its instruction currently says "cover price level and trend, OPEC+ supply, geopolitical risk, SPR/policy, analyst targets." Change the brief, change what your agent knows. -3. **Enforcement machinery** — pass `cutoff_date` and an independent verifier model checks the brief for post-cutoff leakage, rewrites or rejects it (returning a `[SEARCH_VERIFICATION_FAILED]` sentinel after 3 failed attempts). The harness overrides the agent-supplied cutoff with the true origin date, so a backtested agent can't leak by "forgetting" the argument. Knobs (verifier model, attempts, confidence threshold, `enforce_cutoff=False` for live forecasting) live on `ContextRetrievalConfig`. +3. **Enforcement machinery** — when the origin is in the past (strictly before UTC today), pass `cutoff_date` and an independent verifier model checks the brief for post-cutoff leakage, rewrites or rejects it (returning a `[SEARCH_VERIFICATION_FAILED]` sentinel after 3 failed attempts). The harness overrides the agent-supplied cutoff with the true origin date, so a backtested agent can't leak by "forgetting" the argument. Origins on or after today are treated as live and skip the fence automatically (same as `enforce_cutoff=False`). Knobs (verifier model, attempts, confidence threshold, `enforce_cutoff=False` to skip even on historical dates) live on `ContextRetrievalConfig`. In Langfuse, the verifier is a `search_web.leakage_verifier` generation nested under the `search_web` tool span of the agent trace. Worked change — an **inventory-first** search strategy, as a drop-in `ToolSpec` factory (put it next to your notebook or in `tools.py`): diff --git a/implementations/boc_rate_decisions/starter_agent/skills/research-playbook/SKILL.md b/implementations/boc_rate_decisions/starter_agent/skills/research-playbook/SKILL.md index ec9c0ca8..d5d69d9f 100644 --- a/implementations/boc_rate_decisions/starter_agent/skills/research-playbook/SKILL.md +++ b/implementations/boc_rate_decisions/starter_agent/skills/research-playbook/SKILL.md @@ -17,11 +17,13 @@ Always pass `cutoff_date` equal to the `as_of` date in your payload. It is the temporal fence that keeps post-origin information out of a historical forecast. A forecast that "knew" what happened after `as_of` is not a forecast. -`search_web` runs an independent verifier on every result and returns +On historical origins (strictly before today's date), `search_web` runs an +independent verifier on every result and returns `[SEARCH_VERIFICATION_FAILED]` instead of content it couldn't confirm as pre-cutoff. Treat that as no verified news for the query — proceed on your other signals and say so, never filling the gap from your own background -knowledge. +knowledge. On a live origin (`as_of` is today), the verifier is skipped so +current news is not stripped. ## How to search diff --git a/implementations/energy_oil_forecasting/README.md b/implementations/energy_oil_forecasting/README.md index 0a9eddae..86887847 100644 --- a/implementations/energy_oil_forecasting/README.md +++ b/implementations/energy_oil_forecasting/README.md @@ -55,7 +55,7 @@ An earlier set of information-session notebooks is archived in [`playground/ener ## The Forecasting Tasks -Each forecasting origin defines a strict information cutoff (`as_of`). Predictors receive price history up to `as_of` and answer up to three tasks: +Each forecasting origin defines a strict information cutoff (`as_of`). Predictors receive price history up to `as_of` and answer up to three tasks. News-grounded agents apply the same fence to `search_web` when `as_of` is in the past (an independent verifier, visible as `search_web.leakage_verifier` under the Langfuse agent trace) and skip it for a live origin. ### Task A: Trajectory Forecast (Track 1) diff --git a/implementations/energy_oil_forecasting/analyst_agent/agent.py b/implementations/energy_oil_forecasting/analyst_agent/agent.py index 043064b0..ca9e302a 100644 --- a/implementations/energy_oil_forecasting/analyst_agent/agent.py +++ b/implementations/energy_oil_forecasting/analyst_agent/agent.py @@ -444,10 +444,12 @@ def build_wti_news_config( """Build an :class:`AgentConfig` with bounded Google Search. Wires a :class:`~aieng.forecasting.methods.agentic.agent_factory.ContextRetrievalConfig` - sub-agent that enforces a temporal cutoff on every search call, preventing - future information from contaminating historical backtests. An - independent verifier call audits each search result against the cutoff - before it reaches the analyst (see :class:`ContextRetrievalConfig`). + sub-agent that enforces a temporal cutoff on retrospective search calls + (``as_of`` strictly before UTC today), preventing future information + from contaminating historical backtests. Live origins skip the fence. + An independent verifier call audits each historical search result + against the cutoff before it reaches the analyst (see + :class:`ContextRetrievalConfig`). Parameters ---------- diff --git a/implementations/energy_oil_forecasting/starter_agent/skills/research-playbook/SKILL.md b/implementations/energy_oil_forecasting/starter_agent/skills/research-playbook/SKILL.md index 2e15bf54..fdb55ad8 100644 --- a/implementations/energy_oil_forecasting/starter_agent/skills/research-playbook/SKILL.md +++ b/implementations/energy_oil_forecasting/starter_agent/skills/research-playbook/SKILL.md @@ -17,11 +17,13 @@ Always pass `cutoff_date` equal to the `as_of` date in your payload. It is the temporal fence that keeps post-origin information out of a historical forecast. A forecast that "knew" what happened after `as_of` is not a forecast. -`search_web` runs an independent verifier on every result and returns +On historical origins (strictly before today's date), `search_web` runs an +independent verifier on every result and returns `[SEARCH_VERIFICATION_FAILED]` instead of content it couldn't confirm as pre-cutoff. Treat that as no verified news for the query — proceed on your other signals and say so, never filling the gap from your own background -knowledge. +knowledge. On a live origin (`as_of` is today), the verifier is skipped so +current news is not stripped. ## How to search diff --git a/implementations/food_price_forecasting/starter_agent/skills/research-playbook/SKILL.md b/implementations/food_price_forecasting/starter_agent/skills/research-playbook/SKILL.md index ec020b56..00f21f88 100644 --- a/implementations/food_price_forecasting/starter_agent/skills/research-playbook/SKILL.md +++ b/implementations/food_price_forecasting/starter_agent/skills/research-playbook/SKILL.md @@ -17,11 +17,13 @@ Always pass `cutoff_date` equal to the `as_of` date in your payload. It is the temporal fence that keeps post-origin information out of a historical forecast. A forecast that "knew" what happened after `as_of` is not a forecast. -`search_web` runs an independent verifier on every result and returns +On historical origins (strictly before today's date), `search_web` runs an +independent verifier on every result and returns `[SEARCH_VERIFICATION_FAILED]` instead of content it couldn't confirm as pre-cutoff. Treat that as no verified news for the query — proceed on your other signals and say so, never filling the gap from your own background -knowledge. +knowledge. On a live origin (`as_of` is today), the verifier is skipped so +current news is not stripped. ## How to search diff --git a/implementations/sp500_forecasting/starter_agent/skills/research-playbook/SKILL.md b/implementations/sp500_forecasting/starter_agent/skills/research-playbook/SKILL.md index 50215fa7..f455ad78 100644 --- a/implementations/sp500_forecasting/starter_agent/skills/research-playbook/SKILL.md +++ b/implementations/sp500_forecasting/starter_agent/skills/research-playbook/SKILL.md @@ -17,11 +17,13 @@ Always pass `cutoff_date` equal to the `as_of` date in your payload. It is the temporal fence that keeps post-origin information out of a historical forecast. A forecast that "knew" what happened after `as_of` is not a forecast. -`search_web` runs an independent verifier on every result and returns +On historical origins (strictly before today's date), `search_web` runs an +independent verifier on every result and returns `[SEARCH_VERIFICATION_FAILED]` instead of content it couldn't confirm as pre-cutoff. Treat that as no verified news for the query — proceed on your other signals and say so, never filling the gap from your own background -knowledge. +knowledge. On a live origin (`as_of` is today), the verifier is skipped so +current news is not stripped. ## How to search From a6f4a28e63eee1043188e1e2d3d8ba87743dfdfc Mon Sep 17 00:00:00 2001 From: Ethan Jackson Date: Wed, 26 Aug 2026 14:27:26 -0400 Subject: [PATCH 2/3] fix(agentic): price search_web and verifier generations in Langfuse Langfuse infers USD from usage types named input/output. The inner search and leakage-verifier calls were sending input_tokens/output_tokens, so token counts showed up but totalCost stayed null. Co-authored-by: Cursor --- .../methods/agentic/agent_factory.py | 25 ++++++++--- .../methods/agentic/test_agent_factory.py | 44 +++++++++++++++++-- 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py b/aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py index a6e95f68..a5e719bd 100644 --- a/aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py +++ b/aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py @@ -313,7 +313,16 @@ def _verification_skip_reason(effective_cutoff: str | None, *, enforce_cutoff: b def _usage_from_litellm(resp: Any) -> dict[str, int]: - """Map a LiteLLM response's token counts into Langfuse ``usage_details``.""" + """Map LiteLLM token counts into Langfuse ``usage_details``. + + Keys must be ``input`` / ``output`` (optionally ``input_cached_tokens``). + Those are the usage types on Langfuse's model price table. Sending + ``input_tokens`` / ``output_tokens`` still shows counts in the UI but + matches no price, so ``totalCost`` stays null — which is what happened + on ``search_web.google_search`` and ``search_web.leakage_verifier`` + until this mapping was aligned with the LLM-process path and ADK + OpenInference. + """ usage = getattr(resp, "usage", None) if usage is None: return {} @@ -322,11 +331,15 @@ def _usage_from_litellm(resp: Any) -> dict[str, int]: out_tok = int(getattr(usage, "completion_tokens", 0) or 0) except (TypeError, ValueError): return {} - details: dict[str, int] = {} - if in_tok: - details["input_tokens"] = in_tok - if out_tok: - details["output_tokens"] = out_tok + details: dict[str, int] = {"input": in_tok, "output": out_tok} + prompt_details = getattr(usage, "prompt_tokens_details", None) + if prompt_details is not None: + try: + cached = int(getattr(prompt_details, "cached_tokens", 0) or 0) + except (TypeError, ValueError): + cached = 0 + if cached: + details["input_cached_tokens"] = cached return details diff --git a/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_agent_factory.py b/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_agent_factory.py index e73cdc36..38483304 100644 --- a/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_agent_factory.py +++ b/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_agent_factory.py @@ -18,6 +18,7 @@ ContextRetrievalConfig, _build_search_tool, _is_retrospective_cutoff, + _usage_from_litellm, _verification_skip_reason, build_adk_agent, ) @@ -733,6 +734,34 @@ async def _fake_acompletion(**kwargs): # type: ignore[override] assert result == "Live news." +class TestUsageFromLitellm: + """Langfuse prices ``input`` / ``output`` usage types, not ``input_tokens``.""" + + def test_maps_prompt_and_completion_tokens(self) -> None: + """LiteLLM prompt/completion counts become Langfuse input/output keys.""" + resp = SimpleNamespace(usage=SimpleNamespace(prompt_tokens=284, completion_tokens=522)) + assert _usage_from_litellm(resp) == {"input": 284, "output": 522} + + def test_includes_cached_tokens_when_present(self) -> None: + """Cached prompt tokens map to the Langfuse price-table usage type.""" + resp = SimpleNamespace( + usage=SimpleNamespace( + prompt_tokens=1000, + completion_tokens=10, + prompt_tokens_details=SimpleNamespace(cached_tokens=800), + ) + ) + assert _usage_from_litellm(resp) == { + "input": 1000, + "output": 10, + "input_cached_tokens": 800, + } + + def test_missing_usage_returns_empty(self) -> None: + """A response with no usage object contributes no Langfuse usage_details.""" + assert _usage_from_litellm(SimpleNamespace()) == {} + + class TestSearchToolLangfuseTracing: """Inner search and verifier emit nested Langfuse generations.""" @@ -741,7 +770,7 @@ def _search_response(content: str) -> MagicMock: resp = MagicMock() resp.choices[0].message.content = content resp.choices[0].provider_specific_fields = {} - resp.usage = None + resp.usage = SimpleNamespace(prompt_tokens=284, completion_tokens=522) return resp @staticmethod @@ -755,7 +784,7 @@ def _verify_response() -> MagicMock: resp = MagicMock() resp.choices[0].message.content = json.dumps(payload) resp.choices[0].provider_specific_fields = {} - resp.usage = None + resp.usage = SimpleNamespace(prompt_tokens=676, completion_tokens=475) return resp @pytest.mark.asyncio @@ -765,6 +794,7 @@ async def test_search_and_verifier_generations_nested_on_historical_cutoff(self) tool = _build_search_tool(config, openai_base_url="https://proxy.example.com/v1", openai_api_key="test-key") gen_names: list[str] = [] gen_metadata: list[dict[str, Any]] = [] + gen_updates: list[dict[str, Any]] = [] @contextmanager def _fake_generation( @@ -776,7 +806,13 @@ def _fake_generation( ) -> Iterator[MagicMock]: gen_names.append(name) gen_metadata.append(dict(metadata or {})) - yield MagicMock() + gen = MagicMock() + + def _update(**kwargs: Any) -> None: + gen_updates.append(kwargs) + + gen.update.side_effect = _update + yield gen async def _fake_acompletion(**kwargs): # type: ignore[override] if kwargs["model"] == f"openai/{config.verifier_model}": @@ -795,6 +831,8 @@ async def _fake_acompletion(**kwargs): # type: ignore[override] assert gen_metadata[1]["effective_cutoff"] == "2024-01-15" assert gen_metadata[0]["attempt"] == 1 assert gen_metadata[1]["attempt"] == 1 + assert gen_updates[0]["usage_details"] == {"input": 284, "output": 522} + assert gen_updates[1]["usage_details"] == {"input": 676, "output": 475} @pytest.mark.asyncio async def test_live_origin_records_search_generation_without_verifier(self) -> None: From a56cdc74d9f66d6a529c839f22185c97d6fa76e6 Mon Sep 17 00:00:00 2001 From: Ethan Jackson Date: Wed, 26 Aug 2026 14:29:56 -0400 Subject: [PATCH 3/3] lint --- .../03_one_agent_three_tasks.ipynb | 223 +++++++++--------- .../04_systematic_backtest_eval.ipynb | 2 +- 2 files changed, 112 insertions(+), 113 deletions(-) diff --git a/implementations/energy_oil_forecasting/03_one_agent_three_tasks.ipynb b/implementations/energy_oil_forecasting/03_one_agent_three_tasks.ipynb index fb33f8ec..60d1c5dd 100644 --- a/implementations/energy_oil_forecasting/03_one_agent_three_tasks.ipynb +++ b/implementations/energy_oil_forecasting/03_one_agent_three_tasks.ipynb @@ -76,7 +76,6 @@ " PROPHET_SHOCK_TRAJ_CACHE,\n", " PROPHET_TRAJ_CACHE,\n", " SCENARIO_CACHE,\n", - " SCENARIO_ORIGIN,\n", " SHOCK_ANALYST_CACHE,\n", " SHOCK_HORIZON,\n", " SHOCK_ORIGINS,\n", @@ -612,9 +611,9 @@ "Saved 3 agent trajectory runs.\n", "\n", "Agent trajectory summary:\n", - " 2026-02-02 WTI=$62.14 h5=$65.2 | h10=$64.8 | h21=$64.0\n", - " 2026-02-23 WTI=$66.31 h5=$66.8 | h10=$67.2 | h21=$67.5\n", - " 2026-03-02 WTI=$71.23 h5=$65.4 | h10=$65.8 | h21=$66.3\n" + " 2026-02-02 WTI=$62.14 h5=$65.2 | h10=$64.8 | h21=$63.5\n", + " 2026-02-23 WTI=$66.31 h5=$66.8 | h10=$67.3 | h21=$68.1\n", + " 2026-03-02 WTI=$71.23 h5=$65.5 | h10=$66.2 | h21=$67.0\n" ] } ], @@ -666,11 +665,11 @@ "\n", "| Horizon | Agent ($) | 80% CI | Actual ($) | Agent err | Prophet err |\n", "|---|---|---|---|---|---|\n", - "| 5 bdays | **$65.4** | [64.1 – 67.2] | $94.8 | -29.4 | -30.2 |\n", - "| 10 bdays | **$65.8** | [63.5 – 68.5] | $93.5 | -27.7 | -29.2 |\n", - "| 21 bdays | **$66.3** | [62.5 – 70.8] | $101.4 | -35.1 | -36.9 |\n", + "| 5 bdays | **$65.5** | [64.2 – 67.5] | $94.8 | -29.3 | -30.2 |\n", + "| 10 bdays | **$66.2** | [64.0 – 69.5] | $93.5 | -27.3 | -29.2 |\n", + "| 21 bdays | **$67.0** | [63.2 – 72.5] | $101.4 | -34.4 | -36.9 |\n", "\n", - "> **Agent rationale:** The forecasts assume that WTI remains supported above the $64.00 range. Short-term momentum is slightly positive, leading to gradual appreciation across horizons. Quantiles are non-decreasing and broaden at further horizons to reflect cumulative volatility." + "> **Agent rationale:** The forecast is constructed under the assumption that the geopolitical risk premium will remain the dominant driver of WTI prices through March 2026. The March 1 origin price of $65.21 reflects a market balancing OPEC+ production restraint against geopolitical fears. Quantiles are non-decreasing, and the point forecast is aligned with the 0.50 quantile, reflecting a moderate outlook with a bias toward volatility as the market waits for more concrete signals regarding the Middle East conflict." ], "text/plain": [ "" @@ -1052,14 +1051,14 @@ { "error_y": { "array": [ - 1.2999999999999972, - 2.200000000000003, - 3.799999999999997 + 1.5, + 2.299999999999997, + 3.4000000000000057 ], "arrayminus": [ - 1.1000000000000083, - 1.7999999999999972, - 2.799999999999997 + 1.3000000000000045, + 2.1999999999999957, + 3.299999999999997 ], "color": "#2ca02c", "symmetric": false, @@ -1086,7 +1085,7 @@ "y": [ 65.2, 64.8, - 64 + 63.5 ], "yaxis": "y" }, @@ -1409,14 +1408,14 @@ { "error_y": { "array": [ - 2.3500000000000085, - 3.200000000000003, - 5 + 2.3000000000000114, + 3.1000000000000085, + 5.400000000000006 ], "arrayminus": [ - 2.049999999999997, - 3, - 4.700000000000003 + 1.8999999999999917, + 2.5, + 5.099999999999994 ], "color": "#2ca02c", "symmetric": false, @@ -1442,8 +1441,8 @@ "xaxis": "x2", "y": [ 66.85, - 67.2, - 67.5 + 67.3, + 68.1 ], "yaxis": "y2" }, @@ -1766,13 +1765,13 @@ { "error_y": { "array": [ - 1.7999999999999972, - 2.700000000000003, - 4.5 + 2, + 3.299999999999997, + 5.5 ], "arrayminus": [ - 1.3000000000000114, - 2.299999999999997, + 1.2999999999999972, + 2.200000000000003, 3.799999999999997 ], "color": "#2ca02c", @@ -1798,9 +1797,9 @@ ], "xaxis": "x3", "y": [ - 65.4, - 65.8, - 66.3 + 65.5, + 66.2, + 67 ], "yaxis": "y3" } @@ -2826,7 +2825,7 @@ " 21 bdays\n", " 74.6\n", " 65.2\n", - " 64.0\n", + " 63.5\n", " \n", " \n", " 2026-02-23\n", @@ -2839,32 +2838,32 @@ " 10 bdays\n", " 94.8\n", " 64.7\n", - " 67.2\n", + " 67.3\n", " \n", " \n", " 21 bdays\n", " 92.3\n", " 64.4\n", - " 67.5\n", + " 68.1\n", " \n", " \n", " 2026-03-02\n", " 5 bdays\n", " 94.8\n", " 64.6\n", - " 65.4\n", + " 65.5\n", " \n", " \n", " 10 bdays\n", " 93.5\n", " 64.3\n", - " 65.8\n", + " 66.2\n", " \n", " \n", " 21 bdays\n", " 101.4\n", " 64.5\n", - " 66.3\n", + " 67.0\n", " \n", " \n", "\n", @@ -2875,13 +2874,13 @@ "Origin Horizon \n", "2026-02-02 5 bdays 64.4 62.6 65.2\n", " 10 bdays 62.3 63.4 64.8\n", - " 21 bdays 74.6 65.2 64.0\n", + " 21 bdays 74.6 65.2 63.5\n", "2026-02-23 5 bdays 71.2 64.6 66.8\n", - " 10 bdays 94.8 64.7 67.2\n", - " 21 bdays 92.3 64.4 67.5\n", - "2026-03-02 5 bdays 94.8 64.6 65.4\n", - " 10 bdays 93.5 64.3 65.8\n", - " 21 bdays 101.4 64.5 66.3" + " 10 bdays 94.8 64.7 67.3\n", + " 21 bdays 92.3 64.4 68.1\n", + "2026-03-02 5 bdays 94.8 64.6 65.5\n", + " 10 bdays 93.5 64.3 66.2\n", + " 21 bdays 101.4 64.5 67.0" ] }, "metadata": {}, @@ -2892,7 +2891,7 @@ "output_type": "stream", "text": [ "\n", - "Mean MAE Prophet: $19.23 Agent: $18.09\n" + "Mean MAE Prophet: $19.23 Agent: $17.94\n" ] } ], @@ -3112,7 +3111,7 @@ "output_type": "stream", "text": [ "Saved 6 shock forecasts.\n", - "Agent Brier score: 0.2425\n" + "Agent Brier score: 0.2592\n" ] } ], @@ -3162,13 +3161,13 @@ "\n", "| | |\n", "|---|---|\n", - "| **Prediction** | P(up > +$5) = **25%** `██░░░░░░░░ 25%` |\n", - "| **Confidence** | Medium 🟡 |\n", - "| **Rationale** | As of February 1, 2026, WTI is trading at ~$65.42/bbl. The market is currently characterized by a delicate balance between structural oversupply concerns and a significant, though unconfirmed, geopolitical risk premium. Specifically, tensions surrounding the Strait of Hormuz are contributing an estimated $4-$10/bbl to the current price. While a $5/bbl move upward (to >$70.42) in five trading days is substantial, it is well within the realm of volatility observed in this environment if a supply-related headline (e.g., an escalation in the Middle East or a physical disruption in the Strait) were to materialize. Given the calibration guidance, the presence of 'escalating unconfirmed risk' justifies a probability in the 20-40% range. |\n", - "| **Key signals** | Heightened geopolitical tension in the Middle East and the Strait of Hormuz · Estimated $4-$10/bbl risk premium currently embedded in WTI prices · OPEC+ production restraint maintaining a tight floor on prices · High market sensitivity to supply disruption headlines |\n", + "| **Prediction** | P(up > +$5) = **15%** `██░░░░░░░░ 15%` |\n", + "| **Confidence** | High 🟢 |\n", + "| **Rationale** | As of February 1, 2026, WTI is trading at $65.42/bbl. The market is currently balanced by a structural supply surplus countered by a modest, recurring geopolitical risk premium stemming from Middle East tensions. While these risks provide a floor for prices, they do not constitute a confirmed, imminent physical supply disruption that would trigger a sustained $5/bbl (approx. 7.6%) spike within a 5-day window. OPEC+ production restraint and IEA demand projections suggest a stable, range-bound environment. A $5/bbl move in 5 days would likely require a significant, unexpected 'black swan' event or an extreme escalation, which currently lacks supporting indicators. The probability is therefore set toward the lower end of the base rate range. |\n", + "| **Key signals** | OPEC+ commitment to current production levels through Q1 2026 limiting extreme supply volatility · Persistent but contained geopolitical risk premium in Middle East shipping lanes · Balanced IEA demand/supply outlook for 2026 |\n", "| **Actual outcome** | No shock — price moved **+2.22/bbl** |\n", "| **Verdict** | Actual: +$2.22/bbl — no shock |\n", - "| **Brier score** | 0.062 🟢 |\n" + "| **Brier score** | 0.022 🟢 |\n" ], "text/plain": [ "" @@ -3185,13 +3184,13 @@ "\n", "| | |\n", "|---|---|\n", - "| **Prediction** | P(up > +$5) = **35%** `████░░░░░░ 35%` |\n", + "| **Prediction** | P(up > +$5) = **25%** `██░░░░░░░░ 25%` |\n", "| **Confidence** | Medium 🟡 |\n", - "| **Rationale** | The market is currently driven by a significant geopolitical risk premium, with the Strait of Hormuz acting as a flashpoint between the US and Iran. While current prices have consolidated near $63/bbl after recent volatility, the risk of a sudden escalation that disrupts supply in the Strait is non-negligible. A $5/bbl move represents an approximate 8% increase, which is within the range of potential volatility jumps during periods of heightened geopolitical tension. Given the guidance for unconfirmed risk (20-40%), a 35% probability reflects that while an escalation is not certain, the market remains fragile to supply-side shocks. |\n", - "| **Key signals** | Heightened geopolitical risk premium related to US-Iran tensions near the Strait of Hormuz. · OPEC+ decision to pause production increments for March 2026, signaling a desire for price support. · Recent market volatility and sensitivity to news flows regarding maritime security in the Middle East. |\n", + "| **Rationale** | While there is no immediate, confirmed supply disruption, the market is currently carrying a geopolitical risk premium centered on potential conflict between the U.S. and Iran affecting the Strait of Hormuz. With WTI at $63.29, an 'upshock' to above $68.29 within 5 trading days would require a sudden escalation of these already heightened tensions. The market is currently consolidating after a retreat from late-January highs, making a move of this magnitude plausible but less likely than a range-bound or moderate outcome. The 25% estimate reflects the 'escalating unconfirmed risk' category provided in the guidance. |\n", + "| **Key signals** | Geopolitical tension in the Persian Gulf/Strait of Hormuz provides ongoing potential for sudden supply shocks. · OPEC+ maintains production restraint through Q1 2026, limiting downside but also capping extreme upside without a major catalyst. · The market is highly sensitive to diplomatic news; any breakdown in current talk of potential de-escalation could trigger rapid, risk-premium-driven price increases. |\n", "| **Actual outcome** | No shock — price moved **-2.03/bbl** |\n", "| **Verdict** | Actual: +$-2.03/bbl — no shock |\n", - "| **Brier score** | 0.122 🟡 |\n" + "| **Brier score** | 0.062 🟢 |\n" ], "text/plain": [ "" @@ -3208,13 +3207,13 @@ "\n", "| | |\n", "|---|---|\n", - "| **Prediction** | P(up > +$5) = **35%** `████░░░░░░ 35%` |\n", + "| **Prediction** | P(up > +$5) = **25%** `██░░░░░░░░ 25%` |\n", "| **Confidence** | Medium 🟡 |\n", - "| **Rationale** | The WTI market is currently heavily influenced by a 'geopolitical risk premium' related to tensions in the Middle East and the Strait of Hormuz. While there was a recent price decline in early February due to demand concerns and inventory data, the risk of a sudden, sharp supply disruption remains elevated. A $5/bbl move (roughly 8%) within 5 days is significant but plausible if there is a concrete escalation in the Strait of Hormuz or new, definitive news regarding Iranian supply. Given that the market is already sensitive to these risks, a probability of 35% sits between 'unconfirmed risk' and 'confirmed disruption', reflecting the potential for sudden volatility as tensions fluctuate. |\n", - "| **Key signals** | Heightened geopolitical tension in the Strait of Hormuz. · Existence of a $4-$10/bbl 'geopolitical risk premium' in current pricing. · OPEC+ policy to pause production increments in March 2026. · Recent reversal in sentiment following U.S. naval advisories in Iranian waters. |\n", + "| **Rationale** | As of Feb 15, 2026, WTI is trading at $62.84. While prices have recently softened from earlier January highs due to diplomatic de-escalation reports and demand concerns, the market remains highly sensitive to volatility in the Strait of Hormuz. The geopolitical risk premium is significant but currently 'unconfirmed' regarding an actual supply halt. Given the binary risk of a sudden disruption in the Persian Gulf, there is a non-trivial probability (estimated at 25%) that events could trigger a move >$5 within 5 business days, fitting the 'escalating unconfirmed risk' calibration category. |\n", + "| **Key signals** | Heightened geopolitical tension in the Strait of Hormuz acting as a volatility floor · Market sensitivity to U.S.-Iran diplomatic signals and maritime warnings · OPEC+ production restraint providing a buffer against price drops despite IEA demand forecasts |\n", "| **Actual outcome** | No shock — price moved **+3.98/bbl** |\n", "| **Verdict** | Actual: +$3.98/bbl — no shock |\n", - "| **Brier score** | 0.122 🟡 |\n" + "| **Brier score** | 0.062 🟢 |\n" ], "text/plain": [ "" @@ -3231,13 +3230,13 @@ "\n", "| | |\n", "|---|---|\n", - "| **Prediction** | P(up > +$5) = **15%** `██░░░░░░░░ 15%` |\n", + "| **Prediction** | P(up > +$5) = **35%** `████░░░░░░ 35%` |\n", "| **Confidence** | Medium 🟡 |\n", - "| **Rationale** | While there is a geopolitical risk premium associated with tensions in the Middle East, specifically around the Strait of Hormuz, the market has recently been tempered by IEA forecasts of lower demand and higher U.S. inventories. Prices have remained in a range, and while volatility is high, there is no immediate 'confirmed' supply disruption that would justify the high threshold of a $5/bbl increase in just 5 trading days. The current price of $66.43 represents the upper end of the recent consolidation; without a specific catalyst to trigger an breakout, a move to $71.43+ within a week is unlikely to be the base-case outcome. |\n", - "| **Key signals** | Heightened geopolitical risk premium (Strait of Hormuz tensions) provides a structural floor but is currently priced in. · IEA demand concerns and rising U.S. inventory levels act as a dampener on aggressive upside moves. · OPEC+ production policy remains cautious and flexible, preventing panic buying. |\n", + "| **Rationale** | The WTI market is currently navigating significant 'geopolitical risk premium' due to rising tensions in the Persian Gulf and concerns over the Strait of Hormuz. Following a 4% surge on February 18, the market remains highly reactive to news flows regarding potential shipping disruptions. While there is no confirmed physical supply loss (which would necessitate a higher probability), the current environment of escalating, unconfirmed risk fits the '20-40%' range. The recent price action confirms market sensitivity to potential supply shocks, and a move +$5 (to ~$71.43) within 5 trading days is plausible if geopolitical friction intensifies further, though it remains a tail-risk event. |\n", + "| **Key signals** | Escalating tensions and naval activity in the Strait of Hormuz. · High market sensitivity to geopolitical risk, evidenced by the 4% rally on Feb 18. · OPEC+ decision to pause production increments, limiting near-term supply growth. |\n", "| **Actual outcome** | No shock — price moved **+4.92/bbl** |\n", "| **Verdict** | Actual: +$4.92/bbl — no shock |\n", - "| **Brier score** | 0.022 🟢 |\n" + "| **Brier score** | 0.122 🟡 |\n" ], "text/plain": [ "" @@ -3254,13 +3253,13 @@ "\n", "| | |\n", "|---|---|\n", - "| **Prediction** | P(up > +$5) = **25%** `██░░░░░░░░ 25%` |\n", + "| **Prediction** | P(up > +$5) = **15%** `██░░░░░░░░ 15%` |\n", "| **Confidence** | Medium 🟡 |\n", - "| **Rationale** | As of March 1, 2026, the WTI price of ~$65.21 is trading within a context of heightened geopolitical anxiety, specifically regarding U.S.-Iran tensions and potential disruptions at the Strait of Hormuz. While the underlying physical market is adequately supplied, the 'geopolitical risk premium' is significant. A move to >$70.21 (a $5/bbl increase) within 5 trading days would require either a concrete escalation in physical disruption or a sharp tightening of market sentiment. Given the volatility, a 25% probability accounts for the high sensitivity to news, placing it between 'no unusual catalyst' and 'confirmed supply disruption'. |\n", - "| **Key signals** | Escalating tensions between the U.S. and Iran · Market sensitivity regarding the Strait of Hormuz · OPEC+ commitment to production restraint, limiting downside inventory pressure |\n", + "| **Rationale** | WTI closed at $65.21 on March 1, 2026. A move of +$5 to reach ~$70.21 in just five trading days represents a roughly 7.7% increase, a significant volatility jump for a short window absent an acute supply shock. While market sentiment notes 'escalating unconfirmed risks' regarding geopolitical supply chain stability, there is no evidence of an immediate, confirmed disruption. The market appears to be in a consolidation phase near $65, balancing these risks against adequate global supply levels. Consequently, the probability remains toward the lower end of the provided guidance range (base rate to slightly elevated). |\n", + "| **Key signals** | Market focus on geopolitical risks versus stable physical supply · Technical resistance levels near $66-$67 established over the last few weeks · Absence of immediate, confirmed production outage |\n", "| **Actual outcome** | **SHOCK** — price moved **+23.54/bbl** |\n", "| **Verdict** | Actual: +$23.54/bbl (>5) — shock materialised |\n", - "| **Brier score** | 0.562 🔴 |\n" + "| **Brier score** | 0.722 🔴 |\n" ], "text/plain": [ "" @@ -3279,8 +3278,8 @@ "|---|---|\n", "| **Prediction** | P(up > +$5) = **75%** `████████░░ 75%` |\n", "| **Confidence** | High 🟢 |\n", - "| **Rationale** | The WTI market is currently facing a confirmed, high-impact supply disruption following the closure of the Strait of Hormuz in late February 2026. This chokepoint handles ~20% of global seaborne oil trade. With air strikes continuing and tankers avoiding the region due to war-risk, market participants are pricing in an extreme geopolitical risk premium. A move of >$5/bbl (approx. 6%) over a 5-day horizon is highly plausible given the ongoing instability, the lack of a clear timeline for the reopening of the strait, and the psychological impact of the IRGC's activities. This aligns with the 'confirmed supply disruption' category for high-probability outcomes. |\n", - "| **Key signals** | Continued closure of the Strait of Hormuz · Extreme volatility due to the US-Iran military conflict · Global shipping rerouting and imposition of war-risk surcharges |\n", + "| **Rationale** | The WTI market is currently gripped by a major geopolitical supply shock involving the Strait of Hormuz, a critical chokepoint. With military hostilities involving Iran underway and reports of sea mines and merchant vessel boarding, the risk of a protracted supply disruption is acute. While the US and allies are preparing to intervene, the market is pricing in a severe 'war premium' as short-term supply chain confidence has collapsed. Given the extreme volatility observed over the past few days (with prices surging from ~$67 to $81), further upside movement exceeding $5/bbl is highly probable if the conflict intensifies further in the next 5 days. |\n", + "| **Key signals** | Severe disruption to tanker traffic in the Strait of Hormuz · Active military hostilities involving Iran · Surging war-risk insurance premiums · Extreme short-term price volatility |\n", "| **Actual outcome** | No shock — price moved **-1.27/bbl** |\n", "| **Verdict** | Actual: +$-1.27/bbl — no shock |\n", "| **Brier score** | 0.562 🔴 |\n" @@ -3365,11 +3364,11 @@ ], "xaxis": "x", "y": [ + 0.15, + 0.25, 0.25, - 0.35, 0.35, 0.15, - 0.25, 0.75 ], "yaxis": "y" @@ -3435,12 +3434,12 @@ ], "xaxis": "x2", "y": [ - 0.0625, - 0.0925, - 0.1025, - 0.0825, - 0.17850000000000002, - 0.2425 + 0.0225, + 0.0425, + 0.049166666666666664, + 0.06749999999999999, + 0.1985, + 0.25916666666666666 ], "yaxis": "y2" }, @@ -3526,10 +3525,10 @@ "size": 8 }, "showarrow": false, - "text": "25%", + "text": "15%", "x": "2026-02-02", "xref": "x", - "y": 0.25, + "y": 0.15, "yref": "y", "yshift": 12 }, @@ -3539,10 +3538,10 @@ "size": 8 }, "showarrow": false, - "text": "35%", + "text": "25%", "x": "2026-02-09", "xref": "x", - "y": 0.35, + "y": 0.25, "yref": "y", "yshift": 12 }, @@ -3552,10 +3551,10 @@ "size": 8 }, "showarrow": false, - "text": "35%", + "text": "25%", "x": "2026-02-16", "xref": "x", - "y": 0.35, + "y": 0.25, "yref": "y", "yshift": 12 }, @@ -3565,10 +3564,10 @@ "size": 8 }, "showarrow": false, - "text": "15%", + "text": "35%", "x": "2026-02-23", "xref": "x", - "y": 0.15, + "y": 0.35, "yref": "y", "yshift": 12 }, @@ -3578,10 +3577,10 @@ "size": 8 }, "showarrow": false, - "text": "25%", + "text": "15%", "x": "2026-03-02", "xref": "x", - "y": 0.25, + "y": 0.15, "yref": "y", "yshift": 12 }, @@ -4641,7 +4640,7 @@ " \n", " \n", " Analyst Agent\n", - " 0.2425\n", + " 0.2592\n", " \n", " \n", " Prophet\n", @@ -4654,7 +4653,7 @@ "text/plain": [ " Mean Brier score\n", "Method \n", - "Analyst Agent 0.2425\n", + "Analyst Agent 0.2592\n", "Prophet 0.1927" ] }, @@ -4709,7 +4708,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 13, "id": "bb5d02a8", "metadata": {}, "outputs": [ @@ -4770,8 +4769,8 @@ ], "source": [ "# ── Stream 3 task spec (edit this) ────────────────────────────────────────────\n", - "SCENARIO_AS_OF = SCENARIO_ORIGIN # 2026-03-02 — conflict onset\n", - "# SCENARIO_AS_OF = pd.Timestamp(\"2026-02-02\") # pre-shock, quieter market\n", + "# SCENARIO_AS_OF = SCENARIO_ORIGIN # 2026-03-02 — conflict onset\n", + "SCENARIO_AS_OF = pd.Timestamp(\"2026-02-02\") # pre-shock, quieter market\n", "# SCENARIO_AS_OF = pd.Timestamp.today() # live — no deep historical fence\n", "\n", "_SCENARIO_SCHEMA = ScenarioAgentForecastOutput.prompt_schema_json()\n", @@ -4812,7 +4811,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 14, "id": "2bb25d8b", "metadata": {}, "outputs": [ @@ -4921,7 +4920,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 15, "id": "6e0e0d7f", "metadata": {}, "outputs": [ @@ -4937,7 +4936,7 @@ "text/markdown": [ "#### Agent response — Stream 3 *(origin: 2026-02-02, WTI $62.14/bbl)*\n", "\n", - "Base case: **Market Fundamentals Balance Prevails**" + "Base case: **Geopolitical Risk De-escalation**" ], "text/plain": [ "" @@ -4950,16 +4949,16 @@ "data": { "text/markdown": [ "---\n", - "**Market Fundamentals Balance Prevails** ★ **base case**\n", + "**Geopolitical Risk De-escalation** ★ **base case**\n", "\n", - "The market remains range-bound as strong non-OPEC production offsets concerns over OPEC+ supply discipline and tepid global demand growth.\n", + "Tensions in the Strait of Hormuz subside, leading to a quick unwinding of the risk premium that boosted prices in early 2026, causing a return to fundamentals.\n", "\n", "| | |\n", "|---|---|\n", - "| Probability | **50%** `█████░░░░░ 50%` |\n", - "| WTI range (60 days) | $60 – $68 /bbl |\n", - "| Point estimate | **$64 /bbl** |\n", - "| Key drivers | Consistent growth in non-OPEC crude output · OPEC+ successfully maintains current production caps |\n" + "| Probability | **45%** `████░░░░░░ 45%` |\n", + "| WTI range (60 days) | $58 – $64 /bbl |\n", + "| Point estimate | **$62 /bbl** |\n", + "| Key drivers | Reduction in geopolitical tension in the Middle East · Market focus returning to the structural 2026 supply surplus |\n" ], "text/plain": [ "" @@ -4972,16 +4971,16 @@ "data": { "text/markdown": [ "---\n", - "**Geopolitical Escalation Risk Premium**\n", + "**Strait of Hormuz Disruption**\n", "\n", - "Heightened tensions in the Middle East or energy-producing regions trigger a rally as market participants price in a sustained supply disruption premium.\n", + "Escalation in regional conflict leads to temporary closure or severe restriction of oil flows through the Strait of Hormuz, sparking a sharp, supply-shock-driven rally.\n", "\n", "| | |\n", "|---|---|\n", - "| Probability | **30%** `███░░░░░░░ 30%` |\n", - "| WTI range (60 days) | $67 – $75 /bbl |\n", - "| Point estimate | **$71 /bbl** |\n", - "| Key drivers | Sudden escalation in Middle East hostilities · Significant disruption to Iranian or Venezuelan export logistics |\n" + "| Probability | **20%** `██░░░░░░░░ 20%` |\n", + "| WTI range (60 days) | $75 – $85 /bbl |\n", + "| Point estimate | **$80 /bbl** |\n", + "| Key drivers | Physical supply disruption in the Middle East · Panic buying and sudden spikes in volatility |\n" ], "text/plain": [ "" @@ -4994,16 +4993,16 @@ "data": { "text/markdown": [ "---\n", - "**Demand-Led Cyclical Downturn**\n", + "**Fundamentals-Driven Price Correction**\n", "\n", - "Signs of slowing global economic growth and high inventories push prices downward as the market loses confidence in OPEC+ intervention efficacy.\n", + "Geopolitical risks stabilize at a constant, lower level, and the market increasingly prioritizes data showing global oil supply outpacing demand growth.\n", "\n", "| | |\n", "|---|---|\n", - "| Probability | **20%** `██░░░░░░░░ 20%` |\n", - "| WTI range (60 days) | $55 – $62 /bbl |\n", + "| Probability | **35%** `████░░░░░░ 35%` |\n", + "| WTI range (60 days) | $55 – $60 /bbl |\n", "| Point estimate | **$58 /bbl** |\n", - "| Key drivers | Weakening manufacturing and consumer demand indicators · Higher-than-expected rise in global oil storage levels |\n" + "| Key drivers | Evidence of structural 2026 supply surplus · Weaker-than-expected global demand growth |\n" ], "text/plain": [ "" @@ -5017,7 +5016,7 @@ "text/markdown": [ "---\n", "\n", - "> **Overall reasoning:** As of February 2026, WTI sits at approximately $65/bbl, balancing between ongoing structural supply abundance from non-OPEC sources and significant, yet intermittent, geopolitical risk premiums. The base case reflects an environment where market fundamentals—dominated by sufficient global supply and OPEC+’s cautious, disciplined output stance—largely contain volatility. While geopolitical shocks create occasional price spikes, the lack of actual, sustained physical supply loss means the market remains anchored near its recent range. The other scenarios represent the tails of this distribution: either a materialization of supply disruptions or an accumulation of bearish macro data that forces a re-evaluation of demand." + "> **Overall reasoning:** As of February 1, 2026, the WTI price of $65.42 reflects a significant geopolitical risk premium following instability in the Middle East throughout January. The base case, 'Geopolitical Risk De-escalation,' assumes that while tension remains, the acute fear of major physical supply disruptions will fade over the next 60 days, allowing the underlying market reality—a projected structural surplus for 2026—to exert downward pressure on prices from current elevated levels. While the 'Strait of Hormuz Disruption' scenario poses a significant upside tail risk, and a pure 'Fundamentals-Driven Price Correction' remains a viable path should the risk premium evaporate entirely, the most likely path is a moderate reversion toward fundamental levels as the market stabilizes." ], "text/plain": [ "" diff --git a/implementations/energy_oil_forecasting/04_systematic_backtest_eval.ipynb b/implementations/energy_oil_forecasting/04_systematic_backtest_eval.ipynb index 7d45e40f..b912d36e 100644 --- a/implementations/energy_oil_forecasting/04_systematic_backtest_eval.ipynb +++ b/implementations/energy_oil_forecasting/04_systematic_backtest_eval.ipynb @@ -9296,7 +9296,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.12" } }, "nbformat": 4,