From f62e593fa06038be859f8f25b681bfb090e437b6 Mon Sep 17 00:00:00 2001 From: a-kore Date: Thu, 20 Aug 2026 12:48:07 -0400 Subject: [PATCH 1/3] fix: record LLM-process costs in Langfuse LiteLLM's langfuse_otel callback emits no generation when the completion runs inside an already-active Langfuse span. Every LLM-process predict() is wrapped in @langfuse_observe, so that is the configuration this repo always runs in, and token usage never reached Langfuse. An A/B of identical calls differing only in the wrapper confirms it: without @observe a litellm_request generation arrives with model and cost, and with it only the root span does. Langfuse's own data shows these generations stopping in early June 2026, when the current dependency pair landed. Create the generation directly instead, setting model, input, output, and usage_details. Cost is left to Langfuse, which derives it from usage against its own price table, and those prices match the proxy's published rates. A gemini-3.5-flash call billed by the proxy at $0.0039690000 records as $0.003969. The model name is passed bare rather than openai/-prefixed, because Langfuse matches its price table on the bare name. Langfuse tracing is also disabled across the test suite. Several suites load real credentials from .env, so without that guard unit tests with a mocked litellm shipped zero-usage observations to the live project. --- .../methods/llm_processes/_client.py | 77 ++++- .../methods/llm_processes/test__client.py | 75 +++++ aieng-forecasting/tests/conftest.py | 19 ++ .../context/artifacts/AGENTS.md.md | 5 +- .../context/artifacts/README.md.md | 30 +- ...asting__methods__agentic__adk_runner.py.md | 24 +- ...ing__methods__agentic__agent_factory.py.md | 268 +++++++++++++++-- ...casting__methods__agentic__predictor.py.md | 8 +- ...ing__methods__llm_processes___client.py.md | 77 ++++- .../artifacts/implementations__README.md.md | 2 +- ...rate_decisions__starter_agent__agent.py.md | 11 +- ...nt__skills__research-playbook__SKILL.md.md | 6 + ...ting__04_systematic_backtest_eval.ipynb.md | 194 +++++++++--- ...il_forecasting__06_protected_eval.ipynb.md | 2 +- ...oil_forecasting__99_starter_agent.ipynb.md | 51 ++-- ...l_forecasting__adaptive_agent__agent.py.md | 15 +- ...ns__energy_oil_forecasting__analysis.py.md | 283 ++++++++++++++++++ ...orecasting__specs__energy_oil_eval.yaml.md | 15 +- ...ting__specs__energy_oil_eval_smoke.yaml.md | 2 +- ...forecasting__starter_agent____init__.py.md | 7 +- ...il_forecasting__starter_agent__agent.py.md | 108 +++---- ...nt__skills__research-playbook__SKILL.md.md | 6 + ...il_forecasting__starter_agent__tools.py.md | 237 +++++++++++++++ ...tations__energy_oil_forecasting__viz.py.md | 270 +++++++++++++++++ ...ce_forecasting__starter_agent__agent.py.md | 11 +- ...nt__skills__research-playbook__SKILL.md.md | 6 + ...00_forecasting__starter_agent__agent.py.md | 11 +- ...nt__skills__research-playbook__SKILL.md.md | 6 + .../concierge_agent/context/catalog.yaml | 112 ++++--- .../references/catalog-summary.yaml | 8 +- 30 files changed, 1710 insertions(+), 236 deletions(-) create mode 100644 aieng-forecasting/tests/conftest.py create mode 100644 implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__tools.py.md diff --git a/aieng-forecasting/aieng/forecasting/methods/llm_processes/_client.py b/aieng-forecasting/aieng/forecasting/methods/llm_processes/_client.py index 24b91d72..6343f0a0 100644 --- a/aieng-forecasting/aieng/forecasting/methods/llm_processes/_client.py +++ b/aieng-forecasting/aieng/forecasting/methods/llm_processes/_client.py @@ -20,13 +20,14 @@ from __future__ import annotations import asyncio +import contextlib import contextvars import json import logging import os import warnings from concurrent.futures import ThreadPoolExecutor -from typing import Any, Callable, TypeVar +from typing import Any, Callable, Iterator, TypeVar from pydantic import BaseModel, ValidationError @@ -91,6 +92,54 @@ def _noop(fn: Any) -> Any: return _noop +class _NoopGeneration: + """Stand-in used when Langfuse is unavailable, so callers need no branching.""" + + def update(self, **kwargs: Any) -> None: + """Discard the update.""" + return + + +@contextlib.contextmanager +def langfuse_generation(*, name: str, model: str, input_messages: Any) -> Iterator[Any]: + """Emit a Langfuse ``generation`` around one LLM call. + + We create the generation explicitly rather than relying on LiteLLM's + ``langfuse_otel`` callback: that callback emits no generation at all when + the call runs inside an already-active Langfuse span, which is every LLMP + ``predict`` (they are wrapped in :func:`langfuse_observe`). The result was + that token usage — and therefore cost — never reached Langfuse for any + LLMP run. Creating it here works nested and keeps model, usage, and + payload under our control. + + Cost is deliberately not set: Langfuse derives it from ``usage_details`` + and its own per-model prices, which match the Vector proxy's published + rates. + + Yields a handle exposing ``update(**kwargs)`` — a no-op stand-in when + Langfuse is not installed or a generation cannot be started, so predictors + remain usable without the ``agentic``/``llm`` extras. + """ + manager = None + try: + from langfuse import get_client # noqa: PLC0415 + + manager = get_client().start_as_current_observation( + as_type="generation", + name=name, + model=model, + input=input_messages, + ) + except Exception: # pragma: no cover - depends on optional dependency + logger.debug("Langfuse generation unavailable; usage will not be traced.", exc_info=True) + + if manager is None: + yield _NoopGeneration() + return + with manager as generation: + yield generation + + def current_trace_info() -> tuple[str | None, str | None]: """Return ``(trace_id, trace_url)`` from the active Langfuse client, if any.""" try: @@ -299,17 +348,21 @@ async def _one_completion_async( # models that don't support them (e.g. temperature on some o-series). kwargs["drop_params"] = True - resp = await litellm.acompletion(**kwargs) - cost = float(getattr(resp, "_hidden_params", {}).get("response_cost") or 0.0) - usage = getattr(resp, "usage", None) - in_tok = int(getattr(usage, "prompt_tokens", 0) or 0) if usage is not None else 0 - out_tok = int(getattr(usage, "completion_tokens", 0) or 0) if usage is not None else 0 - # Log full usage so we can see thinking-token breakdown when available. - # The proxy may populate completion_tokens_details.reasoning_tokens. - if usage is not None: - logger.debug("LLM usage: %s", vars(usage) if hasattr(usage, "__dict__") else usage) - raw = resp.choices[0].message.content - content = strip_markdown_fence(raw) if raw else raw + # ``model`` is the bare name as configured, before any "openai/" prefixing + # above; that is what Langfuse's price table matches on. + with langfuse_generation(name="llm_completion", model=model, input_messages=messages) as generation: + resp = await litellm.acompletion(**kwargs) + cost = float(getattr(resp, "_hidden_params", {}).get("response_cost") or 0.0) + usage = getattr(resp, "usage", None) + in_tok = int(getattr(usage, "prompt_tokens", 0) or 0) if usage is not None else 0 + out_tok = int(getattr(usage, "completion_tokens", 0) or 0) if usage is not None else 0 + # Log full usage so we can see thinking-token breakdown when available. + # The proxy may populate completion_tokens_details.reasoning_tokens. + if usage is not None: + logger.debug("LLM usage: %s", vars(usage) if hasattr(usage, "__dict__") else usage) + raw = resp.choices[0].message.content + content = strip_markdown_fence(raw) if raw else raw + generation.update(output=content, usage_details={"input": in_tok, "output": out_tok}) return content, cost, in_tok, out_tok diff --git a/aieng-forecasting/tests/aieng/forecasting/methods/llm_processes/test__client.py b/aieng-forecasting/tests/aieng/forecasting/methods/llm_processes/test__client.py index d7154cc6..da9cc2a3 100644 --- a/aieng-forecasting/tests/aieng/forecasting/methods/llm_processes/test__client.py +++ b/aieng-forecasting/tests/aieng/forecasting/methods/llm_processes/test__client.py @@ -7,6 +7,8 @@ from __future__ import annotations +import contextlib +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -121,6 +123,7 @@ def _mock_litellm_response(content: str) -> MagicMock: return resp +_CLIENT = "aieng.forecasting.methods.llm_processes._client" _DUMMY_MESSAGES = [{"role": "user", "content": "forecast"}] _DUMMY_FORMAT = {"type": "json_schema", "json_schema": {"name": "x", "schema": {}, "strict": True}} @@ -257,3 +260,75 @@ async def fake_acompletion(**kwargs): # type: ignore[override] assert kw["reasoning_effort"] == "low" assert "extra_body" not in kw or "reasoning_effort" not in kw.get("extra_body", {}) assert kw.get("drop_params") is True + + +# --------------------------------------------------------------------------- +# langfuse_generation — usage/cost reporting for the Langfuse trace +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_generation_receives_bare_model_usage_and_output() -> None: + """The Langfuse generation gets the bare model name, token usage, and output. + + The bare (un-prefixed) name matters: Langfuse matches its per-model price + table on it, so sending ``openai/`` would yield no cost. + """ + captured: dict = {} + + @contextlib.contextmanager + def _fake_generation(*, name: str, model: str, input_messages: object): + captured.update(name=name, model=model, input_messages=input_messages) + + class _Gen: + def update(self, **kwargs: object) -> None: + captured["update"] = kwargs + + yield _Gen() + + resp = _mock_litellm_response('{"ok": 1}') + resp.usage = SimpleNamespace(prompt_tokens=11, completion_tokens=7) + + with ( + patch(f"{_CLIENT}.langfuse_generation", _fake_generation), + patch("litellm.acompletion", new=AsyncMock(return_value=resp)), + ): + await _one_completion_async( + model="gemini-3.5-flash", + messages=_DUMMY_MESSAGES, + response_format=_DUMMY_FORMAT, + temperature=1.0, + max_tokens=512, + timeout_s=30.0, + reasoning_effort=None, + api_base="https://proxy.example.com/v1", + ) + + assert captured["model"] == "gemini-3.5-flash" # not "openai/gemini-3.5-flash" + assert captured["input_messages"] == _DUMMY_MESSAGES + assert captured["update"]["usage_details"] == {"input": 11, "output": 7} + assert captured["update"]["output"] == '{"ok": 1}' + + +@pytest.mark.asyncio +async def test_completion_succeeds_when_langfuse_is_unavailable() -> None: + """A failing Langfuse client degrades to a no-op; the completion still returns.""" + resp = _mock_litellm_response('{"ok": 1}') + resp.usage = SimpleNamespace(prompt_tokens=3, completion_tokens=4) + + with ( + patch(f"{_CLIENT}.get_client", side_effect=RuntimeError("no langfuse"), create=True), + patch("litellm.acompletion", new=AsyncMock(return_value=resp)), + ): + content, _cost, in_tok, out_tok = await _one_completion_async( + model="gemini-3.5-flash", + messages=_DUMMY_MESSAGES, + response_format=_DUMMY_FORMAT, + temperature=1.0, + max_tokens=512, + timeout_s=30.0, + reasoning_effort=None, + ) + + assert content == '{"ok": 1}' + assert (in_tok, out_tok) == (3, 4) diff --git a/aieng-forecasting/tests/conftest.py b/aieng-forecasting/tests/conftest.py new file mode 100644 index 00000000..b43316f5 --- /dev/null +++ b/aieng-forecasting/tests/conftest.py @@ -0,0 +1,19 @@ +"""Suite-wide test configuration. + +Disables Langfuse tracing for the whole test suite. The LLM-call seam emits a +Langfuse generation per completion (see +``aieng.forecasting.methods.llm_processes._client.langfuse_generation``), and +several suites load real credentials from the repo-root ``.env``. Without this +guard, unit tests with a mocked ``litellm`` would ship real, zero-usage +observations to the live Langfuse project. + +Set at import time rather than in a fixture so it lands before any test module +constructs a Langfuse client. +""" + +from __future__ import annotations + +import os + + +os.environ["LANGFUSE_TRACING_ENABLED"] = "false" diff --git a/implementations/getting_started/concierge_agent/context/artifacts/AGENTS.md.md b/implementations/getting_started/concierge_agent/context/artifacts/AGENTS.md.md index a7d37df5..3e353114 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/AGENTS.md.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/AGENTS.md.md @@ -20,7 +20,7 @@ So "done" always includes a documentation reconciliation step. Before considerin 1. **Grep for what you touched** across docs — the feature name, the module/class/function, the dataset, the spec, the notebook. The fast version: `grep -rn "" --include="*.md" .` (and check notebook markdown cells). Don't rely on memory for where something is mentioned. 2. **Reconcile every hit.** If a doc calls something "planned", "deferred", "not yet wired in", a "seam", or "out of scope" and you just made it real, update that wording. If a doc lists files, notebooks, predictors, specs, or data sources and you added or removed one, fix the list. If you changed a default, a metric, or a command, fix it everywhere it appears. -3. **Update the layered docs together**, not just the nearest one: the use-case README (most detail), the reference-implementations table in the root `README.md`, the method catalog (`aieng-forecasting/aieng/forecasting/methods/README.md`) when you touch a reusable predictor, and `planning-docs/roadmap.md` when something moves from "extension idea" to "shipped". +3. **Update the layered docs together**, not just the nearest one: the use-case README (most detail), the contents table in the root `README.md`, the method catalog (`aieng-forecasting/aieng/forecasting/methods/README.md`) when you touch a reusable predictor, and `planning-docs/roadmap.md` when something moves from "extension idea" to "shipped". Concrete example: integrating Canada's Food Price Report PDFs into the food-price LLM-Process prompt is **not done when the code runs** — it is done when `implementations/food_price_forecasting/README.md` (which currently frames report→prompt wiring as a deferred extension) and the "Reports as predictor context" entry in `planning-docs/roadmap.md` no longer describe it as future work. Shipping the code while those still say "deferred" is the regression the reviewer should catch. @@ -35,6 +35,7 @@ The older planning log, backlog, project charter, and technical-design files und Project shape to keep in mind: - The core library `aieng.forecasting` owns stable infrastructure; reusable predictors live in `aieng.forecasting.methods`; use-case material lives in `implementations//`. +- Strategy guides live under `guides/` (onboard a dataset, create an experiment, customize an agent, audit a result). - YAML specs are co-located under `implementations//specs/`. - Reference implementations: Getting Started, Food Price Forecasting, Energy/Oil (stateless capability track plus an adaptive learning agent), BoC Rate Decisions (quantitative path, cutoff-aware press-release ingestion, and a reasoning-alignment evaluator), and S&P 500 (in active development). - Energy/oil's older information-session notebooks are archived under `playground/energy_case_study/`. @@ -42,7 +43,7 @@ Project shape to keep in mind: ### README files -Search the repo for `README.md` files (excluding `.venv/`) to find every README — there is one at the root, one per package (`aieng-forecasting/`, `implementations/`), the method catalog under `aieng-forecasting/aieng/forecasting/methods/`, and one per use case under `implementations//`. These are the primary user surface and the first thing a new contributor reads; the reconciliation rule above applies to all of them. Keep them accurate and production-quality: describe what the code does and what you can build from it, with no internal program, scheduling, or ownership framing. +Search the repo for `README.md` files (excluding `.venv/`) to find every README — there is one at the root, one per package (`aieng-forecasting/`, `implementations/`), the method catalog under `aieng-forecasting/aieng/forecasting/methods/`, one per use case under `implementations//`, and the strategy-guide index under `guides/`. These are the primary user surface and the first thing a new contributor reads; the reconciliation rule above applies to all of them. Keep them accurate and production-quality: describe what the code does and what you can build from it, with no internal program, scheduling, or ownership framing. --- diff --git a/implementations/getting_started/concierge_agent/context/artifacts/README.md.md b/implementations/getting_started/concierge_agent/context/artifacts/README.md.md index 4279d40e..213fb6e7 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/README.md.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/README.md.md @@ -6,7 +6,19 @@ kind: markdown A foundation for building, evaluating, and comparing forecasting systems — conventional numerical models, LLM Processes, and agentic forecasters — on real economic, financial, and event-prediction tasks. -The repository pairs a small, stable core library with a set of self-contained reference implementations. The library gives you cutoff-safe data handling, a single `Predictor` interface, and a backtest/evaluation harness. Each reference implementation is a worked example of a different forecasting problem and the techniques that suit it. Start from whichever one is closest to what you want to build. +## Contents + +The repo has two layers. A small core library (`aieng.forecasting`) owns cutoff-safe data handling, a shared `Predictor` interface, reusable methods, and the backtest/evaluation harness. Self-contained **reference implementations** under [`implementations/`](implementations/) apply those methods to real forecasting problems — pick the one closest to what you want to build; each directory has its own README. + +| # | Implementation | Use case | Methods | +| --- | --- | --- | --- | +| 0 | [Getting started](implementations/getting_started/) | Canada CPI gasoline, one month ahead — the smallest end-to-end loop | Naive last-value, AutoARIMA; CRPS via `backtest()` / `evaluate()` | +| 1 | [S&P 500](implementations/sp500_forecasting/) | Daily index returns under a leak-safe macro/market covariate panel (1 / 5 / 21 business-day horizons) | Naive, ETS, Kalman, AutoARIMA, linear regression, LightGBM; covariate-aware LLM-Process | +| 2 | [Food price forecasting](implementations/food_price_forecasting/) | Multivariate Canadian food CPI in the style of Canada's Food Price Report (nine sub-indices, 12-month trajectory, avg/avg YoY) | Naive last-value, AutoARIMA; report-grounded LLM-Process (quantile grid and sampled trajectory) | +| 3 | [Energy / oil](implementations/energy_oil_forecasting/) | Daily WTI crude under regime-breaking news (continuous trajectory, binary up-shock, scenario analysis) | Prophet, LLM-Process, news-grounded agent, code-executing agent, adaptive (curriculum-trained) agent | +| 4 | [BoC rate decisions](implementations/boc_rate_decisions/) | Will the Bank of Canada cut, hold, or hike at its next meeting? (ordered categorical; binary cut-vs-not special case) | Climatological frequency, multinomial logistic, categorical LLM-Process, analyst agent; LLM-as-judge reasoning alignment | + +Also in this README: [Setup](#setup) · [Core concepts](#core-concepts) · [Repository layout](#repository-layout) · [Documentation](#documentation) > **👉 First time here? Run the environment check.** After `uv sync` (see [Setup](#setup)), open [`implementations/getting_started/00_environment_check.ipynb`](implementations/getting_started/00_environment_check.ipynb) and run it top to bottom. It's a self-guided preflight that verifies every capability — proxy LLM inference, Langfuse, E2B code execution, StatCan/FRED data access, and an end-to-end mini backtest — and tells you exactly what to fix when something isn't set up. **Do this before anything else.** @@ -27,18 +39,11 @@ Every method can be used in one of two modes, and the distinction runs through t ## Reference implementations -Each is independent and self-contained — pick the one that matches the problem you care about, and read that directory's `README.md` for the full walkthrough. They are numbered in a recommended order that mirrors the bootcamp progression — conventional numerical methods → LLM Processes → agents → agentic evaluation — but any one stands on its own, so jump straight to the problem you care about. - -**Start here → #0 [`getting_started/`](implementations/getting_started/)** — one CPI series, one month ahead. The smallest end-to-end loop: a `Predictor`, a `BacktestSpec` and `EvalSpec`, naive + AutoARIMA baselines, CRPS scoring. The place to learn the evaluation framework before picking a domain below. Also includes [`99_repo_concierge.ipynb`](implementations/getting_started/99_repo_concierge.ipynb) — a lite-model repo guide for “how does this codebase work?” questions (`uv run adk run implementations/getting_started/concierge_agent` from the repo root). +Use cases, methods, and links are in the [contents](#contents) table above. Each implementation is independent — pick the problem you care about and read that directory's `README.md` for the full walkthrough. They are numbered in a recommended order that mirrors the bootcamp progression — conventional numerical methods → LLM Processes → agents → agentic evaluation — but any one stands on its own. -| # | Implementation | The problem | Concepts & techniques it demonstrates | -| --- | -------------------------------------------------------------------- | ------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | [`sp500_forecasting/`](implementations/sp500_forecasting/) | S&P 500 returns under a macro/market covariate panel. | A head-to-head of conventional numerical methods (naive, ETS, Kalman, AutoARIMA, linear regression, LightGBM) plus a covariate-aware LLM-Process, all reading the same leak-safe covariate panel. Cumulative-return targets at 1/5/21-business-day horizons, CRPS + direction metrics, config-driven specs. | -| 2 | [`food_price_forecasting/`](implementations/food_price_forecasting/) | A multivariate food-CPI trajectory, in the style of Canada's Food Price Report. | Nine correlated sub-indices, a 12-step trajectory, a domain metric (avg/avg YoY), baselines vs LLM-Process predictors, leakage-aware backtests, and cached artifacts for fast iteration. | -| 3 | [`energy_oil_forecasting/`](implementations/energy_oil_forecasting/) | Daily WTI crude-oil price under regime-breaking news. | A capability progression — Prophet → LLM-Process → news-grounded agent → code-executing agent — plus an adaptive agent that learns a strategy from data and is scored before vs after. Continuous trajectories, a binary up-shock task, and interactive scenario analysis. | -| 4 | [`boc_rate_decisions/`](implementations/boc_rate_decisions/) | Will the Bank of Canada cut, hold, or hike at its next meeting? | Discrete-event forecasting: ordered-categorical outcomes on an irregular calendar, RPS scoring and one-vs-rest calibration (instead of CRPS), a binary (Brier) special case, cutoff-aware document ingestion, and an LLM-as-judge that scores an agent's reasoning against the official rationale. | +**Start here → #0 [`getting_started/`](implementations/getting_started/)** if the evaluation loop is new to you. That directory also includes [`99_repo_concierge.ipynb`](implementations/getting_started/99_repo_concierge.ipynb) — a lite-model repo guide for “how does this codebase work?” questions (`uv run adk run implementations/getting_started/concierge_agent` from the repo root). -**Not sure where to start building?** Each of the four domain implementations above ends with a `99_starter_agent.ipynb` — a fresh, hackable **starter agent** (a `starter_agent/` module) with toggleable news search and code execution, two lightweight tool-usage skills, an interactive cell, and one scored forecast. It's the consistent "continue from here" entry point for taking any reference use case in an agentic direction, and a quick end-to-end test of that use case's agent stack. +**Not sure where to start building?** Each of the four domain implementations (#1–#4) ends with a `99_starter_agent.ipynb` — a fresh, hackable **starter agent** (a `starter_agent/` module) with toggleable news search and code execution, two lightweight tool-usage skills, an interactive cell, and one scored forecast. It's the consistent "continue from here" entry point for taking any reference use case in an agentic direction, and a quick end-to-end test of that use case's agent stack. ## Time Series Data sources @@ -71,6 +76,7 @@ On Coder workspaces, bootcamp keys (`OPENAI_*`, `E2B_*`, `LANGFUSE_*`) live in y ```text aieng-forecasting/ # Installable library: import as aieng.forecasting implementations/ # Self-contained reference implementations + co-located specs +guides/ # Step-by-step strategy guides for common build-phase tasks scripts/ # Data-fetch scripts + E2B template builder tests/ # Onboarding integration tests (not run in CI) planning-docs/ # Architecture notes and the extension/roadmap catalog @@ -201,6 +207,8 @@ uv run pre-commit run --all-files ## Documentation - Per-implementation READMEs under [`implementations/`](implementations/) — the primary user surface. +- [`guides/`](guides/) — self-contained, step-by-step strategy guides for the most common build-phase tasks: onboarding a dataset, creating an experiment, customizing an agent's strategy, and auditing a result before you believe it. +- [Architecture atlas](https://vectorinstitute.github.io/agentic-forecasting/architecture-atlas.html) ([source](docs/architecture-atlas.html)) — a self-contained visual atlas of the system architecture: the loop, the temporal fence, predictor families, the harness, agent anatomy, and how each reference implementation instantiates them. - [`aieng-forecasting/README.md`](aieng-forecasting/README.md) and [`aieng-forecasting/aieng/forecasting/methods/README.md`](aieng-forecasting/aieng/forecasting/methods/README.md) — the library and the method catalog. - [`planning-docs/roadmap.md`](planning-docs/roadmap.md) — architecture principles and extension ideas. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__adk_runner.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__adk_runner.py.md index 6337eb98..f1a37153 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__adk_runner.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__adk_runner.py.md @@ -187,7 +187,13 @@ class AdkTextRunner: """Underlying ADK runner (session, artifact, memory services).""" return self._runner - async def _resolve_session_id(self, user_id: str | None, session_id: str | None) -> str: + async def _resolve_session_id( + self, + user_id: str | None, + session_id: str | None, + *, + initial_state: dict[str, Any] | None = None, + ) -> str: """Return the ADK session id to use for a single turn. Parameters @@ -197,6 +203,11 @@ class AdkTextRunner: session_id : str or None Explicit session id from the caller. ``None`` triggers sticky-session lookup or new-session creation depending on ``fresh_session_per_message``. + initial_state : dict[str, Any] or None + Seeded into a newly-created session's state. Only takes effect when + this call actually creates a session — has no effect when an + existing sticky session (``fresh_session_per_message=False``) is + reused, since session state can only be seeded at creation. Returns ------- @@ -210,6 +221,7 @@ class AdkTextRunner: new_session = await self._runner.session_service.create_session( app_name=self.config.app_name, user_id=user_id, + state=initial_state, ) sid = new_session.id elif session_id is not None: @@ -221,6 +233,7 @@ class AdkTextRunner: new_session = await self._runner.session_service.create_session( app_name=self.config.app_name, user_id=user_id, + state=initial_state, ) sid = new_session.id self._conversation_session_by_user[user_id] = sid @@ -234,6 +247,7 @@ class AdkTextRunner: user_id: str | None = None, session_id: str | None = None, run_config: RunConfig | None = None, + initial_state: dict[str, Any] | None = None, ) -> str: """Run one user turn; return the first final model text or an empty string. @@ -252,6 +266,12 @@ class AdkTextRunner: run_config : RunConfig | None, optional The run configuration to use for the run. If not provided, the default run configuration is used. + initial_state : dict[str, Any] | None, optional + Seeded into the session's state when this call creates a new + session (see :meth:`_resolve_session_id`). Use this to pass + harness-controlled values (e.g. a forecast's ``as_of`` date) that + tools can read via ``ToolContext.state`` without the LLM being + able to see or influence them. Returns ------- @@ -275,7 +295,7 @@ class AdkTextRunner: user_id = user_id or self.config.default_user_id - session_id = await self._resolve_session_id(user_id, session_id) + session_id = await self._resolve_session_id(user_id, session_id, initial_state=initial_state) content = genai_types.Content(role="user", parts=[genai_types.Part(text=prompt)]) diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md index 494c0ed4..db6a0513 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md @@ -17,6 +17,7 @@ raises :class:`ImportError` with installation guidance. from __future__ import annotations +import json import logging import os import warnings @@ -24,9 +25,12 @@ from pathlib import Path from typing import Any, Callable, Sequence from aieng.forecasting.methods.agentic.outputs import AgentForecastOutput -from aieng.forecasting.models import LITE_MODEL +from aieng.forecasting.models import ADVANCED_MODEL, LITE_MODEL from google.adk.models.base_llm import BaseLlm -from pydantic import BaseModel, Field, field_validator, model_validator +from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator + + +logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- @@ -80,6 +84,13 @@ except ModuleNotFoundError as exc: # final text, giving the predictor the structured JSON it expects. SMR_STATE_KEY = "__smr_output__" +# Session-state key AgentPredictor seeds with the current prediction's as_of +# date before each run (see AdkTextRunner.run_text_async's initial_state). +# search_web reads this via its ADK-injected ToolContext as the authoritative +# cutoff — unlike the LLM-supplied cutoff_date argument, the model can never +# see or influence this key, so it can't be silently omitted or spoofed. +AS_OF_STATE_KEY = "__as_of__" + def _build_set_model_response_tool() -> FunctionTool: """Return a proxy-compatible ``set_model_response`` shim. @@ -118,12 +129,18 @@ class ContextRetrievalConfig(BaseModel): the calling agent can retrieve grounded, sourced web context without a direct Gemini API key. - Temporal cutoff enforcement is soft (LLM-judgment-based): when - ``enforce_cutoff`` is ``True`` and the calling agent passes a - ``cutoff_date`` to the tool, the inner proxy prompt explicitly asks the - model to exclude post-cutoff sources. This is the same trust model used - by the prior Google Search sub-agent — backtest leakage is a - pedagogically useful discussion point, not a hard guarantee. + Temporal cutoff enforcement has two layers. The first is soft + (LLM-judgment-based): when ``enforce_cutoff`` is ``True`` and the calling + agent passes a ``cutoff_date`` to the tool, the inner proxy prompt + explicitly asks the model to exclude post-cutoff sources. This alone is + not a hard guarantee — backtests have shown it leak real post-cutoff + information despite the instruction. The second, hard layer is an + independent verifier call (see ``verifier_model`` etc. below): a separate + LLM call extracts and judges each factual claim in the search result + against the cutoff, strips violations, and retries the search with + feedback when it cannot produce a sufficiently confident result — + returning an explicit failure sentinel rather than silently risky + content if verification never succeeds within the attempt budget. Attributes ---------- @@ -139,12 +156,27 @@ class ContextRetrievalConfig(BaseModel): enforce_cutoff : bool, default=True When ``True``, the ``search_web`` tool appends a cutoff-date constraint to the user prompt whenever ``cutoff_date`` is supplied by - the calling agent. Set to ``False`` for live (non-backtest) agents - where no temporal fence is needed. + the calling agent, and runs the independent leakage verifier + described above. Set to ``False`` for live (non-backtest) agents + where no temporal fence is needed — the verifier is skipped entirely + in that case, at zero extra cost. temperature : float | None, default=None Sampling temperature for the inner search call. max_output_tokens : int | None, default=None Maximum output tokens for the inner search call. + verifier_model : str, default=ADVANCED_MODEL (``"gemini-3.5-flash"``) + Model used for the independent leakage-verification call. Defaults + to a different model than ``search_model`` so the verifier does not + share the same blind spot as the call it's checking. + verifier_max_attempts : int, default=3 + Maximum number of search-then-verify attempts before giving up and + returning the ``[SEARCH_VERIFICATION_FAILED]`` sentinel. + verifier_confidence_threshold : int, default=8 + Minimum self-reported confidence (1-10) the verifier must report, + alongside a clean verdict, for a result to be accepted. Kept + configurable rather than hardcoded because LLM self-reported + confidence is not well-calibrated: too strict a bar (e.g. a literal + 10) risks exhausting retries on results that were actually fine. """ model_config = {"extra": "forbid"} @@ -159,6 +191,9 @@ class ContextRetrievalConfig(BaseModel): enforce_cutoff: bool = True temperature: float | None = Field(default=None, ge=0.0, le=2.0) max_output_tokens: int | None = Field(default=None, ge=1) + verifier_model: str = ADVANCED_MODEL + verifier_max_attempts: int = Field(default=3, ge=1) + verifier_confidence_threshold: int = Field(default=8, ge=1, le=10) class CodeExecutionConfig(BaseModel): @@ -212,6 +247,106 @@ def _build_automatic_function_calling_config( return AutomaticFunctionCallingConfig(disable=True) +class _LeakageVerification(BaseModel): + """Structured verdict from the independent temporal-leakage verifier.""" + + flagged_claims: list[str] = Field(default_factory=list) + filtered_text: str = "" + confidence: int = Field(ge=1, le=10) + clean: bool + + +def _build_leakage_verification_schema() -> dict[str, Any]: + """Strict JSON schema for the verifier's structured output. + + Field order matters: the model must extract/flag claims and produce + ``filtered_text`` before declaring ``confidence``/``clean``, so the + verdict follows the claim-level reasoning instead of preceding it. + """ + return { + "type": "object", + "properties": { + "flagged_claims": {"type": "array", "items": {"type": "string"}}, + "filtered_text": {"type": "string"}, + "confidence": {"type": "integer", "minimum": 1, "maximum": 10}, + "clean": {"type": "boolean"}, + }, + "required": ["flagged_claims", "filtered_text", "confidence", "clean"], + } + + +_LEAKAGE_VERIFIER_INSTRUCTION = """\ +You are an independent fact-checker verifying that a web search result contains \ +no information published on or after a given cutoff date. + +Extract every discrete factual claim relevant to the query. For each claim, \ +judge whether its *content* (the event itself, prices/figures referenced, \ +described developments) could only be known on or after the cutoff date. \ +Do NOT trust a source's claimed publish date, byline timestamp, or URL — \ +page metadata and timestamps are frequently updated after original \ +publication and are not reliable evidence of when the underlying facts \ +became known. Reason from the substance of the claim itself. + +Remove every claim that fails this test and produce `filtered_text`: the \ +original text with only the surviving, pre-cutoff claims. Report the removed \ +claims in `flagged_claims`. Set `confidence` (1-10) to how confident you are \ +that `filtered_text` now contains zero post-cutoff leakage. Set `clean` to \ +true only if you removed all identifiable violations.""" + + +async def _verify_no_leakage( + *, + text: str, + query: str, + cutoff_date: str, + verifier_model: str, + openai_base_url: str, + openai_api_key: str | None, +) -> _LeakageVerification: + """Judge a search result for post-cutoff claims via an independent verifier call. + + Uses a different model (by default) than the search call it's checking, + so the verifier does not share the same knowledge-attribution blind spot + that caused the leak in the first place. Never raises on a malformed + verifier response — a parse failure is treated as a non-clean verdict so + it consumes a retry attempt like any other rejection. + """ + import litellm # noqa: PLC0415 + from aieng.forecasting.methods.llm_processes._client import ( # noqa: PLC0415 + make_json_schema_response_format, + strip_markdown_fence, + ) + + model = verifier_model if verifier_model.startswith("openai/") else f"openai/{verifier_model}" + resp = await litellm.acompletion( + model=model, + api_base=openai_base_url, + api_key=openai_api_key, + messages=[ + {"role": "system", "content": _LEAKAGE_VERIFIER_INSTRUCTION}, + { + "role": "user", + "content": f"Original query: {query}\nCutoff date: {cutoff_date}\n\nSearch result to verify:\n{text}", + }, + ], + response_format=make_json_schema_response_format("LeakageVerification", _build_leakage_verification_schema()), + temperature=0.0, + max_tokens=2048, + timeout=60.0, + ) + raw = resp.choices[0].message.content or "{}" + try: + return _LeakageVerification.model_validate(json.loads(strip_markdown_fence(raw))) + except (json.JSONDecodeError, ValidationError): + logger.warning("Leakage verifier returned unparseable output; treating as non-clean: %r", raw[:200]) + return _LeakageVerification( + flagged_claims=["verifier response could not be parsed"], + filtered_text=text, + confidence=1, + clean=False, + ) + + def _build_search_tool( config: ContextRetrievalConfig, *, @@ -224,25 +359,25 @@ def _build_search_tool( the proxy with ``"tools": [{"googleSearch": {}}]`` so the model does server-side grounding and returns a synthesised answer plus source URLs extracted from ``choices[0].provider_specific_fields["grounding_metadata"]``. - """ - async def search_web(query: str, cutoff_date: str | None = None) -> str: - """Search the web and return a grounded summary with source URLs. + When a ``cutoff_date`` is supplied and ``config.enforce_cutoff`` is + ``True``, the raw result is passed through an independent leakage + verifier (:func:`_verify_no_leakage`) before being returned. On a flagged + result, the search is retried (up to ``config.verifier_max_attempts`` + times) with the previously flagged claims injected as explicit negative + feedback. If no attempt is verified clean, an explicit + ``[SEARCH_VERIFICATION_FAILED]`` sentinel is returned instead of + potentially-leaky content. + """ - Args: - query: What to search for. - cutoff_date: ISO date (YYYY-MM-DD). When provided, only include - information published strictly before this date. + def _format_result(content: str, sources: list[str]) -> str: + if sources: + content += "\n\nSources:\n" + "\n".join(sources[:5]) + return content - Returns - ------- - A grounded summary of search results, with source URLs appended. - """ + async def _do_search(user_content: str) -> tuple[str, list[str]]: import litellm # noqa: PLC0415 - user_content = query - if cutoff_date and config.enforce_cutoff: - user_content += f"\n\nOnly include and cite information published strictly before {cutoff_date}." search_model = config.search_model if not search_model.startswith("openai/"): search_model = f"openai/{search_model}" @@ -265,9 +400,88 @@ def _build_search_tool( sources: list[str] = [ uri for c in gm.get("groundingChunks", []) if (uri := (c.get("web") or {}).get("uri")) is not None ] - if sources: - content += "\n\nSources:\n" + "\n".join(sources[:5]) - return content + return content, sources + + async def search_web(query: str, cutoff_date: str | None = None, tool_context: ToolContext | None = None) -> str: + """Search the web and return a grounded summary with source URLs. + + Args: + query: What to search for. + cutoff_date: ISO date (YYYY-MM-DD). When provided, only include + information published strictly before this date. + + Returns + ------- + A grounded summary of search results, with source URLs appended. + When cutoff verification is enabled and cannot be satisfied + within the attempt budget, returns a ``[SEARCH_VERIFICATION_FAILED]`` + sentinel instead of unverified content. + + Notes + ----- + ``tool_context`` is not part of the LLM-visible tool schema — ADK + injects it automatically because of its type annotation. When + :class:`AgentPredictor` runs this tool, it has already seeded the + session with the current prediction's ``as_of`` date under + :data:`AS_OF_STATE_KEY`; that harness-controlled value is the + authoritative cutoff whenever present, since (unlike ``cutoff_date``) + the calling LLM cannot see, omit, or alter it. This closes a bypass + where the model simply didn't pass ``cutoff_date`` and both the soft + cutoff instruction and the verifier below were silently skipped. + """ + harness_as_of = tool_context.state.get(AS_OF_STATE_KEY) if tool_context is not None else None + if harness_as_of and cutoff_date and harness_as_of != cutoff_date: + logger.warning( + "search_web: cutoff_date=%r disagrees with harness as_of=%r; using the harness value.", + cutoff_date, + harness_as_of, + ) + effective_cutoff = harness_as_of or cutoff_date + + needs_verification = bool(effective_cutoff and config.enforce_cutoff) + if not needs_verification: + content, sources = await _do_search(query) + return _format_result(content, sources) + + negative_guidance = "" + for attempt in range(1, config.verifier_max_attempts + 1): + user_content = ( + query + f"\n\nOnly include and cite information published strictly before {effective_cutoff}." + ) + if negative_guidance: + user_content += f"\n\n{negative_guidance}" + content, sources = await _do_search(user_content) + verdict = await _verify_no_leakage( + text=content, + query=query, + cutoff_date=effective_cutoff, # type: ignore[arg-type] + verifier_model=config.verifier_model, + openai_base_url=openai_base_url, + openai_api_key=openai_api_key, + ) + logger.info( + "search_web verification attempt %d/%d: clean=%s confidence=%d flagged=%d", + attempt, + config.verifier_max_attempts, + verdict.clean, + verdict.confidence, + len(verdict.flagged_claims), + ) + if verdict.clean and verdict.confidence >= config.verifier_confidence_threshold: + return _format_result(verdict.filtered_text, sources) + logger.warning("search_web attempt %d flagged %d claim(s); retrying.", attempt, len(verdict.flagged_claims)) + negative_guidance = ( + f"Your previous search result may have included information published on or after " + f"{effective_cutoff}. Do not repeat or rely on these claims:\n- " + + "\n- ".join(verdict.flagged_claims or ["(unspecified — be more conservative)"]) + ) + + logger.error("search_web exhausted %d attempts without a verified clean result.", config.verifier_max_attempts) + return ( + f"[SEARCH_VERIFICATION_FAILED] Could not verify search results as free of information " + f"published on or after {effective_cutoff} after {config.verifier_max_attempts} attempts. " + "Treat this as no verified news context being available for this query." + ) return search_web diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__predictor.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__predictor.py.md index 6e613431..10baed0c 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__predictor.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__predictor.py.md @@ -34,7 +34,7 @@ from aieng.forecasting.evaluation.prediction import Prediction from aieng.forecasting.evaluation.predictor import Predictor from aieng.forecasting.evaluation.task import ForecastingTask from aieng.forecasting.methods.agentic.adk_runner import AdkTextRunner, AdkTextRunnerConfig -from aieng.forecasting.methods.agentic.agent_factory import AgentConfig, build_adk_agent +from aieng.forecasting.methods.agentic.agent_factory import AS_OF_STATE_KEY, AgentConfig, build_adk_agent from aieng.forecasting.methods.agentic.outputs import AgentForecastOutput from aieng.forecasting.methods.llm_processes._client import strip_markdown_fence, trace_url_for from google.adk.agents.base_agent import BaseAgent @@ -282,7 +282,11 @@ class AgentPredictor(Predictor): validation errors on the agent's JSON are not swallowed. """ prompt = self.prompt_builder(task=task, context=context) - output_str = _run_coroutine_sync(self._runner.run_text_async(prompt)) + # Seed the harness-controlled as_of into the ADK session before the run, + # so search_web can enforce it via ToolContext.state regardless of + # whether the LLM remembers to pass a matching cutoff_date argument. + initial_state = {AS_OF_STATE_KEY: str(context.as_of)[:10]} + output_str = _run_coroutine_sync(self._runner.run_text_async(prompt, initial_state=initial_state)) # Normalise: strip markdown fences before validation so any model can # be swapped in without breaking the parse layer. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__llm_processes___client.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__llm_processes___client.py.md index 877bd51c..c01345ce 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__llm_processes___client.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__llm_processes___client.py.md @@ -25,13 +25,14 @@ into a single response, which would defeat sample-based forecasting. from __future__ import annotations import asyncio +import contextlib import contextvars import json import logging import os import warnings from concurrent.futures import ThreadPoolExecutor -from typing import Any, Callable, TypeVar +from typing import Any, Callable, Iterator, TypeVar from pydantic import BaseModel, ValidationError @@ -96,6 +97,54 @@ def langfuse_observe(name: str) -> Callable[..., Any]: return _noop +class _NoopGeneration: + """Stand-in used when Langfuse is unavailable, so callers need no branching.""" + + def update(self, **kwargs: Any) -> None: + """Discard the update.""" + return + + +@contextlib.contextmanager +def langfuse_generation(*, name: str, model: str, input_messages: Any) -> Iterator[Any]: + """Emit a Langfuse ``generation`` around one LLM call. + + We create the generation explicitly rather than relying on LiteLLM's + ``langfuse_otel`` callback: that callback emits no generation at all when + the call runs inside an already-active Langfuse span, which is every LLMP + ``predict`` (they are wrapped in :func:`langfuse_observe`). The result was + that token usage — and therefore cost — never reached Langfuse for any + LLMP run. Creating it here works nested and keeps model, usage, and + payload under our control. + + Cost is deliberately not set: Langfuse derives it from ``usage_details`` + and its own per-model prices, which match the Vector proxy's published + rates. + + Yields a handle exposing ``update(**kwargs)`` — a no-op stand-in when + Langfuse is not installed or a generation cannot be started, so predictors + remain usable without the ``agentic``/``llm`` extras. + """ + manager = None + try: + from langfuse import get_client # noqa: PLC0415 + + manager = get_client().start_as_current_observation( + as_type="generation", + name=name, + model=model, + input=input_messages, + ) + except Exception: # pragma: no cover - depends on optional dependency + logger.debug("Langfuse generation unavailable; usage will not be traced.", exc_info=True) + + if manager is None: + yield _NoopGeneration() + return + with manager as generation: + yield generation + + def current_trace_info() -> tuple[str | None, str | None]: """Return ``(trace_id, trace_url)`` from the active Langfuse client, if any.""" try: @@ -304,17 +353,21 @@ async def _one_completion_async( # models that don't support them (e.g. temperature on some o-series). kwargs["drop_params"] = True - resp = await litellm.acompletion(**kwargs) - cost = float(getattr(resp, "_hidden_params", {}).get("response_cost") or 0.0) - usage = getattr(resp, "usage", None) - in_tok = int(getattr(usage, "prompt_tokens", 0) or 0) if usage is not None else 0 - out_tok = int(getattr(usage, "completion_tokens", 0) or 0) if usage is not None else 0 - # Log full usage so we can see thinking-token breakdown when available. - # The proxy may populate completion_tokens_details.reasoning_tokens. - if usage is not None: - logger.debug("LLM usage: %s", vars(usage) if hasattr(usage, "__dict__") else usage) - raw = resp.choices[0].message.content - content = strip_markdown_fence(raw) if raw else raw + # ``model`` is the bare name as configured, before any "openai/" prefixing + # above; that is what Langfuse's price table matches on. + with langfuse_generation(name="llm_completion", model=model, input_messages=messages) as generation: + resp = await litellm.acompletion(**kwargs) + cost = float(getattr(resp, "_hidden_params", {}).get("response_cost") or 0.0) + usage = getattr(resp, "usage", None) + in_tok = int(getattr(usage, "prompt_tokens", 0) or 0) if usage is not None else 0 + out_tok = int(getattr(usage, "completion_tokens", 0) or 0) if usage is not None else 0 + # Log full usage so we can see thinking-token breakdown when available. + # The proxy may populate completion_tokens_details.reasoning_tokens. + if usage is not None: + logger.debug("LLM usage: %s", vars(usage) if hasattr(usage, "__dict__") else usage) + raw = resp.choices[0].message.content + content = strip_markdown_fence(raw) if raw else raw + generation.update(output=content, usage_details={"input": in_tok, "output": out_tok}) return content, cost, in_tok, out_tok diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__README.md.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__README.md.md index c8b6e6a4..52ecf07e 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__README.md.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__README.md.md @@ -32,7 +32,7 @@ implementations/ `-- pyproject.toml # local workspace packaging ``` -YAML backtest and eval specs live under each use case in `specs/`. Each directory is independent; see its `README.md` for the walkthrough. +YAML backtest and eval specs live under each use case in `specs/`. Each directory is independent; see its `README.md` for the walkthrough. For the build-phase moves — onboarding data, standing up an experiment, customizing an agent, auditing a result — see [`guides/`](../guides/). Every domain use case (all except `getting_started`) also ships a `starter_agent/` module and a `99_starter_agent.ipynb` — a fresh, hackable **starter agent** that is the consistent "build your own" entry point for that use case (toggleable news search + code execution, two lightweight tool-usage skills, an interactive cell, and one scored forecast). diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__agent.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__agent.py.md index 500357a7..0ed0cf45 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__agent.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__agent.py.md @@ -105,7 +105,16 @@ labour market; market pricing of the upcoming decision (OIS, economist surveys); and macro shocks relevant to Canada (oil, exchange rate, US policy, trade). Ground every claim in the search results you actually retrieve. When a cutoff -date is specified, never report or speculate about events after it.\ +date is specified, never report or speculate about events after it. + +Before finalizing your summary, reason step by step: (1) for each candidate \ +fact, judge its actual recency from the substance of the result itself, \ +never from a source's claimed publish date or byline timestamp — those are \ +frequently stale or updated after original publication; (2) discard \ +anything you cannot confidently place before the cutoff date; (3) only then \ +write your summary. Do not supplement the search results with your own \ +background/training knowledge — if the results are insufficient, say so \ +explicitly rather than filling gaps from memory.\ """ diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__skills__research-playbook__SKILL.md.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__skills__research-playbook__SKILL.md.md index 96ad5034..d824d247 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__skills__research-playbook__SKILL.md.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__skills__research-playbook__SKILL.md.md @@ -21,6 +21,12 @@ Always pass `cutoff_date` equal to the `as_of` date in your payload. It is the temporal fence that keeps post-origin information out of a historical forecast. A forecast that "knew" what happened after `as_of` is not a forecast. +`search_web` runs an independent verifier on every result and returns +`[SEARCH_VERIFICATION_FAILED]` instead of content it couldn't confirm as +pre-cutoff. Treat that as no verified news for the query — proceed on your +other signals and say so, never filling the gap from your own background +knowledge. + ## How to search - **Search before you forecast, not after.** Gather context first, then reason. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__04_systematic_backtest_eval.ipynb.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__04_systematic_backtest_eval.ipynb.md index 003940ad..a3989b15 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__04_systematic_backtest_eval.ipynb.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__04_systematic_backtest_eval.ipynb.md @@ -14,8 +14,10 @@ This notebook simulates a rigorous production forecasting workflow: 3. Select the **top contender configurations** based solely on 2025 historical performance (no peeking at 2026). 4. Let the contenders compete in the **2026 Protected Arena** - (`energy_oil_eval.yaml`) during the geopolitical price shock — - measuring adaptive real-time responsiveness and calibration. + (`energy_oil_eval.yaml`) across the geopolitical price shock and its + aftermath — measuring adaptive real-time responsiveness and calibration. + The eval window runs through the most recent origin whose 21-business-day + horizon still resolves against cached data (see `scripts/fetch_wti.py`). The line-up spans three families behind one `Predictor` interface: **baselines** (Naive, AutoARIMA), **numerical ML** (LightGBM ± a leak-safe covariate panel), @@ -56,7 +58,7 @@ warnings.filterwarnings("ignore") # Set SMOKE_TEST = True to run a 2-origin, 1-sample version of the notebook # for fast local development and end-to-end CI testing. The full specs run # 51 backtest + 8 eval origins; smoke runs 2 + 2. -SMOKE_TEST = True +SMOKE_TEST = False # ── Models ──────────────────────────────────────────────────────────────────── # The project standardises on two Vector-proxy models. Every LLM and agent @@ -324,9 +326,11 @@ print(f"Saved {sum(n in _BASELINE_PREDICTORS for n in backtest_results)} backtes --- ## 5. 2026 Evaluation — Held-Out Test Period -We run every active predictor on **8 weekly origins in early 2026** -(`energy_oil_eval.yaml`) — a period of major geopolitical volatility not seen -during the 2025 backtest. +We run every active predictor on **18 weekly origins spanning Feb–Jun 2026** +(`energy_oil_eval.yaml`) — the major geopolitical volatility spike not seen +during the 2025 backtest, plus its aftermath. The window runs through the +most recent origin that still fully resolves against cached WTI data (the +21-business-day horizon needs data 21 business days past the origin). This evaluation serves two purposes: 1. **Measure out-of-sample robustness** — do the 2025 edges (statistical, @@ -402,55 +406,161 @@ print(df_scorecard.to_string()) ## Cell 16 (markdown) --- -## 7. Core Takeaways +## 7. Diagnostics — reading past the leaderboard -1. **Numerical methods beat the naive baseline** by extracting structure from the - price history — AutoARIMA via local autocorrelation, LightGBM via lagged - gradient-boosted quantiles. In stable regimes this translates to better CRPS. +The scorecard above is a single number per method. That hides *where* the score +comes from and *whether the ranking is even real*. The next cells decompose it +straight from the eval predictions — so they recompute on any rerun, smoke or +full: -2. **Covariates can sharpen LightGBM.** The `LightGBM + cov` variant adds a - leak-safe panel (Brent, natural gas, gasoline, gold, the USD index, the - USL/USO futures-curve contango proxy, and VIX). Comparing it to plain - `LightGBM` isolates how much the cross-market context is worth — the same - lesson that made covariates decisive in the S&P 500 study. +- **CRPS by horizon** — does a method win everywhere, or is its mean dominated by + one horizon? (For a short forecast, the 5-day calls are easy and nearly tied; + the ranking is usually decided by the longest horizon.) +- **Mean CRPS ± standard error** — with only a handful of origins, are the gaps + between methods bigger than the noise, or is the "winner" a coin flip? -3. **Tree models extrapolate poorly through regime shifts.** LightGBM forecasts - the price *level*, and gradient-boosted trees cannot predict outside the range - seen in training. When the 2026 shock pushes WTI to new levels, expect the - tree methods — like AutoARIMA — to lag and produce biased, under-confident - intervals. This is a structural limitation, not a tuning problem. +With the **smoke spec (2 origins → a few scored points)** expect wide error bars +and an unstable ranking. That is exactly why a surprising leaderboard here is not +yet evidence of anything — it is a pipeline check. -4. **LLM/agent methods bring a different prior.** The LLM-process forecasters and - the news-reading agent are run on both `gemini-3.1-flash-lite-preview` and - `gemini-3.5-flash`, so the scorecard shows both *method* and *model* effects — - and whether reading the news helps when the numerical methods are blindsided. +## Cell 17 (code) -5. **The `Predictor` abstraction makes the comparison clean.** The same harness, - scoring functions, covariate panel, and eval spec serve every family, and the - registry lets you switch any predictor on or off without touching the pipeline. +```python +from energy_oil_forecasting import viz +from energy_oil_forecasting.analysis import ( + build_price_frame, + eval_narrative_md, + extract_agent_rationales, + leaderboard_with_uncertainty, + per_horizon_crps, + predictions_to_frame, +) +from IPython.display import HTML, Markdown, display # noqa: A004 + + +# Explode every scored 2026 eval prediction into one tidy row per +# (predictor, origin, horizon): point, 80% interval, realised price, and CRPS. +# Everything in Sections 7–10 reads from this frame, so it all recomputes when +# you flip SMOKE_TEST off and rerun. +price_df = build_price_frame(data_service) +eval_frame = predictions_to_frame(eval_results, data_service) +eval_board = leaderboard_with_uncertainty(eval_frame) +ph_crps = per_horizon_crps(eval_frame) + +print("━" * 72) +print("MEAN CRPS BY PREDICTOR × HORIZON (lower = better; 'All' = overall mean):") +print("━" * 72) +print(ph_crps.round(2).to_string()) +``` + +## Cell 18 (code) + +```python +# Heatmap of the table above. Read it left-to-right: the short-horizon columns +# are usually a near-uniform green (everyone is right), and one long-horizon +# column carries the colour spread that sets the 'All' ranking. +viz.make_crps_heatmap(ph_crps) +``` + +## Cell 19 (code) + +```python +# Same leaderboard, now with a standard-error bar on each mean. If the bars of +# the top methods overlap, their ordering is not statistically distinguishable — +# the honest verdict when only a few origins have been scored. +viz.make_leaderboard_interval_chart(eval_board) +``` + +## Cell 20 (markdown) + +--- +## 8. What are the top methods actually forecasting? + +A CRPS number doesn't show *behaviour*. Below, each leading method's **median +forecast and 80% interval** are drawn against the realised WTI path at every +eval origin. This is where the leaderboard becomes legible — watch for who +tracks the move, who simply anchors to the last price, and whose intervals are +too narrow to cover the outcome when the market jumps. + +## Cell 21 (code) + +```python +# Plot the leaderboard's top methods, and always include the best LLM/agent +# method for contrast (so the chart compares families even when a baseline leads). +_leaders = list(eval_board.index[:3]) +_best_llm = next((p for p in eval_board.index if eval_board.loc[p, "family"] == "LLM / Agent"), None) +if _best_llm and _best_llm not in _leaders: + _leaders.append(_best_llm) +print(f"Showing: {', '.join(_leaders)}") +viz.make_eval_forecast_chart(eval_frame, price_df, _leaders) +``` + +## Cell 22 (markdown) + +--- +## 9. Reading the agent's reasoning + +The news-reading agent attaches a free-text **rationale** to every forecast, and +a link to the full **Langfuse trace**. These are pulled straight from the +prediction metadata. This is where a surprising score becomes interpretable: you +can read whether the agent actually saw the geopolitical risk, and *how* it +turned that into a price and an interval — including, often, an interval far too +narrow for a regime shift. + +## Cell 23 (code) + +```python +# One card per (agent, origin): the rationale, the per-horizon note, and a link +# to the full reasoning trace. Empty only if no LLM/agent predictor is enabled. +eval_rationales = extract_agent_rationales(eval_results) +display(HTML(viz.render_rationales_html(eval_rationales))) +``` + +## Cell 24 (markdown) + +--- +## 10. Takeaways — computed from this run + +The summary below is **generated from the eval results in memory, not +hard-coded**, so it always matches what actually ran: the real winner, whether +its lead clears the noise floor, the horizon that decided the ranking, the +best-performing family, and a calibration line. Flip `SMOKE_TEST` off, rerun, +and these takeaways update themselves with the full leaderboard. + +## Cell 25 (code) + +```python +display(Markdown(eval_narrative_md(eval_frame, smoke=SMOKE_TEST))) +``` + +## Cell 26 (markdown) --- -## 8. What stateless methods can't do +## 11. What stateless methods can't do -Every method here is calibrated (or prompted) once and never updated between -rounds. This is intentional — it creates a clean baseline — but it leaves a -systematic gap: +Sections 7–10 score and dissect this run on its own terms. But every method here +shares one structural limit, independent of who topped the leaderboard: it is +calibrated (or prompted) **once and never updated between rounds**. That is +intentional — it creates a clean baseline — but it leaves a systematic gap: -- **No error feedback.** If a method consistently produces intervals that are too - narrow in elevated-vol regimes, it keeps making the same mistake. There is no - mechanism to update calibration between rounds. +- **No error feedback.** If a method's intervals are consistently too narrow in + an elevated-vol regime (read the coverage line in Section 10, and the squashed + error bars in Section 8), it keeps making the same mistake. Nothing updates its + calibration between origins. -- **No strategy evolution.** Each prediction starts from the same prior (the same - fitted model, or the same prompt). Resolved outcomes disappear without +- **No strategy evolution.** Each prediction starts from the same prior — the + same fitted model, or the same prompt. Resolved outcomes disappear without influencing future forecasts. - **Context without memory.** Even the news agent re-reads the world each origin; - it does not accumulate what worked. + it does not accumulate what worked. The rationales in Section 9 are written + fresh every time, with no record of how the last one resolved. -→ **Notebook 5** introduces adaptive agents that study the 2025 backtest, -record systematic observations, and calibrate their strategies accordingly. At -inference time, each agent receives the live stateless estimate and decides how -to adjust it — applying what it learned from training. +→ **Notebook 5** introduces adaptive agents that study the 2025 backtest, record +systematic observations, and calibrate their strategies accordingly. At inference +time, each agent receives the live stateless estimate and decides how to adjust +it — applying what it learned from training. → **Notebook 6** evaluates whether any training approach actually improved -out-of-sample performance on the held-out 2026 data. +out-of-sample performance on the held-out 2026 data — measured against the +stateless baseline this notebook just established. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__06_protected_eval.ipynb.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__06_protected_eval.ipynb.md index 672e8a9f..fa126a84 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__06_protected_eval.ipynb.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__06_protected_eval.ipynb.md @@ -103,7 +103,7 @@ This has a concrete implication for this evaluation: fetch data via yfinance and reason from what it computed — not from memorized facts about 2025 WTI prices. -- The **evaluation period** (Feb–Mar 2026) is definitively post-cutoff. +- The **evaluation period** (Feb–Jun 2026) is definitively post-cutoff. During eval, the agent must rely entirely on: 1. Live Google Search (with `cutoff_date` enforcement per origin) 2. Code execution (for statistical analysis of fetched data) diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__99_starter_agent.ipynb.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__99_starter_agent.ipynb.md index 33691c76..ceb8021c 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__99_starter_agent.ipynb.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__99_starter_agent.ipynb.md @@ -8,13 +8,14 @@ kind: notebook **If you're not sure what to do next, continue from here.** -This notebook is a fresh, hackable agent for the WTI crude-oil use case — deliberately *not* wired into the numbered curriculum. It gives you our common building blocks behind simple toggles, so you can start building something of your own: +This notebook is a fresh, hackable agent for the WTI crude-oil use case — deliberately *not* wired into the numbered curriculum. An agent is a **persona** plus a **toolbelt**, and you assemble that toolbelt right here in the notebook from a menu of one-line tool factories: -- **optional news search** — bounded, cutoff-aware Google Search (proxy-only) -- **optional code execution** — an E2B Python sandbox -- **two lightweight skills** — *tool-usage playbooks* in `starter_agent/skills/` +- **`news_search()`** — bounded, cutoff-aware Google Search (proxy-only) +- **`arima_forecast()`** — an AutoARIMA statistical anchor the agent can call directly (no code-gen) +- **`code_sandbox()`** — an E2B Python sandbox for the agent to compute its own diagnostics +- each tool pulls in its own *playbook* skill from `starter_agent/skills/` -It does two things: lets you **talk to the agent** (open-ended, Track 2) and **score one real forecast** (Track 1). The live cells are gated by `RUN_AGENT` so a fresh `Run All` is safe and free; flip it to `True` to actually call the model. +The factories live in `starter_agent/tools.py` — open it to see how a tool is built, or add your own. It does two things: lets you **talk to the agent** (open-ended, Track 2) and **score one real forecast** (Track 1). The live cells are gated by `RUN_AGENT` so a fresh `Run All` is safe and free; flip it to `True` to actually call the model. ## Cell 2 (code) @@ -47,6 +48,7 @@ RUN_AGENT = False from energy_oil_forecasting.starter_agent import ( build_starter_agent_config, build_starter_agent_predictor, + tools, ) @@ -56,24 +58,31 @@ print("RUN_AGENT =", RUN_AGENT, "| model =", AGENT_MODEL) ## Cell 3 (markdown) --- -## 1. Meet your agent +## 1. Build your agent's toolbelt -`build_starter_agent_config` returns an `AgentConfig` with two toggles. The default turns **news search on** (proxy-only, no extra key) and **code execution off** (it needs `E2B_API_KEY` and is slower). Flip them and re-run — the loaded skills follow the enabled tools. +This is where you compose the agent. `build_starter_agent_config` takes a `tools=[...]` list — the toolbelt — and folds each tool onto the agent (its config, its skill, its instructions). **Comment a line to drop a tool; uncomment to add one**, then re-run. That's the whole model: an agent is a persona plus the tools you hand it. ## Cell 4 (code) ```python -config = build_starter_agent_config( - model=AGENT_MODEL, - enable_search=True, # ← cutoff-aware Google Search (proxy-only) - enable_code_exec=False, # ← E2B Python sandbox (needs E2B_API_KEY); try True! -) - -print("Agent:", config.name) -print("Search enabled: ", config.context_retrieval.enabled) -print("Code-exec enabled: ", config.code_execution.enabled) -print("Skills loaded: ", [p.name for p in config.skills_dirs]) -print("\n── System instruction (edit this in starter_agent/agent.py) ──\n") +# ── Your agent's toolbelt ────────────────────────────── +# Each factory returns one tool. Comment a line to drop it, uncomment to add it. +# See starter_agent/tools.py for how each is built — and to write your own. +toolbelt = [ + tools.news_search(), # cutoff-aware Google Search (proxy-only, no extra key) + tools.arima_forecast(), # AutoARIMA anchor — the agent calls a forecast directly, no code-gen + # tools.code_sandbox(), # E2B Python sandbox (needs E2B_API_KEY, slower) — uncomment to add +] + +config = build_starter_agent_config(model=AGENT_MODEL, tools=toolbelt) + +print("Agent: ", config.name) +print("Toolbelt:", [t.label for t in toolbelt]) +print(" search enabled: ", config.context_retrieval.enabled) +print(" forecast tool: ", bool(config.function_tools)) +print(" code-exec enabled:", config.code_execution.enabled) +print("Skills loaded: ", [p.name for p in config.skills_dirs]) +print("\n── System instruction (edit the persona in starter_agent/agent.py) ──\n") print(config.instruction[:1200], "...") ``` @@ -164,11 +173,11 @@ else: This agent is a starting point. Here are concrete next steps, easiest first — each is a small edit, then re-run the cells above. -1. **Flip code execution on.** Set `enable_code_exec=True` in §1 (needs `E2B_API_KEY`). The agent loads the `code-analysis-playbook` skill and can compute its own diagnostics before forecasting. Compare the rationale. +1. **Change the toolbelt.** In §1, uncomment `tools.code_sandbox()` (needs `E2B_API_KEY`) to let the agent compute its own diagnostics, or drop `arima_forecast()` and compare the rationale with and without a statistical anchor. Adding a tool automatically loads its playbook skill and its instructions. 2. **Edit the agent's personality.** Open `starter_agent/agent.py` and change `_build_starter_instruction()` — make it more cautious, more contrarian, focused on one driver. Re-run §1 to see the new instruction. -3. **Sharpen the skills.** The two files in `starter_agent/skills/` are short on purpose. Add your best queries to `research-playbook`, or a new diagnostic to `code-analysis-playbook`. The agent picks them up automatically. +3. **Sharpen the skills.** The files in `starter_agent/skills/` are short on purpose. Add your best queries to `research-playbook`, or a new diagnostic to `code-analysis-playbook`. The agent picks them up automatically. 4. **Change the question and the origin.** Try a different `QUESTION` in §2 and a different origin in §3. -5. **Add a tool.** Give the agent a conventional forecast tool as a statistical anchor — see `analyst_agent.build_wti_tool_config` for the `ForecastTool` pattern. +5. **Write your own tool.** Open `starter_agent/tools.py` and add a factory that returns a `ToolSpec` — point `arima_forecast()` at a different series, swap AutoARIMA for another predictor, or wrap a brand-new function tool. Then add it to the toolbelt in §1. 6. **Score it properly.** Run it across several origins with `backtest()` (see `04_systematic_backtest_eval.ipynb`) and compare CRPS against the baselines. Bigger ideas — an agent that *learns* a strategy (notebooks 05–06), news vs. no-news lift, live prospective forecasting — are in the use-case `README.md` and `planning-docs/roadmap.md`. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__adaptive_agent__agent.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__adaptive_agent__agent.py.md index 1041fba0..f9ffc196 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__adaptive_agent__agent.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__adaptive_agent__agent.py.md @@ -183,6 +183,10 @@ def _build_adaptive_analyst_instruction() -> str: "## Temporal discipline\n\n" "Every forecast is anchored to an `as_of` date. Never use information beyond " "that date — in web search, code analysis, or reasoning.\n\n" + "If `search_web` returns a result beginning with `[SEARCH_VERIFICATION_FAILED]`, " + "treat it as no verified news context for that query. Do not use your own " + "background knowledge to fill the gap — proceed on price history and other " + "available signals only, and note the gap in your reasoning.\n\n" "When fetching data inside `run_code`, always pass `end=as_of_date` to " "yfinance to enforce the temporal cutoff — for example:\n\n" "```python\nraw = ticker.history(start='2004-01-01', end='2026-02-16', " @@ -221,7 +225,16 @@ markdown summary (3-5 paragraphs) covering relevant aspects of: Ground your summary in the search results you actually retrieve. \ When a cutoff date is specified, do not report or speculate about events \ -that occurred after that date.\ +that occurred after that date. + +Before finalizing your summary, reason step by step: (1) for each candidate \ +fact, judge its actual recency from the substance of the result itself, \ +never from a source's claimed publish date or byline timestamp — those are \ +frequently stale or updated after original publication; (2) discard \ +anything you cannot confidently place before the cutoff date; (3) only then \ +write your summary. Do not supplement the search results with your own \ +background/training knowledge — if the results are insufficient, say so \ +explicitly rather than filling gaps from memory.\ """ diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__analysis.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__analysis.py.md index 0d07636d..6058a6d3 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__analysis.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__analysis.py.md @@ -166,9 +166,292 @@ def select_top_predictors( return [str(x) for x in leaderboard.head(n)["predictor_id"].tolist()] +# ── Per-prediction diagnostics ──────────────────────────────────────────────── +# The leaderboard collapses every forecast into a single mean CRPS. To understand +# *why* a method wins or loses we need the predictions un-aggregated: one row per +# (predictor, origin, horizon) carrying the point estimate, the 80% interval, the +# realised outcome, and the CRPS the harness assigned. Everything below builds on +# this tidy frame so the notebook charts and the written narrative read from the +# same numbers. + +# Map of predictor-name fragments → family label, checked in order. Used to group +# the leaderboard into baselines / numerical-ML / LLM-agent without hard-coding a +# per-predictor table (the registry can grow without touching this). +_FAMILY_RULES: list[tuple[tuple[str, ...], str]] = [ + (("naive", "last value"), "Baseline"), + (("arima", "lightgbm", "prophet"), "Numerical ML"), + (("llmp", "agent", "news", "llm"), "LLM / Agent"), +] + + +def predictor_family(name: str) -> str: + """Classify a predictor display name into a forecasting family.""" + low = name.lower() + for fragments, label in _FAMILY_RULES: + if any(frag in low for frag in fragments): + return label + return "Other" + + +def _qval(quantiles: dict[Any, float], q: float) -> float: + """Read a quantile value tolerating both float and string dict keys.""" + for key in (q, str(q), f"{q:.2f}"): + if key in quantiles: + return float(quantiles[key]) + return float("nan") + + +def _business_horizon(as_of: pd.Timestamp, forecast_date: pd.Timestamp) -> int: + """Trading-day distance between an information cutoff and a forecast date.""" + return max(len(pd.bdate_range(as_of.normalize(), forecast_date.normalize())) - 1, 0) + + +def predictions_to_frame( + results_by_predictor: dict[str, dict[str, BacktestResult]], + data_service: DataService, + *, + actuals_as_of: datetime | None = None, +) -> pd.DataFrame: + """Explode backtest results into one tidy row per scored prediction. + + Parameters + ---------- + results_by_predictor + ``{display_name: {task_id: BacktestResult}}`` — exactly the shape the + notebook holds in ``eval_results`` / ``backtest_results``. + data_service + Used to look up realised target values for error/coverage columns. + actuals_as_of + Cutoff for realised-value lookup; defaults to *now* so every horizon that + has already resolved is scored (see :func:`score_backtest_results`). + + Returns + ------- + pd.DataFrame + Columns: ``predictor``, ``family``, ``as_of``, ``forecast_date``, + ``horizon`` (trading days), ``point``, ``q10``/``q20``/``q50``/``q80``/ + ``q90``, ``actual``, ``crps``, ``abs_error``, ``signed_error``, + ``width80`` (80% interval width), and ``inside80`` (1.0/0.0/NaN). + """ + resolved_as_of = actuals_as_of or datetime.now(tz=timezone.utc).replace(tzinfo=None) + actual_cache: dict[str, dict[pd.Timestamp, float]] = {} + + def _actuals(series_id: str) -> dict[pd.Timestamp, float]: + if series_id not in actual_cache: + df = data_service.get_series(series_id, as_of=resolved_as_of) + actual_cache[series_id] = { + pd.Timestamp(row["timestamp"]).normalize(): float(row["value"]) for _, row in df.iterrows() + } + return actual_cache[series_id] + + rows: list[dict[str, Any]] = [] + for predictor_name, task_results in results_by_predictor.items(): + for result in task_results.values(): + actual_by_date = _actuals(result.spec.task.target_series_id) + for pred, score in zip(result.predictions, result.scores, strict=False): + if not isinstance(pred.payload, ContinuousForecast): + continue + as_of = pd.Timestamp(pred.as_of) + fdate = pd.Timestamp(pred.forecast_date).normalize() + q = pred.payload.quantiles + lo80, hi80 = _qval(q, 0.2), _qval(q, 0.8) + point = float(pred.payload.point_forecast) + actual = actual_by_date.get(fdate) + rows.append( + { + "predictor": predictor_name, + "family": predictor_family(predictor_name), + "as_of": as_of, + "forecast_date": fdate, + "horizon": _business_horizon(as_of, fdate), + "point": point, + "q10": _qval(q, 0.1), + "q20": lo80, + "q50": _qval(q, 0.5), + "q80": hi80, + "q90": _qval(q, 0.9), + "actual": actual, + "crps": float(score), + "abs_error": abs(point - actual) if actual is not None else float("nan"), + "signed_error": (actual - point) if actual is not None else float("nan"), + "width80": hi80 - lo80, + "inside80": float(lo80 <= actual <= hi80) if actual is not None else float("nan"), + } + ) + return pd.DataFrame(rows) + + +def per_horizon_crps(pred_frame: pd.DataFrame) -> pd.DataFrame: + """Pivot mean CRPS to a predictor × horizon matrix with an ``All`` column. + + Rows are sorted by overall mean CRPS (best first) so the table doubles as the + leaderboard and reveals which horizon decides the ranking. + """ + if pred_frame.empty: + return pd.DataFrame() + pivot = pred_frame.pivot_table(index="predictor", columns="horizon", values="crps", aggfunc="mean") + pivot.columns = [f"h={int(h)}d" for h in pivot.columns] + pivot["All"] = pred_frame.groupby("predictor")["crps"].mean() + return pivot.sort_values("All") + + +def leaderboard_with_uncertainty(pred_frame: pd.DataFrame) -> pd.DataFrame: + """Mean CRPS per predictor with a standard error, sorted best-first. + + The ``se`` column (sample standard deviation / √n) is the lens for the + "is this lead real or noise?" question: when the gap between two predictors + is small relative to their SEs — common with only a handful of scored + origins — the ranking is not statistically meaningful. + """ + if pred_frame.empty: + return pd.DataFrame() + grp = pred_frame.groupby("predictor")["crps"] + out = pd.DataFrame( + { + "mean_crps": grp.mean(), + "se": grp.std(ddof=1) / np.sqrt(grp.count()), + "n": grp.count().astype(int), + "family": pred_frame.groupby("predictor")["family"].first(), + } + ) + return out.sort_values("mean_crps") + + +def extract_agent_rationales(results_by_predictor: dict[str, dict[str, BacktestResult]]) -> pd.DataFrame: + """Pull free-text rationale and trace links from agent/LLM prediction metadata. + + Only predictions whose ``metadata`` carries a ``rationale`` (the analyst agent + and any LLM method that returns one) produce rows. The result is the raw + material for inspecting *what the model was thinking* origin by origin. + """ + rows: list[dict[str, Any]] = [] + for predictor_name, task_results in results_by_predictor.items(): + for result in task_results.values(): + for pred in result.predictions: + meta = pred.metadata or {} + if "rationale" not in meta and "horizon_rationale" not in meta: + continue + rows.append( + { + "predictor": predictor_name, + "as_of": pd.Timestamp(pred.as_of), + "horizon": _business_horizon(pd.Timestamp(pred.as_of), pd.Timestamp(pred.forecast_date)), + "point": float(pred.payload.point_forecast) + if isinstance(pred.payload, ContinuousForecast) + else float("nan"), + "rationale": str(meta.get("rationale", "")).strip(), + "horizon_rationale": str(meta.get("horizon_rationale", "")).strip(), + "trace_url": meta.get("langfuse_trace_url", ""), + } + ) + return pd.DataFrame(rows) + + +def eval_narrative_md( + pred_frame: pd.DataFrame, + *, + smoke: bool = False, + period_label: str = "2026 evaluation", +) -> str: + """Generate the eval takeaways as Markdown computed from the results. + + Replaces hard-coded prose so the narrative always matches the run — including + after switching from smoke to the full suite. Reports the actual winner, the + gap to the runner-up relative to the noise floor, the decisive horizon, the + best family, and a calibration line, plus an explicit small-sample caveat. + """ + if pred_frame.empty: + return "_No scored predictions available to summarise._" + + board = leaderboard_with_uncertainty(pred_frame) + horizons = sorted(pred_frame["horizon"].unique()) + n_origins = pred_frame["as_of"].nunique() + n_points = len(pred_frame) + + winner = board.index[0] + win_crps, win_se = board.loc[winner, "mean_crps"], board.loc[winner, "se"] + lines: list[str] = [] + + # 1. Winner + significance vs runner-up. + if len(board) > 1: + runner = board.index[1] + gap = board.loc[runner, "mean_crps"] - win_crps + noise = float(np.nan_to_num(win_se) + np.nan_to_num(board.loc[runner, "se"])) + significant = noise > 0 and gap > noise + verdict = ( + "a gap larger than the combined standard error — a real edge over this window" + if significant + else "**well within the combined standard error**, so the ranking here is not statistically distinguishable from noise" + ) + lines.append( + f"1. **{winner}** has the best mean CRPS ({win_crps:.2f}) on the {period_label}, " + f"ahead of **{runner}** ({board.loc[runner, 'mean_crps']:.2f}) by {gap:.2f} — {verdict}." + ) + else: + lines.append(f"1. **{winner}** scored {win_crps:.2f} mean CRPS on the {period_label}.") + + # 2. Where the ranking is decided (per-horizon spread). + if len(horizons) > 1: + by_h = pred_frame.groupby("horizon")["crps"] + spread = (by_h.max() - by_h.min()).sort_values(ascending=False) + decisive_h = int(spread.index[0]) + easy_h = int(spread.index[-1]) + lines.append( + f"2. The leaderboard is **decided at h={decisive_h}d**, where CRPS ranges {by_h.min()[decisive_h]:.1f}–" + f"{by_h.max()[decisive_h]:.1f} across methods; at the short h={easy_h}d horizon the methods are nearly " + f"tied (range {by_h.min()[easy_h]:.1f}–{by_h.max()[easy_h]:.1f}). A handful of long-horizon points " + f"drives the whole ranking." + ) + + # 3. Best family — does the agentic/LLM bet pay off here? + fam = pred_frame.groupby("family")["crps"].mean().sort_values() + best_fam = fam.index[0] + fam_str = ", ".join(f"{f} {v:.2f}" for f, v in fam.items()) + lines.append(f"3. **By family** (mean CRPS): {fam_str}. Best family this window: **{best_fam}**.") + + # 4. Calibration — is the winner's 80% interval honest? + cov = pred_frame.dropna(subset=["inside80"]).groupby("predictor")["inside80"].mean() * 100 + if winner in cov.index: + n_win = int(board.loc[winner, "n"]) + lines.append( + f"4. **Calibration:** {winner}'s 80% interval covered {cov[winner]:.0f}% of outcomes " + f"(target 80%) over its {n_win} scored point(s). With this few, coverage this far from " + f"target is itself a small-sample artefact, not necessarily mis-calibration." + ) + + # 5. Sample-size caveat — the honest health warning. + caveat = ( + f"⚠️ **Smoke run:** only {n_origins} origin(s) / {n_points} scored points. Treat the ranking as a " + f"pipeline check, not evidence — rerun the full suite before drawing conclusions." + if smoke or n_origins <= 2 + else f"Based on {n_origins} origins / {n_points} scored points." + ) + lines.append(f"5. {caveat}") + return "\n".join(lines) + + +def build_price_frame(data_service: DataService, *, as_of: datetime | None = None) -> pd.DataFrame: + """Return the target price series as a ``price``-column DataFrame for plotting.""" + resolved_as_of = as_of or datetime.now(tz=timezone.utc).replace(tzinfo=None) + series = data_service.get_series("wti_crude_oil_price", as_of=resolved_as_of) + frame = pd.DataFrame( + {"price": series["value"].astype(float).to_numpy()}, + index=pd.to_datetime(series["timestamp"]), + ) + frame.index.name = "date" + return frame.sort_index() + + __all__ = [ "backtest_results_to_frame", + "build_price_frame", "compute_brier_score", + "eval_narrative_md", + "extract_agent_rationales", + "leaderboard_with_uncertainty", + "per_horizon_crps", + "predictions_to_frame", + "predictor_family", "rolling_coverage_pct", "score_backtest_results", "select_top_predictors", diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval.yaml.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval.yaml.md index 0c0a4371..80e1c480 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval.yaml.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval.yaml.md @@ -5,8 +5,12 @@ kind: yaml ```yaml # Energy Oil Eval Spec — 2026 Prospective Competition # -# Runs on 8 weekly origins from Feb 2, 2026 to Mar 23, 2026. -# Covers the high-volatility Persian Gulf geopolitical price shock period. +# Runs on 18 weekly origins from Feb 2, 2026 to Jun 1, 2026. +# Covers the high-volatility Persian Gulf geopolitical price shock and its +# aftermath. The end date is set to the latest origin whose longest horizon +# (21 business days) still resolves against available data — keep it at +# most 21 business days behind the most recent cached WTI price (see +# scripts/fetch_wti.py) so every origin fully resolves. # Target is WTI Crude Oil price (yfinance ticker: CL=F). # Horizons: 5, 10, 21 business days. @@ -14,8 +18,9 @@ spec_id: energy_oil_eval description: >- Prospective/out-of-sample evaluation period in 2026 for daily WTI crude oil. - Evaluates selected contender models on 8 weekly origins during the early 2026 - geopolitical price shock to measure adaptive real-time forecasting performance. + Evaluates selected contender models on 18 weekly origins from the early 2026 + geopolitical price shock through its aftermath, to measure adaptive + real-time forecasting performance. tasks: - task_id: wti_oil_price_forecast @@ -27,7 +32,7 @@ tasks: projected 5, 10, and 21 trading days ahead. start: "2026-02-02" -end: "2026-03-23" +end: "2026-06-01" stride: 5 warmup: 250 ``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval_smoke.yaml.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval_smoke.yaml.md index 80597220..f4fdfe8c 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval_smoke.yaml.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval_smoke.yaml.md @@ -9,7 +9,7 @@ kind: yaml # arena cheaply during development and end-to-end testing. # Use by setting SMOKE_TEST = True in the notebook setup cell. # -# Origin count : 2 (vs. 8 in the full eval) +# Origin count : 2 (vs. 18 in the full eval) # Warmup : 250 trading days (~1 year) of historical prices spec_id: energy_oil_eval_smoke diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent____init__.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent____init__.py.md index 2e5a2e84..080dd814 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent____init__.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent____init__.py.md @@ -5,10 +5,12 @@ kind: python ```python """WTI starter agent — a fresh, hackable template for your own exploration. -Exports the toggle-driven :class:`AgentConfig` factory and the predictor -convenience factory. See ``99_starter_agent.ipynb`` and ``agent.py``. +Exports the toolbelt-driven :class:`AgentConfig` factory, the predictor +convenience factory, and the :mod:`tools` module of per-tool factories you +compose in the notebook. See ``99_starter_agent.ipynb`` and ``agent.py``. """ +from energy_oil_forecasting.starter_agent import tools from energy_oil_forecasting.starter_agent.agent import ( build_starter_agent_config, build_starter_agent_predictor, @@ -18,5 +20,6 @@ from energy_oil_forecasting.starter_agent.agent import ( __all__ = [ "build_starter_agent_config", "build_starter_agent_predictor", + "tools", ] ``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__agent.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__agent.py.md index c7076021..c625c4d9 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__agent.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__agent.py.md @@ -32,7 +32,7 @@ from __future__ import annotations import json from pathlib import Path -from typing import Any, Callable +from typing import Any, Callable, Sequence from aieng.forecasting.data.context import ForecastContext from aieng.forecasting.evaluation.task import ForecastingTask @@ -51,13 +51,13 @@ from aieng.forecasting.models import LITE_MODEL # Reuse the existing WTI prompt builder + history compression — these serialise # the task/context into the agent's JSON payload and are not worth duplicating. from energy_oil_forecasting.analyst_agent import WtiPriceForecastPromptBuilder +from energy_oil_forecasting.starter_agent.tools import ToolSpec, news_search -# Skills live next to this module. +# Skills live next to this module. The forecasting contract is always loaded; +# each tool loads its own playbook via its ToolSpec (see tools.py). _SKILLS_ROOT = Path(__file__).parent / "skills" _FORECASTING_SKILL = _SKILLS_ROOT / "forecasting" -_RESEARCH_SKILL = _SKILLS_ROOT / "research-playbook" -_CODE_ANALYSIS_SKILL = _SKILLS_ROOT / "code-analysis-playbook" # --------------------------------------------------------------------------- @@ -93,19 +93,6 @@ def _build_starter_instruction() -> str: _STARTER_INSTRUCTION = _build_starter_instruction() -_CONTEXT_RETRIEVAL_INSTRUCTION = """\ -You are an oil-market intelligence specialist with web search. - -Return a concise structured markdown summary (3-5 paragraphs) covering, as the -query warrants: WTI/Brent price level and trend; OPEC+ supply decisions; -geopolitical risk in the Persian Gulf and key shipping lanes; US SPR / energy -policy; notable supply-disruption signals; and published analyst price targets. - -Ground every claim in the search results you actually retrieve. When a cutoff -date is specified, never report or speculate about events after it.\ -""" - - # --------------------------------------------------------------------------- # Config factory # --------------------------------------------------------------------------- @@ -113,59 +100,75 @@ date is specified, never report or speculate about events after it.\ def build_starter_agent_config( model: str = LITE_MODEL, - search_model: str = LITE_MODEL, *, - enable_search: bool = True, - enable_code_exec: bool = False, + tools: Sequence[ToolSpec] = (), ) -> AgentConfig: - """Build the WTI starter :class:`AgentConfig`. + """Build the WTI starter :class:`AgentConfig` from a toolbelt. + + An agent is a persona plus a list of tools. The persona is fixed here (edit + ``_build_starter_instruction``); the *toolbelt* is what you compose in the + notebook — a list of :class:`~energy_oil_forecasting.starter_agent.tools.ToolSpec` + from the factories in :mod:`energy_oil_forecasting.starter_agent.tools`:: + + from energy_oil_forecasting.starter_agent import tools, build_starter_agent_config + + config = build_starter_agent_config( + model=AGENT_MODEL, + tools=[tools.news_search(), tools.arima_forecast()], + ) + + Each spec lands in a different ``AgentConfig`` field; this function folds the + list, routing every fragment to the right place — search sub-agent, code + sandbox, function tools — and loading each tool's playbook skill and prompt + supplement. Adding or removing a tool is one line in the notebook. Parameters ---------- model : str Model for the analyst agent (default: lite). Pass the advanced model (``"gemini-3.5-flash"``) for higher-quality runs. - search_model : str - Model for the bounded web-search sub-tool. - enable_search : bool, default=True - Wire a cutoff-aware ``search_web`` tool and load the - ``research-playbook`` skill. Proxy-only — no extra API key. - enable_code_exec : bool, default=False - Wire an E2B Python sandbox and load the ``code-analysis-playbook`` - skill. Needs ``E2B_API_KEY`` and is slower, so it is off by default — - flip it on to let the agent compute its own diagnostics. + tools : Sequence[ToolSpec], default=() + The agent's toolbelt. Build entries with the factories in ``tools.py`` + (``news_search()``, ``code_sandbox()``, ``arima_forecast()``), or write + your own factory that returns a ``ToolSpec``. Returns ------- AgentConfig """ - # Every attached skill is loaded on demand: ADK injects each skill's name + - # description into the system prompt, and the agent reads the full SKILL.md - # only when relevant — so toggling a tool just adds its skill, no persona edits. + # The forecasting contract is always loaded. Each tool's own playbook is + # loaded on demand: ADK injects each skill's name + description into the + # system prompt, and the agent reads the full SKILL.md only when relevant — + # so adding a tool just adds its skill, no persona edits. skills_dirs: list[Path] = [_FORECASTING_SKILL] - if enable_search: - skills_dirs.append(_RESEARCH_SKILL) - if enable_code_exec: - skills_dirs.append(_CODE_ANALYSIS_SKILL) - - context_retrieval = ( - ContextRetrievalConfig( - enabled=True, - instruction=_CONTEXT_RETRIEVAL_INSTRUCTION, - search_model=search_model, - ) - if enable_search - else ContextRetrievalConfig() - ) + instruction = _STARTER_INSTRUCTION + context_retrieval = ContextRetrievalConfig() + code_execution = CodeExecutionConfig() + function_tools: list[Any] = [] + max_output_tokens: int | None = None + + for spec in tools: + if spec.skill_dir is not None: + skills_dirs.append(spec.skill_dir) + if spec.instruction_supplement: + instruction += spec.instruction_supplement + if spec.context_retrieval is not None: + context_retrieval = spec.context_retrieval + if spec.code_execution is not None: + code_execution = spec.code_execution + if spec.function_tool is not None: + function_tools.append(spec.function_tool) + if spec.max_output_tokens is not None: + max_output_tokens = max(max_output_tokens or 0, spec.max_output_tokens) return AgentConfig( name="wti_starter_agent", model=model, - instruction=_STARTER_INSTRUCTION, - # 16k headroom: enough for a complete run_code script + structured output. - max_output_tokens=16_384 if enable_code_exec else None, + instruction=instruction, + max_output_tokens=max_output_tokens, context_retrieval=context_retrieval, - code_execution=CodeExecutionConfig(enabled=enable_code_exec), + code_execution=code_execution, + function_tools=function_tools, skills_dirs=skills_dirs, ) @@ -228,6 +231,7 @@ def build_starter_agent_predictor(config: AgentConfig) -> AgentPredictor: def __getattr__(name: str) -> Any: """Expose ``root_agent`` lazily for schema-free interactive use via ``adk web``.""" if name == "root_agent": - return build_adk_agent(build_starter_agent_config()) + # Interactive default: news search on (proxy-only, no extra key). + return build_adk_agent(build_starter_agent_config(tools=[news_search()])) raise AttributeError(f"module {__name__!r} has no attribute {name!r}") ``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__skills__research-playbook__SKILL.md.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__skills__research-playbook__SKILL.md.md index a6969936..f43e96ae 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__skills__research-playbook__SKILL.md.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__skills__research-playbook__SKILL.md.md @@ -21,6 +21,12 @@ Always pass `cutoff_date` equal to the `as_of` date in your payload. It is the temporal fence that keeps post-origin information out of a historical forecast. A forecast that "knew" what happened after `as_of` is not a forecast. +`search_web` runs an independent verifier on every result and returns +`[SEARCH_VERIFICATION_FAILED]` instead of content it couldn't confirm as +pre-cutoff. Treat that as no verified news for the query — proceed on your +other signals and say so, never filling the gap from your own background +knowledge. + ## How to search - **Search before you forecast, not after.** Gather context first, then reason. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__tools.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__tools.py.md new file mode 100644 index 00000000..1cd0911d --- /dev/null +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__tools.py.md @@ -0,0 +1,237 @@ +# Source: implementations/energy_oil_forecasting/starter_agent/tools.py + +kind: python + +```python +"""The starter agent's toolbelt — one factory per tool, composed in the notebook. + +An agent is a *persona* plus a *list of tools*. This module makes that list the +thing you edit: each function here returns a :class:`ToolSpec` describing one +capability, and you assemble an agent by handing a list of them to +:func:`~energy_oil_forecasting.starter_agent.agent.build_starter_agent_config`:: + + from energy_oil_forecasting.starter_agent import tools, build_starter_agent_config + + config = build_starter_agent_config( + model=AGENT_MODEL, + tools=[ + tools.news_search(), # cutoff-aware Google Search (proxy-only) + tools.arima_forecast(), # AutoARIMA statistical anchor — no code-gen + # tools.code_sandbox(), # E2B Python sandbox (needs E2B_API_KEY) + ], + ) + +Each tool lands in a *different* field of the underlying ``AgentConfig`` (search +is a sub-agent, code execution is a sandbox capability, the forecast is a +plain function tool). A :class:`ToolSpec` carries everything a tool needs — the +config fragment it fills, its playbook skill, and any prompt supplement — so the +config factory can route it without the notebook ever touching ADK plumbing. + +To add your own tool, write a factory that returns a ``ToolSpec``. Point it at a +different series, swap AutoARIMA for another predictor, or wrap a brand-new +function tool — the notebook composition and the config fold both keep working. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from aieng.forecasting.data import DataService +from aieng.forecasting.methods.agentic import ForecastTool +from aieng.forecasting.methods.agentic.agent_factory import ( + CodeExecutionConfig, + ContextRetrievalConfig, +) +from aieng.forecasting.methods.numerical.darts_arima import DartsAutoARIMAPredictor +from aieng.forecasting.models import LITE_MODEL +from energy_oil_forecasting.data import WTI_SERIES_ID, build_wti_service + + +# Skills live next to this module; each tool loads its own playbook. +_SKILLS_ROOT = Path(__file__).parent / "skills" +_RESEARCH_SKILL = _SKILLS_ROOT / "research-playbook" +_CODE_ANALYSIS_SKILL = _SKILLS_ROOT / "code-analysis-playbook" + + +# --------------------------------------------------------------------------- +# ToolSpec — the seam between the notebook's toolbelt and AgentConfig +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ToolSpec: + """One item on the agent's toolbelt. + + A tool is more than a callable: it may need a config fragment, a skill that + teaches the agent how to use it, and a line of instruction. This descriptor + bundles all of that so + :func:`~energy_oil_forecasting.starter_agent.agent.build_starter_agent_config` + can fold a list of specs onto a single ``AgentConfig`` — routing each field + to the right place — without the caller knowing the internals. + + Attributes + ---------- + label : str + Short human-readable name, shown when the config is printed. + context_retrieval : ContextRetrievalConfig or None + Web-search sub-agent config, if this tool provides search. + code_execution : CodeExecutionConfig or None + E2B sandbox config, if this tool provides code execution. + function_tool : Any or None + A ready-to-register ADK function tool (e.g. from + ``ForecastTool.as_function_tool()``). + skill_dir : Path or None + A playbook skill directory to load alongside the tool. + instruction_supplement : str + Text appended to the agent's system instruction when this tool is on. + max_output_tokens : int or None + A per-tool floor on the response budget (e.g. code execution needs + headroom for a full script). The config takes the max across all tools. + """ + + label: str + context_retrieval: ContextRetrievalConfig | None = None + code_execution: CodeExecutionConfig | None = None + function_tool: Any | None = None + skill_dir: Path | None = None + instruction_supplement: str = "" + max_output_tokens: int | None = None + + +# --------------------------------------------------------------------------- +# Tool-specific prompt text +# --------------------------------------------------------------------------- + + +_CONTEXT_RETRIEVAL_INSTRUCTION = """\ +You are an oil-market intelligence specialist with web search. + +Return a concise structured markdown summary (3-5 paragraphs) covering, as the +query warrants: WTI/Brent price level and trend; OPEC+ supply decisions; +geopolitical risk in the Persian Gulf and key shipping lanes; US SPR / energy +policy; notable supply-disruption signals; and published analyst price targets. + +Ground every claim in the search results you actually retrieve. When a cutoff +date is specified, never report or speculate about events after it. + +Before finalizing your summary, reason step by step: (1) for each candidate \ +fact, judge its actual recency from the substance of the result itself, \ +never from a source's claimed publish date or byline timestamp — those are \ +frequently stale or updated after original publication; (2) discard \ +anything you cannot confidently place before the cutoff date; (3) only then \ +write your summary. Do not supplement the search results with your own \ +background/training knowledge — if the results are insufficient, say so \ +explicitly rather than filling gaps from memory.\ +""" + + +def _forecast_tool_supplement(series_id: str, frequency: str) -> str: + """Instruction appended when the statistical forecast tool is attached. + + Kept in the factory (not hard-coded) so a tool built for a different series + or frequency describes itself correctly to the agent. + """ + return f""" + +## Statistical forecast tool + +You have access to `run_forecast`, a conventional statistical baseline +(AutoARIMA) you can call directly. Unlike open-ended code, this tool has a fixed, +auditable interface and returns a structured forecast you can reason from. + +Call it ONCE before producing your forecast, with: +- `series_id`: "{series_id}" +- `cutoff_date`: the `as_of` date from the payload (YYYY-MM-DD). This is the + information cutoff — the model uses only data on or before it. +- `horizons`: the `horizons` list from the payload. +- `frequency`: "{frequency}" (the business calendar the series trades on). + +The tool returns JSON with point forecasts and 80%/90% prediction intervals per +horizon. Treat it as a disciplined statistical anchor: combine it with any +market context you have. You may adjust away from the baseline when fundamentals +or geopolitical risk justify it — document your reasoning in the `rationale` +fields.\ +""" + + +# --------------------------------------------------------------------------- +# Tool factories — each returns one ToolSpec +# --------------------------------------------------------------------------- + + +def news_search(*, search_model: str = LITE_MODEL) -> ToolSpec: + """Build a cutoff-aware Google Search tool, run by a bounded sub-agent (proxy-only). + + Wires a ``search_web`` tool and loads the ``research-playbook`` skill. No + extra API key — everything routes through the Vector proxy. + + Parameters + ---------- + search_model : str + Model for the web-search sub-agent. Defaults to the lite model. + """ + return ToolSpec( + label="news_search", + context_retrieval=ContextRetrievalConfig( + enabled=True, + instruction=_CONTEXT_RETRIEVAL_INSTRUCTION, + search_model=search_model, + ), + skill_dir=_RESEARCH_SKILL, + ) + + +def code_sandbox() -> ToolSpec: + """Build an E2B Python sandbox for the agent to compute its own diagnostics. + + Wires the code-execution capability and loads the ``code-analysis-playbook`` + skill. Needs ``E2B_API_KEY`` and is slower than the other tools, so it is + off by default — add it to the toolbelt to turn it on. + """ + return ToolSpec( + label="code_sandbox", + code_execution=CodeExecutionConfig(enabled=True), + skill_dir=_CODE_ANALYSIS_SKILL, + # 16k headroom: enough for a complete run_code script + structured output. + max_output_tokens=16_384, + ) + + +def arima_forecast( + *, + series_id: str = WTI_SERIES_ID, + frequency: str = "B", + num_samples: int = 200, + data_service: DataService | None = None, +) -> ToolSpec: + """Build a conventional statistical anchor: AutoARIMA behind a `run_forecast` tool. + + Lets the agent invoke a statistical forecast *directly* — a rigid, auditable + interface — instead of writing forecasting code. In contrast to + :func:`code_sandbox` (open-ended), this trades flexibility for control and + reproducibility. The tool reads series data server-side; it never enters the + LLM context. + + Parameters + ---------- + series_id : str + Series the agent should forecast. Defaults to the WTI target. + frequency : str + Business calendar passed to the predictor (``"B"`` for WTI). + num_samples : int + Monte Carlo sample count for AutoARIMA. Kept modest to bound latency. + data_service : DataService or None + Pre-populated data service. When ``None``, a cache-backed WTI service is + built. Pass one to point the tool at your own series (or to avoid a data + fetch in tests). + """ + service = data_service if data_service is not None else build_wti_service() + tool = ForecastTool(service, predictor=DartsAutoARIMAPredictor(num_samples=num_samples)) + return ToolSpec( + label="arima_forecast", + function_tool=tool.as_function_tool(), + instruction_supplement=_forecast_tool_supplement(series_id, frequency), + ) +``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__viz.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__viz.py.md index e8693676..ddc06844 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__viz.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__viz.py.md @@ -1220,4 +1220,274 @@ def prob_bar(val: float, width: int = 10) -> str: def conf_bar(conf: str) -> str: """Map confidence label to emoji indicator.""" return {"high": "🟢", "medium": "🟡", "low": "🔴"}.get(conf.lower(), "⚪") + + +# ── NB4 eval-diagnostic charts ──────────────────────────────────────────────── +# These read the tidy per-prediction frame from ``analysis.predictions_to_frame`` +# (one row per predictor × origin × horizon) and answer three questions the bare +# leaderboard can't: *where* the ranking is decided (heatmap), whether a lead is +# real or noise (leaderboard with error bars), and *what* the methods actually +# forecast vs reality (trajectory chart). + +# Qualitative palette for an arbitrary, growing predictor set. Stable per call: +# colours are assigned by sorted predictor name so a method keeps its colour +# across the heatmap, leaderboard, and trajectory charts within one notebook run. +_PREDICTOR_PALETTE = [ + "#1f77b4", + "#ff7f0e", + "#2ca02c", + "#d62728", + "#9467bd", + "#8c564b", + "#e377c2", + "#7f7f7f", + "#bcbd22", + "#17becf", + "#393b79", + "#b5651d", +] + + +def predictor_colors(predictors: list[str]) -> dict[str, str]: + """Assign a stable colour to each predictor name.""" + return {name: _PREDICTOR_PALETTE[i % len(_PREDICTOR_PALETTE)] for i, name in enumerate(predictors)} + + +def make_crps_heatmap(per_horizon_df: pd.DataFrame) -> go.Figure: + """Predictor × horizon mean-CRPS heatmap (lower = better, sorted best-first). + + Expects the output of ``analysis.per_horizon_crps`` — horizon columns plus a + final ``All`` column. Reveals which horizon decides the ranking: typically the + short horizons are a wash and one long horizon dominates the mean. + """ + df = per_horizon_df.copy() + # Best predictor on top: reverse so plotly's bottom-up y-axis shows it first. + df = df.iloc[::-1] + z = df.to_numpy(dtype=float) + fig = go.Figure( + go.Heatmap( + z=z, + x=list(df.columns), + y=list(df.index), + colorscale="RdYlGn_r", + colorbar={"title": "CRPS"}, + text=[[f"{v:.2f}" if np.isfinite(v) else "" for v in row] for row in z], + texttemplate="%{text}", + textfont={"size": 12}, + hovertemplate="%{y}
%{x}: %{z:.3f}", + ) + ) + fig.update_layout( + title={"text": "Mean CRPS by Predictor × Horizon (lower = better)", "font": {"size": 16}}, + xaxis={"title": "Horizon", "side": "top"}, + yaxis={"title": ""}, + template="plotly_white", + width=720, + height=40 * len(df) + 160, + margin={"t": 90, "b": 40, "l": 230, "r": 40}, + ) + # Visually separate the "All" summary column. + if "All" in per_horizon_df.columns: + fig.add_vline(x=len(per_horizon_df.columns) - 1.5, line={"color": "#333333", "width": 1.5}) + return fig + + +def make_leaderboard_interval_chart(board_df: pd.DataFrame) -> go.Figure: + """Mean CRPS ± standard error per predictor, exposing whether a lead is noise. + + Expects ``analysis.leaderboard_with_uncertainty``. When the error bars of the + top methods overlap heavily, the ranking is not statistically meaningful — the + honest read on a short eval window. + """ + df = board_df.iloc[::-1] # best at top of the bottom-up axis + fam_colors = {"Baseline": "#7f7f7f", "Numerical ML": "#1f77b4", "LLM / Agent": "#2ca02c", "Other": "#b5651d"} + colors = [fam_colors.get(f, "#b5651d") for f in df["family"]] + fig = go.Figure( + go.Scatter( + x=df["mean_crps"], + y=df.index, + mode="markers", + marker={"size": 11, "color": colors}, + error_x={"type": "data", "array": df["se"].fillna(0.0), "thickness": 1.6, "width": 6, "color": "#888888"}, + hovertemplate="%{y}
CRPS %{x:.3f} ± %{error_x.array:.3f}", + showlegend=False, + ) + ) + best = float(df["mean_crps"].min()) + fig.add_vline( + x=best, + line={"color": "#31a354", "dash": "dot", "width": 1.5}, + annotation_text=" best", + annotation_position="top", + annotation_font={"size": 11, "color": "#31a354"}, + ) + fig.update_layout( + title={"text": "Eval Leaderboard — Mean CRPS ± 1 SE (overlap ⇒ tied)", "font": {"size": 16}}, + xaxis={"title": "Mean CRPS (lower = better)", "showgrid": True, "gridcolor": "#f0f0f0"}, + yaxis={"title": ""}, + template="plotly_white", + width=760, + height=34 * len(df) + 150, + margin={"t": 70, "b": 50, "l": 230, "r": 40}, + ) + return fig + + +def make_eval_forecast_chart( + pred_frame: pd.DataFrame, + price_df: pd.DataFrame, + predictors: list[str], + *, + history_window: int = 25, +) -> go.Figure: + """Per-origin trajectory chart: each method's median + 80% band vs reality. + + One column per forecast origin. Shows the pre-origin price history, the + realised price path, and — for each selected predictor — point forecasts at + each horizon with 80% interval error bars. This is the "what are the top + methods actually doing" view: you can see who tracks the move, who lags, and + whose intervals are too tight. + """ + origins = sorted(pred_frame["as_of"].unique()) + colors = predictor_colors(predictors) + + titles = [] + for o_raw in origins: + o = pd.Timestamp(o_raw) + rows = price_df[price_df.index >= o.normalize()] + spot = f"${float(rows.iloc[0]['price']):.0f}" if not rows.empty else "" + titles.append(f"{o.strftime('%b %d, %Y')} WTI {spot}") + + fig = psp.make_subplots( + rows=1, cols=len(origins), subplot_titles=titles, shared_yaxes=True, horizontal_spacing=0.03 + ) + + for col, origin_raw in enumerate(origins, start=1): + origin = pd.Timestamp(origin_raw) + show_legend = col == 1 + sub = pred_frame[pred_frame["as_of"] == origin] + last_fdate = pd.Timestamp(sub["forecast_date"].max()) + + # Pre-origin history + realised future path. + hist = price_df[price_df.index <= origin.normalize()].iloc[-history_window:] + future = price_df[(price_df.index > origin.normalize()) & (price_df.index <= last_fdate)] + fig.add_trace( + go.Scatter( + x=hist.index.tolist(), + y=hist["price"].tolist(), + mode="lines", + line={"color": CLR_HISTORY, "width": 1.5}, + name="WTI history", + showlegend=show_legend, + legendgroup="hist", + ), + row=1, + col=col, + ) + fig.add_trace( + go.Scatter( + x=future.index.tolist(), + y=future["price"].tolist(), + mode="lines+markers", + line={"color": CLR_ACTUAL, "width": 2.5}, + marker={"size": 5}, + name="Realised price", + showlegend=show_legend, + legendgroup="actual", + ), + row=1, + col=col, + ) + + # Each predictor's median + 80% interval at every horizon. + for name in predictors: + pr = sub[sub["predictor"] == name].sort_values("forecast_date") + if pr.empty: + continue + err_hi = (pr["q80"] - pr["point"]).clip(lower=0).fillna(0.0) + err_lo = (pr["point"] - pr["q20"]).clip(lower=0).fillna(0.0) + fig.add_trace( + go.Scatter( + x=pr["forecast_date"].tolist(), + y=pr["point"].tolist(), + mode="lines+markers", + line={"color": colors[name], "width": 1.4, "dash": "dot"}, + marker={"size": 8, "symbol": "diamond"}, + error_y={ + "type": "data", + "symmetric": False, + "array": err_hi.tolist(), + "arrayminus": err_lo.tolist(), + "color": colors[name], + "thickness": 1.4, + "width": 4, + }, + name=name, + showlegend=show_legend, + legendgroup=name, + ), + row=1, + col=col, + ) + + fig.add_vline( + x=origin.timestamp() * 1000, line={"color": "#aaaaaa", "dash": "dash", "width": 1}, row=1, col=col + ) + + fig.update_layout( + title={"text": "Eval Forecasts vs Reality — Median + 80% Interval by Origin", "font": {"size": 16}}, + template="plotly_white", + width=max(420 * len(origins), 720), + height=480, + margin={"t": 80, "b": 110, "l": 60, "r": 20}, + legend={"orientation": "h", "y": -0.18, "x": 0.0, "xanchor": "left", "font": {"size": 11}}, + ) + fig.update_xaxes(showgrid=True, gridcolor="#f0f0f0", tickfont={"size": 10}) + fig.update_yaxes(showgrid=True, gridcolor="#f0f0f0", tickfont={"size": 11}) + return fig + + +def render_rationales_html(rationale_df: pd.DataFrame, *, max_chars: int = 700) -> str: + """Render agent/LLM rationales as readable HTML cards with trace links. + + Expects ``analysis.extract_agent_rationales``. One card per (predictor, + origin), showing the overall rationale, the per-horizon note, and a link to + the Langfuse trace so the full agent reasoning is one click away. + """ + if rationale_df.empty: + return "

No agent/LLM rationales found in this run's metadata.

" + + def _clip(text: str) -> str: + text = (text or "").strip() + return text if len(text) <= max_chars else text[:max_chars].rsplit(" ", 1)[0] + " …" + + # One representative card per (predictor, origin) — the rationale is shared + # across horizons, so dedupe to the first row of each group. + seen: set[tuple[str, str]] = set() + cards: list[str] = [] + for _, r in rationale_df.sort_values(["predictor", "as_of"]).iterrows(): + key = (r["predictor"], str(pd.Timestamp(r["as_of"]).date())) + if key in seen: + continue + seen.add(key) + link = ( + f"🔗 Langfuse trace" + if r.get("trace_url") + else "" + ) + horizon_note = ( + f"
Horizon note: {_clip(r['horizon_rationale'])}
" + if r.get("horizon_rationale") + else "" + ) + cards.append( + f"
" + f"
" + f"{r['predictor']}" + f"{key[1]}   point ${r['point']:.1f}   {link}
" + f"
{_clip(r['rationale'])}
" + f"{horizon_note}
" + ) + return f"
{''.join(cards)}
" ``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__agent.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__agent.py.md index 139e8d5a..cf88644e 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__agent.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__agent.py.md @@ -141,7 +141,16 @@ commodity and input costs (grains, energy, fertiliser); supply-chain and weather disruptions; and the CAD exchange rate. Ground every claim in the search results you actually retrieve. When a cutoff -date is specified, never report or speculate about events after it.\ +date is specified, never report or speculate about events after it. + +Before finalizing your summary, reason step by step: (1) for each candidate \ +fact, judge its actual recency from the substance of the result itself, \ +never from a source's claimed publish date or byline timestamp — those are \ +frequently stale or updated after original publication; (2) discard \ +anything you cannot confidently place before the cutoff date; (3) only then \ +write your summary. Do not supplement the search results with your own \ +background/training knowledge — if the results are insufficient, say so \ +explicitly rather than filling gaps from memory.\ """ diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__skills__research-playbook__SKILL.md.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__skills__research-playbook__SKILL.md.md index 467eb3ca..2c9e2463 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__skills__research-playbook__SKILL.md.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__skills__research-playbook__SKILL.md.md @@ -21,6 +21,12 @@ Always pass `cutoff_date` equal to the `as_of` date in your payload. It is the temporal fence that keeps post-origin information out of a historical forecast. A forecast that "knew" what happened after `as_of` is not a forecast. +`search_web` runs an independent verifier on every result and returns +`[SEARCH_VERIFICATION_FAILED]` instead of content it couldn't confirm as +pre-cutoff. Treat that as no verified news for the query — proceed on your +other signals and say so, never filling the gap from your own background +knowledge. + ## How to search - **Search before you forecast, not after.** Gather context first, then reason. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__agent.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__agent.py.md index 03cc28e0..9819c41d 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__agent.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__agent.py.md @@ -155,7 +155,16 @@ the VIX and credit spreads; earnings-season tone; and major geopolitical or policy shocks. Ground every claim in the search results you actually retrieve. When a cutoff -date is specified, never report or speculate about events after it.\ +date is specified, never report or speculate about events after it. + +Before finalizing your summary, reason step by step: (1) for each candidate \ +fact, judge its actual recency from the substance of the result itself, \ +never from a source's claimed publish date or byline timestamp — those are \ +frequently stale or updated after original publication; (2) discard \ +anything you cannot confidently place before the cutoff date; (3) only then \ +write your summary. Do not supplement the search results with your own \ +background/training knowledge — if the results are insufficient, say so \ +explicitly rather than filling gaps from memory.\ """ diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__skills__research-playbook__SKILL.md.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__skills__research-playbook__SKILL.md.md index dce71dea..359dd966 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__skills__research-playbook__SKILL.md.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__skills__research-playbook__SKILL.md.md @@ -21,6 +21,12 @@ Always pass `cutoff_date` equal to the `as_of` date in your payload. It is the temporal fence that keeps post-origin information out of a historical forecast. A forecast that "knew" what happened after `as_of` is not a forecast. +`search_web` runs an independent verifier on every result and returns +`[SEARCH_VERIFICATION_FAILED]` instead of content it couldn't confirm as +pre-cutoff. Treat that as no verified news for the query — proceed on your +other signals and say so, never filling the gap from your own background +knowledge. + ## How to search - **Search before you forecast, not after.** Gather context first, then reason. diff --git a/implementations/getting_started/concierge_agent/context/catalog.yaml b/implementations/getting_started/concierge_agent/context/catalog.yaml index 9a6f8110..38083728 100644 --- a/implementations/getting_started/concierge_agent/context/catalog.yaml +++ b/implementations/getting_started/concierge_agent/context/catalog.yaml @@ -1,9 +1,9 @@ source_url: https://github.com/VectorInstitute/agentic-forecasting -git_ref: 0ac6b3098bdb529c08f2445895d82490de2404fd +git_ref: 1ca028d0df744fae68e47af998223d3347b92b1d branch: main -built_at: '2026-06-30T15:09:29+00:00' -ingest_source: /home/coder/agentic-forecasting -entry_count: 196 +built_at: '2026-08-20T16:47:29+00:00' +ingest_source: /home/akore/vscodeprojects/agentic-forecasting +entry_count: 197 entries: - path: AGENTS.md kind: markdown @@ -22,7 +22,7 @@ entries: - Model selection - Code quality (not on commit) - Test philosophy - chars: 6702 + chars: 6845 artifact: artifacts/AGENTS.md.md - path: README.md kind: markdown @@ -31,6 +31,7 @@ entries: symbols: [] sections: - Agentic Forecasting + - Contents - What's here - Two ways to use a forecaster - Reference implementations @@ -46,7 +47,7 @@ entries: - Extending the foundation - Code quality - Documentation - chars: 14796 + chars: 14063 artifact: artifacts/README.md.md - path: aieng-forecasting/aieng/forecasting/__init__.py kind: python @@ -497,7 +498,7 @@ entries: - AdkTextRunnerConfig - AdkTextRunner sections: [] - chars: 15228 + chars: 16225 artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__adk_runner.py.md - path: aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py kind: python @@ -507,10 +508,11 @@ entries: - _LiteLLMNoiseFilter - ContextRetrievalConfig - CodeExecutionConfig + - _LeakageVerification - AgentConfig - build_adk_agent sections: [] - chars: 24986 + chars: 35486 artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md - path: aieng-forecasting/aieng/forecasting/methods/agentic/curriculum.py kind: python @@ -555,7 +557,7 @@ entries: - ForecastPromptBuilder - AgentPredictor sections: [] - chars: 14369 + chars: 14715 artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__predictor.py.md - path: aieng-forecasting/aieng/forecasting/methods/baselines/__init__.py kind: python @@ -623,6 +625,8 @@ entries: symbols: - bootstrap_litellm - langfuse_observe + - _NoopGeneration + - langfuse_generation - current_trace_info - trace_url_for - set_current_trace_name @@ -631,7 +635,7 @@ entries: - sample_n_async - run_async sections: [] - chars: 18293 + chars: 20531 artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__llm_processes___client.py.md - path: aieng-forecasting/aieng/forecasting/methods/llm_processes/base.py kind: python @@ -801,7 +805,7 @@ entries: - Directory layout - Relationship to `aieng-forecasting` - Adding a new use case - chars: 3778 + chars: 3921 artifact: artifacts/implementations__README.md.md - path: implementations/__init__.py kind: python @@ -925,7 +929,7 @@ entries: - build_boc_news_config - build_boc_agent_predictor sections: [] - chars: 16347 + chars: 17929 artifact: artifacts/implementations__boc_rate_decisions__analyst_agent__agent.py.md - path: implementations/boc_rate_decisions/data.py kind: python @@ -1199,7 +1203,7 @@ entries: - _StarterForecastPromptBuilder - build_starter_agent_predictor sections: [] - chars: 9637 + chars: 10211 artifact: artifacts/implementations__boc_rate_decisions__starter_agent__agent.py.md - path: implementations/boc_rate_decisions/starter_agent/skills/code-analysis-playbook/SKILL.md kind: markdown @@ -1238,7 +1242,7 @@ entries: - How to search - Domain focus (edit this for your use case) - Room to grow - chars: 1947 + chars: 2250 artifact: artifacts/implementations__boc_rate_decisions__starter_agent__skills__research-playbook__SKILL.md.md - path: implementations/energy_oil_forecasting/01_wti_case_study.ipynb kind: notebook @@ -1281,7 +1285,7 @@ entries: sections: - "WTI Crude Oil Price Forecasting \u2014 Stateless Methods: Systematic Backtest\ \ (Notebook 4 of 7)" - chars: 18735 + chars: 22648 artifact: artifacts/implementations__energy_oil_forecasting__04_systematic_backtest_eval.ipynb.md - path: implementations/energy_oil_forecasting/05_adaptive_agent_training.ipynb kind: notebook @@ -1321,7 +1325,7 @@ entries: symbols: [] sections: - "WTI Crude Oil \u2014 Your Starter Agent" - chars: 7573 + chars: 8586 artifact: artifacts/implementations__energy_oil_forecasting__99_starter_agent.ipynb.md - path: implementations/energy_oil_forecasting/README.md kind: markdown @@ -1374,7 +1378,7 @@ entries: - build_wti_adaptive_config - build_wti_adaptive_predictor sections: [] - chars: 19884 + chars: 20792 artifact: artifacts/implementations__energy_oil_forecasting__adaptive_agent__agent.py.md - path: implementations/energy_oil_forecasting/adaptive_agent/curriculum/snapshot_utils.py kind: python @@ -1564,14 +1568,28 @@ entries: - backtest_results_to_frame - trajectory_mae_table - select_top_predictors + - predictor_family + - predictions_to_frame + - per_horizon_crps + - leaderboard_with_uncertainty + - extract_agent_rationales + - eval_narrative_md + - build_price_frame - backtest_results_to_frame + - build_price_frame - compute_brier_score + - eval_narrative_md + - extract_agent_rationales + - leaderboard_with_uncertainty + - per_horizon_crps + - predictions_to_frame + - predictor_family - rolling_coverage_pct - score_backtest_results - select_top_predictors - trajectory_mae_table sections: [] - chars: 6984 + chars: 19850 artifact: artifacts/implementations__energy_oil_forecasting__analysis.py.md - path: implementations/energy_oil_forecasting/analyst_agent/__init__.py kind: python @@ -1603,7 +1621,7 @@ entries: - build_wti_tool_config - build_wti_agent_predictor sections: [] - chars: 22599 + chars: 27592 artifact: artifacts/implementations__energy_oil_forecasting__analyst_agent__agent.py.md - path: implementations/energy_oil_forecasting/analyst_agent/skills/statistical-analysis/SKILL.md kind: markdown @@ -1787,11 +1805,15 @@ entries: symbols: [] sections: - "Energy Oil Eval Spec \u2014 2026 Prospective Competition" - - '# Runs on 8 weekly origins from Feb 2, 2026 to Mar 23, 2026.' - - Covers the high-volatility Persian Gulf geopolitical price shock period. + - '# Runs on 18 weekly origins from Feb 2, 2026 to Jun 1, 2026.' + - Covers the high-volatility Persian Gulf geopolitical price shock and its + - aftermath. The end date is set to the latest origin whose longest horizon + - "(21 business days) still resolves against available data \u2014 keep it at" + - most 21 business days behind the most recent cached WTI price (see + - scripts/fetch_wti.py) so every origin fully resolves. - 'Target is WTI Crude Oil price (yfinance ticker: CL=F).' - 'Horizons: 5, 10, 21 business days.' - chars: 1019 + chars: 1316 artifact: artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval.yaml.md - path: implementations/energy_oil_forecasting/specs/energy_oil_eval_smoke.yaml kind: yaml @@ -1803,9 +1825,9 @@ entries: - '# Two-origin subset of energy_oil_eval.yaml for running the 2026 protected' - arena cheaply during development and end-to-end testing. - Use by setting SMOKE_TEST = True in the notebook setup cell. - - '# Origin count : 2 (vs. 8 in the full eval)' + - '# Origin count : 2 (vs. 18 in the full eval)' - 'Warmup : 250 trading days (~1 year) of historical prices' - chars: 1084 + chars: 1085 artifact: artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval_smoke.yaml.md - path: implementations/energy_oil_forecasting/specs/energy_oil_smoke.yaml kind: yaml @@ -1828,8 +1850,9 @@ entries: symbols: - build_starter_agent_config - build_starter_agent_predictor + - tools sections: [] - chars: 542 + chars: 688 artifact: artifacts/implementations__energy_oil_forecasting__starter_agent____init__.py.md - path: implementations/energy_oil_forecasting/starter_agent/agent.py kind: python @@ -1840,7 +1863,7 @@ entries: - _StarterForecastPromptBuilder - build_starter_agent_predictor sections: [] - chars: 9265 + chars: 9841 artifact: artifacts/implementations__energy_oil_forecasting__starter_agent__agent.py.md - path: implementations/energy_oil_forecasting/starter_agent/skills/code-analysis-playbook/SKILL.md kind: markdown @@ -1880,8 +1903,21 @@ entries: - How to search - Domain focus (edit this for your use case) - Room to grow - chars: 1830 + chars: 2133 artifact: artifacts/implementations__energy_oil_forecasting__starter_agent__skills__research-playbook__SKILL.md.md +- path: implementations/energy_oil_forecasting/starter_agent/tools.py + kind: python + domain: impl.energy_oil_forecasting + summary: "The starter agent's toolbelt \u2014 one factory per tool, composed in\ + \ the notebook." + symbols: + - ToolSpec + - news_search + - code_sandbox + - arima_forecast + sections: [] + chars: 9826 + artifact: artifacts/implementations__energy_oil_forecasting__starter_agent__tools.py.md - path: implementations/energy_oil_forecasting/tasks.py kind: python domain: impl.energy_oil_forecasting @@ -1923,8 +1959,13 @@ entries: - verdict_label - prob_bar - conf_bar + - predictor_colors + - make_crps_heatmap + - make_leaderboard_interval_chart + - make_eval_forecast_chart + - render_rationales_html sections: [] - chars: 43163 + chars: 54100 artifact: artifacts/implementations__energy_oil_forecasting__viz.py.md - path: implementations/food_price_forecasting/01_food_data_exploration.ipynb kind: notebook @@ -2202,7 +2243,7 @@ entries: - _StarterForecastPromptBuilder - build_starter_agent_predictor sections: [] - chars: 10665 + chars: 11239 artifact: artifacts/implementations__food_price_forecasting__starter_agent__agent.py.md - path: implementations/food_price_forecasting/starter_agent/skills/code-analysis-playbook/SKILL.md kind: markdown @@ -2242,7 +2283,7 @@ entries: - How to search - Domain focus (edit this for your use case) - Room to grow - chars: 1871 + chars: 2174 artifact: artifacts/implementations__food_price_forecasting__starter_agent__skills__research-playbook__SKILL.md.md - path: implementations/getting_started/00_environment_check.ipynb kind: notebook @@ -2645,7 +2686,7 @@ entries: - build_sp500_multivariate_service - sp500_logret_series_id sections: [] - chars: 31632 + chars: 32091 artifact: artifacts/implementations__sp500_forecasting__data.py.md - path: implementations/sp500_forecasting/leaderboard.py kind: python @@ -2799,7 +2840,7 @@ entries: - _StarterForecastPromptBuilder - build_starter_agent_predictor sections: [] - chars: 11869 + chars: 12443 artifact: artifacts/implementations__sp500_forecasting__starter_agent__agent.py.md - path: implementations/sp500_forecasting/starter_agent/skills/code-analysis-playbook/SKILL.md kind: markdown @@ -2839,7 +2880,7 @@ entries: - How to search - Domain focus (edit this for your use case) - Room to grow - chars: 1860 + chars: 2163 artifact: artifacts/implementations__sp500_forecasting__starter_agent__skills__research-playbook__SKILL.md.md - path: planning-docs/roadmap.md kind: markdown @@ -2901,12 +2942,13 @@ entries: - path: scripts/fetch_fred.py kind: python domain: scripts - summary: Populate the local FRED cache with series used by the CFPR experiment. + summary: Populate the local FRED cache with series used by the food-price and S&P + 500 experiments. symbols: - build_data_service - main sections: [] - chars: 6020 + chars: 8603 artifact: artifacts/scripts__fetch_fred.py.md - path: scripts/fetch_sp500_market.py kind: python diff --git a/implementations/getting_started/concierge_agent/skills/repo-navigation/references/catalog-summary.yaml b/implementations/getting_started/concierge_agent/skills/repo-navigation/references/catalog-summary.yaml index 14913f1b..11b04760 100644 --- a/implementations/getting_started/concierge_agent/skills/repo-navigation/references/catalog-summary.yaml +++ b/implementations/getting_started/concierge_agent/skills/repo-navigation/references/catalog-summary.yaml @@ -1,9 +1,9 @@ # Concierge catalog summary (regenerated by scripts/build_concierge_context.py) source_url: https://github.com/VectorInstitute/agentic-forecasting branch: main -built_at: '2026-06-30T15:09:29+00:00' -git_ref: 0ac6b3098bdb529c08f2445895d82490de2404fd -entry_count: 196 +built_at: '2026-08-20T16:47:29+00:00' +git_ref: 1ca028d0df744fae68e47af998223d3347b92b1d +entry_count: 197 domains: docs: 4 core.root: 3 @@ -14,7 +14,7 @@ domains: impl.README.md: 1 impl.__init__.py: 1 impl.boc_rate_decisions: 27 - impl.energy_oil_forecasting: 45 + impl.energy_oil_forecasting: 46 impl.food_price_forecasting: 21 impl.getting_started: 16 impl.sp500_forecasting: 19 From 04507bf406cc5524c3d9523fc4835f29462fe3ea Mon Sep 17 00:00:00 2001 From: a-kore Date: Mon, 24 Aug 2026 16:29:55 -0400 Subject: [PATCH 2/3] fix agent+llmp langfuse tracing --- .../aieng/forecasting/langfuse_tracing.py | 30 +++------ .../methods/agentic/agent_factory.py | 14 +++-- .../methods/llm_processes/_client.py | 63 +++++++++++-------- .../methods/agentic/test_agent_factory.py | 10 +-- .../methods/llm_processes/test__client.py | 4 +- .../forecasting/test_langfuse_tracing.py | 48 ++++++-------- ...aieng__forecasting__langfuse_tracing.py.md | 30 +++------ ...ing__methods__agentic__agent_factory.py.md | 14 +++-- ...ing__methods__llm_processes___client.py.md | 63 +++++++++++-------- .../concierge_agent/context/catalog.yaml | 10 +-- .../references/catalog-summary.yaml | 4 +- 11 files changed, 141 insertions(+), 149 deletions(-) diff --git a/aieng-forecasting/aieng/forecasting/langfuse_tracing.py b/aieng-forecasting/aieng/forecasting/langfuse_tracing.py index 0a94e5db..74258f12 100644 --- a/aieng-forecasting/aieng/forecasting/langfuse_tracing.py +++ b/aieng-forecasting/aieng/forecasting/langfuse_tracing.py @@ -24,12 +24,11 @@ def _langfuse_credentials_present() -> bool: class _LangfuseTracingBootstrap: - """Registers LiteLLM + ADK exporters at most once per process.""" + """Registers the Langfuse client and ADK instrumentation once per process.""" - __slots__ = ("_google_adk_instrumented", "_langfuse_client_initialized", "_litellm_instrumented") + __slots__ = ("_google_adk_instrumented", "_langfuse_client_initialized") def __init__(self) -> None: - self._litellm_instrumented = False self._google_adk_instrumented = False self._langfuse_client_initialized = False @@ -46,7 +45,6 @@ def init(self) -> None: # this, ADK spans are emitted into a no-op provider and never reach Langfuse. self._ensure_langfuse_client() - self._register_litellm_langfuse_otel() self._instrument_google_adk() def _ensure_langfuse_client(self) -> None: @@ -64,21 +62,6 @@ def _ensure_langfuse_client(self) -> None: return self._langfuse_client_initialized = True - def _register_litellm_langfuse_otel(self) -> None: - """Register LiteLLM Langfuse callback.""" - if self._litellm_instrumented: - return - try: - import litellm # noqa: PLC0415 - except ImportError: - logger.debug("litellm not installed; skipping LiteLLM Langfuse callback.") - return - - existing = list(getattr(litellm, "callbacks", None) or []) - if "langfuse_otel" not in existing: - litellm.callbacks = [*existing, "langfuse_otel"] - self._litellm_instrumented = True - def _instrument_google_adk(self) -> None: """Instrument Google ADK.""" if self._google_adk_instrumented: @@ -120,11 +103,14 @@ def init_langfuse_tracing() -> None: ``TracerProvider`` receives Langfuse's span processor. This is required for ADK spans emitted via ``openinference-instrumentation-google-adk`` to reach Langfuse. - 2. Appends ``"langfuse_otel"`` to ``litellm.callbacks`` once (if - ``litellm`` is importable). - 3. Runs ``GoogleADKInstrumentor().instrument()`` once (if + 2. Runs ``GoogleADKInstrumentor().instrument()`` once (if ``openinference-instrumentation-google-adk`` is importable). + LiteLLM's ``langfuse_otel`` callback is deliberately not registered: it is + unusable against the Langfuse v4 SDK and stamps a zero ``llm.cost.total`` + on the active span, which suppresses Langfuse's own cost calculation. See + ``planning-docs/litellm-langfuse-compat.md``. + Set ``LANGFUSE_HOST`` or ``LANGFUSE_BASE_URL`` for non-default regions. For short-lived processes, call ``langfuse.get_client().flush()`` before exit so pending spans are exported. diff --git a/aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py b/aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py index 5866997d..891e0a3d 100644 --- a/aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py +++ b/aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py @@ -669,12 +669,16 @@ def build_adk_agent( if isinstance(model, str) and config.openai_base_url: from google.adk.models.lite_llm import LiteLlm # noqa: PLC0415 - # Prefix with "openai/" so LiteLLM uses the OpenAI-compatible path. - # LiteLLM strips the prefix before sending, so the proxy receives the - # bare model name. - litellm_model = model if model.startswith("openai/") else f"openai/{model}" + # Route via LiteLLM's OpenAI-compatible path with ``custom_llm_provider`` + # rather than an ``openai/`` model prefix. ADK stamps ``LlmRequest.model`` + # from this name and OpenInference reports it to Langfuse, which matches + # its per-model price table on the bare name. A prefixed name matches + # nothing, so the generation is logged at zero cost. Both forms route + # identically. See planning-docs/litellm-langfuse-compat.md. + bare_model = model[len("openai/") :] if model.startswith("openai/") else model model = LiteLlm( - model=litellm_model, + model=bare_model, + custom_llm_provider="openai", api_base=config.openai_base_url, api_key=config.openai_api_key, ) diff --git a/aieng-forecasting/aieng/forecasting/methods/llm_processes/_client.py b/aieng-forecasting/aieng/forecasting/methods/llm_processes/_client.py index 6343f0a0..e0c5d4e9 100644 --- a/aieng-forecasting/aieng/forecasting/methods/llm_processes/_client.py +++ b/aieng-forecasting/aieng/forecasting/methods/llm_processes/_client.py @@ -24,7 +24,6 @@ import contextvars import json import logging -import os import warnings from concurrent.futures import ThreadPoolExecutor from typing import Any, Callable, Iterator, TypeVar @@ -40,21 +39,23 @@ def bootstrap_litellm() -> None: - """One-time wiring of LiteLLM callbacks. + """Suppress LiteLLM and OpenTelemetry logging noise, once per process. Lazy and idempotent so non-LLM predictors do not require Langfuse env vars. - The Langfuse OTEL callback is registered only when ``LANGFUSE_PUBLIC_KEY`` - is set in the environment. + + LiteLLM's ``langfuse_otel`` callback is deliberately not registered. It is + unusable against the Langfuse v4 SDK this repo depends on, and it stamps + ``llm.cost.total`` on the active span from LiteLLM's own ``response_cost``, + which is ``0`` for every proxy-routed model. Langfuse honours a supplied + cost instead of deriving one from usage, so the callback forced agent-path + generations to $0. Instead, :func:`langfuse_generation` creates LLM-process + generations directly and OpenInference covers the agent path, so both price + correctly from ``usage_details``. See + ``planning-docs/litellm-langfuse-compat.md``. """ global _BOOTSTRAP_DONE # noqa: PLW0603 if _BOOTSTRAP_DONE: return - import litellm # noqa: PLC0415 - - if os.environ.get("LANGFUSE_PUBLIC_KEY"): - existing = list(getattr(litellm, "callbacks", []) or []) - if "langfuse_otel" not in existing: - litellm.callbacks = [*existing, "langfuse_otel"] # Suppress LiteLLM startup and OTEL noise (mirrors agent_factory.py filter). # Bedrock/SageMaker "no botocore" and OTEL proxy-server notices are harmless. @@ -102,23 +103,31 @@ def update(self, **kwargs: Any) -> None: @contextlib.contextmanager def langfuse_generation(*, name: str, model: str, input_messages: Any) -> Iterator[Any]: - """Emit a Langfuse ``generation`` around one LLM call. - - We create the generation explicitly rather than relying on LiteLLM's - ``langfuse_otel`` callback: that callback emits no generation at all when - the call runs inside an already-active Langfuse span, which is every LLMP - ``predict`` (they are wrapped in :func:`langfuse_observe`). The result was - that token usage — and therefore cost — never reached Langfuse for any - LLMP run. Creating it here works nested and keeps model, usage, and - payload under our control. - - Cost is deliberately not set: Langfuse derives it from ``usage_details`` - and its own per-model prices, which match the Vector proxy's published + """Create a Langfuse ``generation`` around one LLM call. + + LiteLLM's ``langfuse_otel`` callback emits no generation when the call runs + inside an already-active Langfuse span, which is every LLM-process + ``predict`` because they are wrapped in :func:`langfuse_observe`. Token + usage, and so cost, never reached Langfuse for those runs. Creating the + generation here works when nested and keeps the model, usage, and payload + under this module's control. + + Cost is deliberately not set. Langfuse derives it from ``usage_details`` + against its own per-model prices, which match the Vector proxy's published rates. - Yields a handle exposing ``update(**kwargs)`` — a no-op stand-in when - Langfuse is not installed or a generation cannot be started, so predictors - remain usable without the ``agentic``/``llm`` extras. + ``start_as_current_observation(as_type="generation")`` is a first-class + Langfuse v4 instrumentation API. LiteLLM's Langfuse bridge targets the v2 + SDK, pinning ``langfuse = ^2.45.0``, while this repo requires + ``langfuse>=4.5.1``. Support for v4 is BerriAI/litellm#24123, open and + unanswered since 2026-03-19. Retire this helper in favour of the callback + once that issue is closed and ``langfuse_otel`` is confirmed to emit a + generation under an active Langfuse span. For background and the A/B + evidence, see ``planning-docs/litellm-langfuse-compat.md``. + + Yields a handle exposing ``update(**kwargs)``. That handle is a no-op + stand-in when Langfuse is not installed or a generation cannot be started, + so predictors remain usable without the ``agentic`` and ``llm`` extras. """ manager = None try: @@ -356,8 +365,8 @@ async def _one_completion_async( usage = getattr(resp, "usage", None) in_tok = int(getattr(usage, "prompt_tokens", 0) or 0) if usage is not None else 0 out_tok = int(getattr(usage, "completion_tokens", 0) or 0) if usage is not None else 0 - # Log full usage so we can see thinking-token breakdown when available. - # The proxy may populate completion_tokens_details.reasoning_tokens. + # Full usage exposes the thinking-token breakdown when the proxy + # populates completion_tokens_details.reasoning_tokens. if usage is not None: logger.debug("LLM usage: %s", vars(usage) if hasattr(usage, "__dict__") else usage) raw = resp.choices[0].message.content diff --git a/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_agent_factory.py b/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_agent_factory.py index c49b098f..9bee828f 100644 --- a/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_agent_factory.py +++ b/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_agent_factory.py @@ -108,10 +108,12 @@ def test_string_model_wrapped_in_litellm_when_proxy_set(self) -> None: agent = build_adk_agent(config) assert isinstance(agent.model, LiteLlm) - # LiteLlm receives the "openai/" prefix so LiteLLM routes via the - # OpenAI-compatible proxy path; the prefix is stripped before the - # proxy sees the model name. - assert agent.model.model == "openai/gemini-3.1-flash-lite-preview" + # The bare model name is kept and the OpenAI-compatible proxy route is + # selected via custom_llm_provider instead of an "openai/" prefix: + # OpenInference reports this name to Langfuse, which matches its price + # table on the bare name (a prefixed name logs zero cost). + assert agent.model.model == "gemini-3.1-flash-lite-preview" + assert agent.model._additional_args["custom_llm_provider"] == "openai" def test_string_model_kept_as_string_without_proxy(self, monkeypatch: pytest.MonkeyPatch) -> None: """Without a proxy URL the model is passed as a plain string to LlmAgent.""" diff --git a/aieng-forecasting/tests/aieng/forecasting/methods/llm_processes/test__client.py b/aieng-forecasting/tests/aieng/forecasting/methods/llm_processes/test__client.py index da9cc2a3..a7d7fc36 100644 --- a/aieng-forecasting/tests/aieng/forecasting/methods/llm_processes/test__client.py +++ b/aieng-forecasting/tests/aieng/forecasting/methods/llm_processes/test__client.py @@ -263,7 +263,7 @@ async def fake_acompletion(**kwargs): # type: ignore[override] # --------------------------------------------------------------------------- -# langfuse_generation — usage/cost reporting for the Langfuse trace +# langfuse_generation: usage and cost reporting for the Langfuse trace # --------------------------------------------------------------------------- @@ -312,7 +312,7 @@ def update(self, **kwargs: object) -> None: @pytest.mark.asyncio async def test_completion_succeeds_when_langfuse_is_unavailable() -> None: - """A failing Langfuse client degrades to a no-op; the completion still returns.""" + """A failing Langfuse client degrades to a no-op and the completion returns.""" resp = _mock_litellm_response('{"ok": 1}') resp.usage = SimpleNamespace(prompt_tokens=3, completion_tokens=4) diff --git a/aieng-forecasting/tests/aieng/forecasting/test_langfuse_tracing.py b/aieng-forecasting/tests/aieng/forecasting/test_langfuse_tracing.py index 7d2c2d0b..53ba00d0 100644 --- a/aieng-forecasting/tests/aieng/forecasting/test_langfuse_tracing.py +++ b/aieng-forecasting/tests/aieng/forecasting/test_langfuse_tracing.py @@ -84,13 +84,6 @@ def test_get_client_exception_does_not_propagate(self) -> None: bootstrap._ensure_langfuse_client() assert bootstrap._langfuse_client_initialized is False - def test_missing_litellm_package_does_not_raise(self) -> None: - """Absent ``litellm`` skips callback registration without raising.""" - with patch.dict(sys.modules, {"litellm": None}): - bootstrap = _LangfuseTracingBootstrap() - bootstrap._register_litellm_langfuse_otel() - assert bootstrap._litellm_instrumented is False - def test_missing_openinference_package_does_not_raise(self) -> None: """Missing OpenInference ADK shim skips instrumentation silently.""" with patch.dict( @@ -130,25 +123,24 @@ def test_instrumentor_exception_does_not_propagate(self) -> None: class TestBootstrapLiteLLMCallbackContract: - """LiteLLM global ``callbacks`` list is updated idempotently.""" + """LiteLLM's ``langfuse_otel`` callback must never be registered.""" - def test_langfuse_otel_not_appended_when_already_present(self) -> None: - """Existing langfuse_otel entry must not be duplicated.""" - litellm_mod = MagicMock() - litellm_mod.callbacks = ["langfuse_otel", "other_hook"] - with patch.dict(sys.modules, {"litellm": litellm_mod}): - bootstrap = _LangfuseTracingBootstrap() - bootstrap._register_litellm_langfuse_otel() - assert litellm_mod.callbacks.count("langfuse_otel") == 1 - - def test_existing_callbacks_are_preserved(self) -> None: - """Appending ``langfuse_otel`` keeps prior callback entries.""" - litellm_mod = MagicMock() - litellm_mod.callbacks = ["other_hook"] - with patch.dict(sys.modules, {"litellm": litellm_mod}): + def test_langfuse_otel_is_not_registered(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Init must not add ``langfuse_otel`` to ``litellm.callbacks``. + + The callback is unusable against the Langfuse v4 SDK and stamps a zero + ``llm.cost.total`` on the active span, which suppresses Langfuse's own + cost calculation and forced agent-path generations to $0. See + ``planning-docs/litellm-langfuse-compat.md``. + """ + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-test") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-test") + deps = _all_deps_present(litellm_callbacks=["other_hook"]) + with patch.dict(sys.modules, deps): bootstrap = _LangfuseTracingBootstrap() - bootstrap._register_litellm_langfuse_otel() - assert "other_hook" in litellm_mod.callbacks + bootstrap.init() + assert "langfuse_otel" not in deps["litellm"].callbacks + assert deps["litellm"].callbacks == ["other_hook"], "unrelated callbacks must be left alone" # --------------------------------------------------------------------------- @@ -168,11 +160,10 @@ def test_no_op_without_credentials(self, monkeypatch: pytest.MonkeyPatch) -> Non bootstrap = _LangfuseTracingBootstrap() bootstrap.init() assert not bootstrap._langfuse_client_initialized - assert not bootstrap._litellm_instrumented assert not bootstrap._google_adk_instrumented - def test_repeated_init_does_not_duplicate_litellm_callback(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Calling ``init`` twice adds ``langfuse_otel`` at most once.""" + def test_repeated_init_instruments_adk_only_once(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Calling ``init`` twice instruments Google ADK at most once.""" monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-test") monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-test") deps = _all_deps_present() @@ -180,7 +171,8 @@ def test_repeated_init_does_not_duplicate_litellm_callback(self, monkeypatch: py bootstrap = _LangfuseTracingBootstrap() bootstrap.init() bootstrap.init() - assert deps["litellm"].callbacks.count("langfuse_otel") == 1 + instrumentor = deps["openinference.instrumentation.google_adk"].GoogleADKInstrumentor + assert instrumentor.return_value.instrument.call_count == 1 # --------------------------------------------------------------------------- diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__langfuse_tracing.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__langfuse_tracing.py.md index 105042e6..528f590d 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__langfuse_tracing.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__langfuse_tracing.py.md @@ -29,12 +29,11 @@ def _langfuse_credentials_present() -> bool: class _LangfuseTracingBootstrap: - """Registers LiteLLM + ADK exporters at most once per process.""" + """Registers the Langfuse client and ADK instrumentation once per process.""" - __slots__ = ("_google_adk_instrumented", "_langfuse_client_initialized", "_litellm_instrumented") + __slots__ = ("_google_adk_instrumented", "_langfuse_client_initialized") def __init__(self) -> None: - self._litellm_instrumented = False self._google_adk_instrumented = False self._langfuse_client_initialized = False @@ -51,7 +50,6 @@ class _LangfuseTracingBootstrap: # this, ADK spans are emitted into a no-op provider and never reach Langfuse. self._ensure_langfuse_client() - self._register_litellm_langfuse_otel() self._instrument_google_adk() def _ensure_langfuse_client(self) -> None: @@ -69,21 +67,6 @@ class _LangfuseTracingBootstrap: return self._langfuse_client_initialized = True - def _register_litellm_langfuse_otel(self) -> None: - """Register LiteLLM Langfuse callback.""" - if self._litellm_instrumented: - return - try: - import litellm # noqa: PLC0415 - except ImportError: - logger.debug("litellm not installed; skipping LiteLLM Langfuse callback.") - return - - existing = list(getattr(litellm, "callbacks", None) or []) - if "langfuse_otel" not in existing: - litellm.callbacks = [*existing, "langfuse_otel"] - self._litellm_instrumented = True - def _instrument_google_adk(self) -> None: """Instrument Google ADK.""" if self._google_adk_instrumented: @@ -125,11 +108,14 @@ def init_langfuse_tracing() -> None: ``TracerProvider`` receives Langfuse's span processor. This is required for ADK spans emitted via ``openinference-instrumentation-google-adk`` to reach Langfuse. - 2. Appends ``"langfuse_otel"`` to ``litellm.callbacks`` once (if - ``litellm`` is importable). - 3. Runs ``GoogleADKInstrumentor().instrument()`` once (if + 2. Runs ``GoogleADKInstrumentor().instrument()`` once (if ``openinference-instrumentation-google-adk`` is importable). + LiteLLM's ``langfuse_otel`` callback is deliberately not registered: it is + unusable against the Langfuse v4 SDK and stamps a zero ``llm.cost.total`` + on the active span, which suppresses Langfuse's own cost calculation. See + ``planning-docs/litellm-langfuse-compat.md``. + Set ``LANGFUSE_HOST`` or ``LANGFUSE_BASE_URL`` for non-default regions. For short-lived processes, call ``langfuse.get_client().flush()`` before exit so pending spans are exported. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md index db6a0513..d09560ce 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md @@ -674,12 +674,16 @@ def build_adk_agent( if isinstance(model, str) and config.openai_base_url: from google.adk.models.lite_llm import LiteLlm # noqa: PLC0415 - # Prefix with "openai/" so LiteLLM uses the OpenAI-compatible path. - # LiteLLM strips the prefix before sending, so the proxy receives the - # bare model name. - litellm_model = model if model.startswith("openai/") else f"openai/{model}" + # Route via LiteLLM's OpenAI-compatible path with ``custom_llm_provider`` + # rather than an ``openai/`` model prefix. ADK stamps ``LlmRequest.model`` + # from this name and OpenInference reports it to Langfuse, which matches + # its per-model price table on the bare name. A prefixed name matches + # nothing, so the generation is logged at zero cost. Both forms route + # identically. See planning-docs/litellm-langfuse-compat.md. + bare_model = model[len("openai/") :] if model.startswith("openai/") else model model = LiteLlm( - model=litellm_model, + model=bare_model, + custom_llm_provider="openai", api_base=config.openai_base_url, api_key=config.openai_api_key, ) diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__llm_processes___client.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__llm_processes___client.py.md index c01345ce..6311d1b1 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__llm_processes___client.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__llm_processes___client.py.md @@ -29,7 +29,6 @@ import contextlib import contextvars import json import logging -import os import warnings from concurrent.futures import ThreadPoolExecutor from typing import Any, Callable, Iterator, TypeVar @@ -45,21 +44,23 @@ _BOOTSTRAP_DONE = False def bootstrap_litellm() -> None: - """One-time wiring of LiteLLM callbacks. + """Suppress LiteLLM and OpenTelemetry logging noise, once per process. Lazy and idempotent so non-LLM predictors do not require Langfuse env vars. - The Langfuse OTEL callback is registered only when ``LANGFUSE_PUBLIC_KEY`` - is set in the environment. + + LiteLLM's ``langfuse_otel`` callback is deliberately not registered. It is + unusable against the Langfuse v4 SDK this repo depends on, and it stamps + ``llm.cost.total`` on the active span from LiteLLM's own ``response_cost``, + which is ``0`` for every proxy-routed model. Langfuse honours a supplied + cost instead of deriving one from usage, so the callback forced agent-path + generations to $0. Instead, :func:`langfuse_generation` creates LLM-process + generations directly and OpenInference covers the agent path, so both price + correctly from ``usage_details``. See + ``planning-docs/litellm-langfuse-compat.md``. """ global _BOOTSTRAP_DONE # noqa: PLW0603 if _BOOTSTRAP_DONE: return - import litellm # noqa: PLC0415 - - if os.environ.get("LANGFUSE_PUBLIC_KEY"): - existing = list(getattr(litellm, "callbacks", []) or []) - if "langfuse_otel" not in existing: - litellm.callbacks = [*existing, "langfuse_otel"] # Suppress LiteLLM startup and OTEL noise (mirrors agent_factory.py filter). # Bedrock/SageMaker "no botocore" and OTEL proxy-server notices are harmless. @@ -107,23 +108,31 @@ class _NoopGeneration: @contextlib.contextmanager def langfuse_generation(*, name: str, model: str, input_messages: Any) -> Iterator[Any]: - """Emit a Langfuse ``generation`` around one LLM call. - - We create the generation explicitly rather than relying on LiteLLM's - ``langfuse_otel`` callback: that callback emits no generation at all when - the call runs inside an already-active Langfuse span, which is every LLMP - ``predict`` (they are wrapped in :func:`langfuse_observe`). The result was - that token usage — and therefore cost — never reached Langfuse for any - LLMP run. Creating it here works nested and keeps model, usage, and - payload under our control. - - Cost is deliberately not set: Langfuse derives it from ``usage_details`` - and its own per-model prices, which match the Vector proxy's published + """Create a Langfuse ``generation`` around one LLM call. + + LiteLLM's ``langfuse_otel`` callback emits no generation when the call runs + inside an already-active Langfuse span, which is every LLM-process + ``predict`` because they are wrapped in :func:`langfuse_observe`. Token + usage, and so cost, never reached Langfuse for those runs. Creating the + generation here works when nested and keeps the model, usage, and payload + under this module's control. + + Cost is deliberately not set. Langfuse derives it from ``usage_details`` + against its own per-model prices, which match the Vector proxy's published rates. - Yields a handle exposing ``update(**kwargs)`` — a no-op stand-in when - Langfuse is not installed or a generation cannot be started, so predictors - remain usable without the ``agentic``/``llm`` extras. + ``start_as_current_observation(as_type="generation")`` is a first-class + Langfuse v4 instrumentation API. LiteLLM's Langfuse bridge targets the v2 + SDK, pinning ``langfuse = ^2.45.0``, while this repo requires + ``langfuse>=4.5.1``. Support for v4 is BerriAI/litellm#24123, open and + unanswered since 2026-03-19. Retire this helper in favour of the callback + once that issue is closed and ``langfuse_otel`` is confirmed to emit a + generation under an active Langfuse span. For background and the A/B + evidence, see ``planning-docs/litellm-langfuse-compat.md``. + + Yields a handle exposing ``update(**kwargs)``. That handle is a no-op + stand-in when Langfuse is not installed or a generation cannot be started, + so predictors remain usable without the ``agentic`` and ``llm`` extras. """ manager = None try: @@ -361,8 +370,8 @@ async def _one_completion_async( usage = getattr(resp, "usage", None) in_tok = int(getattr(usage, "prompt_tokens", 0) or 0) if usage is not None else 0 out_tok = int(getattr(usage, "completion_tokens", 0) or 0) if usage is not None else 0 - # Log full usage so we can see thinking-token breakdown when available. - # The proxy may populate completion_tokens_details.reasoning_tokens. + # Full usage exposes the thinking-token breakdown when the proxy + # populates completion_tokens_details.reasoning_tokens. if usage is not None: logger.debug("LLM usage: %s", vars(usage) if hasattr(usage, "__dict__") else usage) raw = resp.choices[0].message.content diff --git a/implementations/getting_started/concierge_agent/context/catalog.yaml b/implementations/getting_started/concierge_agent/context/catalog.yaml index 38083728..0457cc6c 100644 --- a/implementations/getting_started/concierge_agent/context/catalog.yaml +++ b/implementations/getting_started/concierge_agent/context/catalog.yaml @@ -1,7 +1,7 @@ source_url: https://github.com/VectorInstitute/agentic-forecasting -git_ref: 1ca028d0df744fae68e47af998223d3347b92b1d +git_ref: c44102e33c27f0e2749cbe17b66e7599e86e255e branch: main -built_at: '2026-08-20T16:47:29+00:00' +built_at: '2026-08-24T19:58:28+00:00' ingest_source: /home/akore/vscodeprojects/agentic-forecasting entry_count: 197 entries: @@ -429,7 +429,7 @@ entries: - init_langfuse_tracing - print_langfuse_trace_url sections: [] - chars: 6632 + chars: 6141 artifact: artifacts/aieng-forecasting__aieng__forecasting__langfuse_tracing.py.md - path: aieng-forecasting/aieng/forecasting/methods/README.md kind: markdown @@ -512,7 +512,7 @@ entries: - AgentConfig - build_adk_agent sections: [] - chars: 35486 + chars: 35818 artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md - path: aieng-forecasting/aieng/forecasting/methods/agentic/curriculum.py kind: python @@ -635,7 +635,7 @@ entries: - sample_n_async - run_async sections: [] - chars: 20531 + chars: 21351 artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__llm_processes___client.py.md - path: aieng-forecasting/aieng/forecasting/methods/llm_processes/base.py kind: python diff --git a/implementations/getting_started/concierge_agent/skills/repo-navigation/references/catalog-summary.yaml b/implementations/getting_started/concierge_agent/skills/repo-navigation/references/catalog-summary.yaml index 11b04760..1a114dd1 100644 --- a/implementations/getting_started/concierge_agent/skills/repo-navigation/references/catalog-summary.yaml +++ b/implementations/getting_started/concierge_agent/skills/repo-navigation/references/catalog-summary.yaml @@ -1,8 +1,8 @@ # Concierge catalog summary (regenerated by scripts/build_concierge_context.py) source_url: https://github.com/VectorInstitute/agentic-forecasting branch: main -built_at: '2026-08-20T16:47:29+00:00' -git_ref: 1ca028d0df744fae68e47af998223d3347b92b1d +built_at: '2026-08-24T19:58:28+00:00' +git_ref: c44102e33c27f0e2749cbe17b66e7599e86e255e entry_count: 197 domains: docs: 4 From 320f351d3d4176171835fd4879e2d7c0983e6c90 Mon Sep 17 00:00:00 2001 From: a-kore Date: Mon, 24 Aug 2026 16:54:50 -0400 Subject: [PATCH 3/3] docs: drop code references to a removed planning doc The compat write-up was not committed, leaving five docstrings and comments pointing at planning-docs/litellm-langfuse-compat.md. Source should not cite planning documents in any case: they move and are deleted independently of the code. Each site keeps the explanation it needs inline. The retirement condition for langfuse_generation stays in its docstring, so the exit criteria survive without an external reference. --- .../aieng/forecasting/langfuse_tracing.py | 3 +-- .../aieng/forecasting/methods/agentic/agent_factory.py | 2 +- .../aieng/forecasting/methods/llm_processes/_client.py | 6 ++---- .../tests/aieng/forecasting/test_langfuse_tracing.py | 3 +-- ...casting__aieng__forecasting__langfuse_tracing.py.md | 3 +-- ..._forecasting__methods__agentic__agent_factory.py.md | 2 +- ..._forecasting__methods__llm_processes___client.py.md | 6 ++---- .../concierge_agent/context/catalog.yaml | 10 +++++----- .../repo-navigation/references/catalog-summary.yaml | 4 ++-- 9 files changed, 16 insertions(+), 23 deletions(-) diff --git a/aieng-forecasting/aieng/forecasting/langfuse_tracing.py b/aieng-forecasting/aieng/forecasting/langfuse_tracing.py index 74258f12..1936a7a8 100644 --- a/aieng-forecasting/aieng/forecasting/langfuse_tracing.py +++ b/aieng-forecasting/aieng/forecasting/langfuse_tracing.py @@ -108,8 +108,7 @@ def init_langfuse_tracing() -> None: LiteLLM's ``langfuse_otel`` callback is deliberately not registered: it is unusable against the Langfuse v4 SDK and stamps a zero ``llm.cost.total`` - on the active span, which suppresses Langfuse's own cost calculation. See - ``planning-docs/litellm-langfuse-compat.md``. + on the active span, which suppresses Langfuse's own cost calculation. Set ``LANGFUSE_HOST`` or ``LANGFUSE_BASE_URL`` for non-default regions. For short-lived processes, call ``langfuse.get_client().flush()`` before diff --git a/aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py b/aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py index 891e0a3d..e5dd7daf 100644 --- a/aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py +++ b/aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py @@ -674,7 +674,7 @@ def build_adk_agent( # from this name and OpenInference reports it to Langfuse, which matches # its per-model price table on the bare name. A prefixed name matches # nothing, so the generation is logged at zero cost. Both forms route - # identically. See planning-docs/litellm-langfuse-compat.md. + # identically. bare_model = model[len("openai/") :] if model.startswith("openai/") else model model = LiteLlm( model=bare_model, diff --git a/aieng-forecasting/aieng/forecasting/methods/llm_processes/_client.py b/aieng-forecasting/aieng/forecasting/methods/llm_processes/_client.py index e0c5d4e9..fcbaa336 100644 --- a/aieng-forecasting/aieng/forecasting/methods/llm_processes/_client.py +++ b/aieng-forecasting/aieng/forecasting/methods/llm_processes/_client.py @@ -50,8 +50,7 @@ def bootstrap_litellm() -> None: cost instead of deriving one from usage, so the callback forced agent-path generations to $0. Instead, :func:`langfuse_generation` creates LLM-process generations directly and OpenInference covers the agent path, so both price - correctly from ``usage_details``. See - ``planning-docs/litellm-langfuse-compat.md``. + correctly from ``usage_details``. """ global _BOOTSTRAP_DONE # noqa: PLW0603 if _BOOTSTRAP_DONE: @@ -122,8 +121,7 @@ def langfuse_generation(*, name: str, model: str, input_messages: Any) -> Iterat ``langfuse>=4.5.1``. Support for v4 is BerriAI/litellm#24123, open and unanswered since 2026-03-19. Retire this helper in favour of the callback once that issue is closed and ``langfuse_otel`` is confirmed to emit a - generation under an active Langfuse span. For background and the A/B - evidence, see ``planning-docs/litellm-langfuse-compat.md``. + generation under an active Langfuse span. Yields a handle exposing ``update(**kwargs)``. That handle is a no-op stand-in when Langfuse is not installed or a generation cannot be started, diff --git a/aieng-forecasting/tests/aieng/forecasting/test_langfuse_tracing.py b/aieng-forecasting/tests/aieng/forecasting/test_langfuse_tracing.py index 53ba00d0..17f148ad 100644 --- a/aieng-forecasting/tests/aieng/forecasting/test_langfuse_tracing.py +++ b/aieng-forecasting/tests/aieng/forecasting/test_langfuse_tracing.py @@ -130,8 +130,7 @@ def test_langfuse_otel_is_not_registered(self, monkeypatch: pytest.MonkeyPatch) The callback is unusable against the Langfuse v4 SDK and stamps a zero ``llm.cost.total`` on the active span, which suppresses Langfuse's own - cost calculation and forced agent-path generations to $0. See - ``planning-docs/litellm-langfuse-compat.md``. + cost calculation and forced agent-path generations to $0. """ monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-test") monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-test") diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__langfuse_tracing.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__langfuse_tracing.py.md index 528f590d..3aa109fd 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__langfuse_tracing.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__langfuse_tracing.py.md @@ -113,8 +113,7 @@ def init_langfuse_tracing() -> None: LiteLLM's ``langfuse_otel`` callback is deliberately not registered: it is unusable against the Langfuse v4 SDK and stamps a zero ``llm.cost.total`` - on the active span, which suppresses Langfuse's own cost calculation. See - ``planning-docs/litellm-langfuse-compat.md``. + on the active span, which suppresses Langfuse's own cost calculation. Set ``LANGFUSE_HOST`` or ``LANGFUSE_BASE_URL`` for non-default regions. For short-lived processes, call ``langfuse.get_client().flush()`` before diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md index d09560ce..be9c2a65 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md @@ -679,7 +679,7 @@ def build_adk_agent( # from this name and OpenInference reports it to Langfuse, which matches # its per-model price table on the bare name. A prefixed name matches # nothing, so the generation is logged at zero cost. Both forms route - # identically. See planning-docs/litellm-langfuse-compat.md. + # identically. bare_model = model[len("openai/") :] if model.startswith("openai/") else model model = LiteLlm( model=bare_model, diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__llm_processes___client.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__llm_processes___client.py.md index 6311d1b1..63511409 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__llm_processes___client.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__llm_processes___client.py.md @@ -55,8 +55,7 @@ def bootstrap_litellm() -> None: cost instead of deriving one from usage, so the callback forced agent-path generations to $0. Instead, :func:`langfuse_generation` creates LLM-process generations directly and OpenInference covers the agent path, so both price - correctly from ``usage_details``. See - ``planning-docs/litellm-langfuse-compat.md``. + correctly from ``usage_details``. """ global _BOOTSTRAP_DONE # noqa: PLW0603 if _BOOTSTRAP_DONE: @@ -127,8 +126,7 @@ def langfuse_generation(*, name: str, model: str, input_messages: Any) -> Iterat ``langfuse>=4.5.1``. Support for v4 is BerriAI/litellm#24123, open and unanswered since 2026-03-19. Retire this helper in favour of the callback once that issue is closed and ``langfuse_otel`` is confirmed to emit a - generation under an active Langfuse span. For background and the A/B - evidence, see ``planning-docs/litellm-langfuse-compat.md``. + generation under an active Langfuse span. Yields a handle exposing ``update(**kwargs)``. That handle is a no-op stand-in when Langfuse is not installed or a generation cannot be started, diff --git a/implementations/getting_started/concierge_agent/context/catalog.yaml b/implementations/getting_started/concierge_agent/context/catalog.yaml index 0457cc6c..21831953 100644 --- a/implementations/getting_started/concierge_agent/context/catalog.yaml +++ b/implementations/getting_started/concierge_agent/context/catalog.yaml @@ -1,7 +1,7 @@ source_url: https://github.com/VectorInstitute/agentic-forecasting -git_ref: c44102e33c27f0e2749cbe17b66e7599e86e255e +git_ref: 04507bf406cc5524c3d9523fc4835f29462fe3ea branch: main -built_at: '2026-08-24T19:58:28+00:00' +built_at: '2026-08-24T20:54:40+00:00' ingest_source: /home/akore/vscodeprojects/agentic-forecasting entry_count: 197 entries: @@ -429,7 +429,7 @@ entries: - init_langfuse_tracing - print_langfuse_trace_url sections: [] - chars: 6141 + chars: 6087 artifact: artifacts/aieng-forecasting__aieng__forecasting__langfuse_tracing.py.md - path: aieng-forecasting/aieng/forecasting/methods/README.md kind: markdown @@ -512,7 +512,7 @@ entries: - AgentConfig - build_adk_agent sections: [] - chars: 35818 + chars: 35772 artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md - path: aieng-forecasting/aieng/forecasting/methods/agentic/curriculum.py kind: python @@ -635,7 +635,7 @@ entries: - sample_n_async - run_async sections: [] - chars: 21351 + chars: 21206 artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__llm_processes___client.py.md - path: aieng-forecasting/aieng/forecasting/methods/llm_processes/base.py kind: python diff --git a/implementations/getting_started/concierge_agent/skills/repo-navigation/references/catalog-summary.yaml b/implementations/getting_started/concierge_agent/skills/repo-navigation/references/catalog-summary.yaml index 1a114dd1..a715b400 100644 --- a/implementations/getting_started/concierge_agent/skills/repo-navigation/references/catalog-summary.yaml +++ b/implementations/getting_started/concierge_agent/skills/repo-navigation/references/catalog-summary.yaml @@ -1,8 +1,8 @@ # Concierge catalog summary (regenerated by scripts/build_concierge_context.py) source_url: https://github.com/VectorInstitute/agentic-forecasting branch: main -built_at: '2026-08-24T19:58:28+00:00' -git_ref: c44102e33c27f0e2749cbe17b66e7599e86e255e +built_at: '2026-08-24T20:54:40+00:00' +git_ref: 04507bf406cc5524c3d9523fc4835f29462fe3ea entry_count: 197 domains: docs: 4