Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion aieng-forecasting/aieng/forecasting/methods/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -111,5 +111,5 @@ from aieng.forecasting.methods.agentic import (
| `agentic/outputs.py` | `ContinuousAgentForecastOutput` | Canonical continuous forecasting output schema. Declares `modality = "continuous"`, requires one forecast per task horizon and the standard quantile grid, then converts to `ContinuousForecast` payloads. |
| `agentic/outputs.py` | `DiscreteAgentForecastOutput` | Binary event output schema (`modality = "discrete"`): one probability plus `reasoning` / `key_signals` metadata, converted to a `BinaryForecast` payload. |
| `agentic/outputs.py` | `CategoricalAgentForecastOutput` | Ordered-categorical output schema (`modality = "categorical"`): one `{label, probability}` row per task category, validated against `task.categories` and converted to a `CategoricalForecast` payload. |
| `agentic/predictor.py` | `AgentPredictor` | Track 1 `Predictor` that builds prompts, runs an ADK agent through `AdkTextRunner`, validates structured JSON, and converts it to `Prediction` objects. Accepts an optional injected runner for tests or custom observability. |
| `agentic/predictor.py` | `AgentPredictor` | Track 1 `Predictor` that builds prompts, runs an ADK agent through `AdkTextRunner`, validates structured JSON, and converts it to `Prediction` objects. Empty agent responses raise a descriptive error before JSON parsing. Accepts an optional injected runner for tests or custom observability. |
| `agentic/predictor.py` | `ForecastPromptBuilder` | Protocol for task-specific prompt builders that turn `(task, context)` into the text passed to the agent. |
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,9 @@ def predict(self, task: ForecastingTask, context: ForecastContext) -> list[Predi
# be swapped in without breaking the parse layer.
output_str = strip_markdown_fence(output_str)

if not output_str.strip():
raise ValueError("Agent returned an empty response; the output token budget may have been exhausted")

# Validate the output against the output schema; tolerate JSON
# responses that ``model_validate_json`` cannot parse but
# ``json.loads`` + ``model_validate`` can.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,13 @@ def fail_validate_json(*_args: Any, **_kwargs: Any) -> Any:
class TestPredictErrorHandling:
"""``predict()`` swallows conversion errors but propagates schema errors."""

def test_empty_response_raises_descriptive_error(self) -> None:
"""An exhausted agent response reports the agent failure, not a JSON error."""
predictor, _ = _make_predictor(response=" ")

with pytest.raises(ValueError, match="empty response"):
predictor.predict(_task([1]), _context())

def test_horizon_mismatch_returns_empty_list_and_logs(self, caplog: pytest.LogCaptureFixture) -> None:
"""Output that validates but fails to_predictions yields ``[]`` and logs."""
# Output covers horizon 1, task asks for [1, 2]; conversion will raise.
Expand Down
4 changes: 4 additions & 0 deletions implementations/energy_oil_forecasting/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,10 @@ notebook 05).
| Role per task | `tasks.py` | Prompt builders, `build_wti_news_predictor(task)` |
| Learning agent | `adaptive_agent/` | Persistent, mutable strategy state updated via self-directed study (notebooks 05–06) |

The WTI code-execution preset reserves 32K output tokens for tool work and the
final structured forecast. If an agent still returns no text, `AgentPredictor`
reports an empty response directly instead of surfacing a JSON parsing error.

---

## Data Source & Setup
Expand Down
8 changes: 4 additions & 4 deletions implementations/energy_oil_forecasting/analyst_agent/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -488,7 +488,7 @@ def build_wti_news_config(
def build_wti_code_exec_config(
model: str = LITE_MODEL,
search_model: str = LITE_MODEL,
max_output_tokens: int = 16_384,
max_output_tokens: int = 32_768,
verifier_model: str = ADVANCED_MODEL,
verifier_max_attempts: int = 3,
verifier_confidence_threshold: int = 8,
Expand All @@ -511,11 +511,11 @@ def build_wti_code_exec_config(
Model for the context-retrieval (web-search) sub-tool. Defaults to
the lite model (``gemini-3.1-flash-lite-preview``) independently of ``model`` so that Gemini
handles Google Search even when the analyst uses a different provider.
max_output_tokens : int, default=16_384
max_output_tokens : int, default=32_768
Maximum tokens per model response. The default is set well above
LiteLLM's OpenAI-compatible endpoint default of 4096, which is not
enough for Claude to write a complete ``run_code`` Python script in a
single function call — causing repeated retries with empty arguments.
enough for an analyst to run tools and return the final structured
forecast in one response.
verifier_model : str
Model for the independent temporal-leakage verifier that audits each
``search_web`` result against ``cutoff_date`` before it is returned.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -493,7 +493,7 @@ def build_wti_news_config(
def build_wti_code_exec_config(
model: str = LITE_MODEL,
search_model: str = LITE_MODEL,
max_output_tokens: int = 16_384,
max_output_tokens: int = 32_768,
verifier_model: str = ADVANCED_MODEL,
verifier_max_attempts: int = 3,
verifier_confidence_threshold: int = 8,
Expand All @@ -516,11 +516,11 @@ def build_wti_code_exec_config(
Model for the context-retrieval (web-search) sub-tool. Defaults to
the lite model (``gemini-3.1-flash-lite-preview``) independently of ``model`` so that Gemini
handles Google Search even when the analyst uses a different provider.
max_output_tokens : int, default=16_384
max_output_tokens : int, default=32_768
Maximum tokens per model response. The default is set well above
LiteLLM's OpenAI-compatible endpoint default of 4096, which is not
enough for Claude to write a complete ``run_code`` Python script in a
single function call — causing repeated retries with empty arguments.
enough for an analyst to run tools and return the final structured
forecast in one response.
verifier_model : str
Model for the independent temporal-leakage verifier that audits each
``search_web`` result against ``cutoff_date`` before it is returned.
Expand Down
10 changes: 10 additions & 0 deletions implementations/tests/energy_oil_forecasting/test_analyst_agent.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
"""Focused tests for the WTI analyst capability presets."""

from energy_oil_forecasting.analyst_agent import build_wti_code_exec_config


def test_code_execution_has_headroom_for_tool_use_and_final_output() -> None:
"""Code execution must leave room for both sandbox work and forecast JSON."""
config = build_wti_code_exec_config()

assert config.max_output_tokens == 32_768