From 8d6c07d1ec8bf144892833775dbcb88f5026256e Mon Sep 17 00:00:00 2001 From: Oliver Slapinski Date: Sun, 16 Aug 2026 17:58:45 -0400 Subject: [PATCH] fix(agentic): report exhausted responses clearly Signed-off-by: Oliver Slapinski --- aieng-forecasting/aieng/forecasting/methods/README.md | 2 +- .../aieng/forecasting/methods/agentic/predictor.py | 3 +++ .../forecasting/methods/agentic/test_predictor.py | 7 +++++++ implementations/energy_oil_forecasting/README.md | 4 ++++ .../energy_oil_forecasting/analyst_agent/agent.py | 8 ++++---- ..._energy_oil_forecasting__analyst_agent__agent.py.md | 8 ++++---- .../tests/energy_oil_forecasting/test_analyst_agent.py | 10 ++++++++++ 7 files changed, 33 insertions(+), 9 deletions(-) create mode 100644 implementations/tests/energy_oil_forecasting/test_analyst_agent.py diff --git a/aieng-forecasting/aieng/forecasting/methods/README.md b/aieng-forecasting/aieng/forecasting/methods/README.md index d591f412..52c817f7 100644 --- a/aieng-forecasting/aieng/forecasting/methods/README.md +++ b/aieng-forecasting/aieng/forecasting/methods/README.md @@ -111,5 +111,5 @@ from aieng.forecasting.methods.agentic import ( | `agentic/outputs.py` | `ContinuousAgentForecastOutput` | Canonical continuous forecasting output schema. Declares `modality = "continuous"`, requires one forecast per task horizon and the standard quantile grid, then converts to `ContinuousForecast` payloads. | | `agentic/outputs.py` | `DiscreteAgentForecastOutput` | Binary event output schema (`modality = "discrete"`): one probability plus `reasoning` / `key_signals` metadata, converted to a `BinaryForecast` payload. | | `agentic/outputs.py` | `CategoricalAgentForecastOutput` | Ordered-categorical output schema (`modality = "categorical"`): one `{label, probability}` row per task category, validated against `task.categories` and converted to a `CategoricalForecast` payload. | -| `agentic/predictor.py` | `AgentPredictor` | Track 1 `Predictor` that builds prompts, runs an ADK agent through `AdkTextRunner`, validates structured JSON, and converts it to `Prediction` objects. Accepts an optional injected runner for tests or custom observability. | +| `agentic/predictor.py` | `AgentPredictor` | Track 1 `Predictor` that builds prompts, runs an ADK agent through `AdkTextRunner`, validates structured JSON, and converts it to `Prediction` objects. Empty agent responses raise a descriptive error before JSON parsing. Accepts an optional injected runner for tests or custom observability. | | `agentic/predictor.py` | `ForecastPromptBuilder` | Protocol for task-specific prompt builders that turn `(task, context)` into the text passed to the agent. | diff --git a/aieng-forecasting/aieng/forecasting/methods/agentic/predictor.py b/aieng-forecasting/aieng/forecasting/methods/agentic/predictor.py index 25010e60..d99eb668 100644 --- a/aieng-forecasting/aieng/forecasting/methods/agentic/predictor.py +++ b/aieng-forecasting/aieng/forecasting/methods/agentic/predictor.py @@ -287,6 +287,9 @@ def predict(self, task: ForecastingTask, context: ForecastContext) -> list[Predi # be swapped in without breaking the parse layer. output_str = strip_markdown_fence(output_str) + if not output_str.strip(): + raise ValueError("Agent returned an empty response; the output token budget may have been exhausted") + # Validate the output against the output schema; tolerate JSON # responses that ``model_validate_json`` cannot parse but # ``json.loads`` + ``model_validate`` can. diff --git a/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_predictor.py b/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_predictor.py index aab25e2f..b30017f5 100644 --- a/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_predictor.py +++ b/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_predictor.py @@ -287,6 +287,13 @@ def fail_validate_json(*_args: Any, **_kwargs: Any) -> Any: class TestPredictErrorHandling: """``predict()`` swallows conversion errors but propagates schema errors.""" + def test_empty_response_raises_descriptive_error(self) -> None: + """An exhausted agent response reports the agent failure, not a JSON error.""" + predictor, _ = _make_predictor(response=" ") + + with pytest.raises(ValueError, match="empty response"): + predictor.predict(_task([1]), _context()) + def test_horizon_mismatch_returns_empty_list_and_logs(self, caplog: pytest.LogCaptureFixture) -> None: """Output that validates but fails to_predictions yields ``[]`` and logs.""" # Output covers horizon 1, task asks for [1, 2]; conversion will raise. diff --git a/implementations/energy_oil_forecasting/README.md b/implementations/energy_oil_forecasting/README.md index 04a0bc96..79ac3725 100644 --- a/implementations/energy_oil_forecasting/README.md +++ b/implementations/energy_oil_forecasting/README.md @@ -112,6 +112,10 @@ notebook 05). | Role per task | `tasks.py` | Prompt builders, `build_wti_news_predictor(task)` | | Learning agent | `adaptive_agent/` | Persistent, mutable strategy state updated via self-directed study (notebooks 05–06) | +The WTI code-execution preset reserves 32K output tokens for tool work and the +final structured forecast. If an agent still returns no text, `AgentPredictor` +reports an empty response directly instead of surfacing a JSON parsing error. + --- ## Data Source & Setup diff --git a/implementations/energy_oil_forecasting/analyst_agent/agent.py b/implementations/energy_oil_forecasting/analyst_agent/agent.py index 42055c55..bf7a820e 100644 --- a/implementations/energy_oil_forecasting/analyst_agent/agent.py +++ b/implementations/energy_oil_forecasting/analyst_agent/agent.py @@ -488,7 +488,7 @@ def build_wti_news_config( def build_wti_code_exec_config( model: str = LITE_MODEL, search_model: str = LITE_MODEL, - max_output_tokens: int = 16_384, + max_output_tokens: int = 32_768, verifier_model: str = ADVANCED_MODEL, verifier_max_attempts: int = 3, verifier_confidence_threshold: int = 8, @@ -511,11 +511,11 @@ def build_wti_code_exec_config( Model for the context-retrieval (web-search) sub-tool. Defaults to the lite model (``gemini-3.1-flash-lite-preview``) independently of ``model`` so that Gemini handles Google Search even when the analyst uses a different provider. - max_output_tokens : int, default=16_384 + max_output_tokens : int, default=32_768 Maximum tokens per model response. The default is set well above LiteLLM's OpenAI-compatible endpoint default of 4096, which is not - enough for Claude to write a complete ``run_code`` Python script in a - single function call — causing repeated retries with empty arguments. + enough for an analyst to run tools and return the final structured + forecast in one response. verifier_model : str Model for the independent temporal-leakage verifier that audits each ``search_web`` result against ``cutoff_date`` before it is returned. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__analyst_agent__agent.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__analyst_agent__agent.py.md index e05eeae0..a726e5d3 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__analyst_agent__agent.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__analyst_agent__agent.py.md @@ -493,7 +493,7 @@ def build_wti_news_config( def build_wti_code_exec_config( model: str = LITE_MODEL, search_model: str = LITE_MODEL, - max_output_tokens: int = 16_384, + max_output_tokens: int = 32_768, verifier_model: str = ADVANCED_MODEL, verifier_max_attempts: int = 3, verifier_confidence_threshold: int = 8, @@ -516,11 +516,11 @@ def build_wti_code_exec_config( Model for the context-retrieval (web-search) sub-tool. Defaults to the lite model (``gemini-3.1-flash-lite-preview``) independently of ``model`` so that Gemini handles Google Search even when the analyst uses a different provider. - max_output_tokens : int, default=16_384 + max_output_tokens : int, default=32_768 Maximum tokens per model response. The default is set well above LiteLLM's OpenAI-compatible endpoint default of 4096, which is not - enough for Claude to write a complete ``run_code`` Python script in a - single function call — causing repeated retries with empty arguments. + enough for an analyst to run tools and return the final structured + forecast in one response. verifier_model : str Model for the independent temporal-leakage verifier that audits each ``search_web`` result against ``cutoff_date`` before it is returned. diff --git a/implementations/tests/energy_oil_forecasting/test_analyst_agent.py b/implementations/tests/energy_oil_forecasting/test_analyst_agent.py new file mode 100644 index 00000000..044dba57 --- /dev/null +++ b/implementations/tests/energy_oil_forecasting/test_analyst_agent.py @@ -0,0 +1,10 @@ +"""Focused tests for the WTI analyst capability presets.""" + +from energy_oil_forecasting.analyst_agent import build_wti_code_exec_config + + +def test_code_execution_has_headroom_for_tool_use_and_final_output() -> None: + """Code execution must leave room for both sandbox work and forecast JSON.""" + config = build_wti_code_exec_config() + + assert config.max_output_tokens == 32_768