diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index b379fadd9..24fbad1a9 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -16,7 +16,7 @@ from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.models import CreatedVisualization, DatasetItem +from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem from gooddata_eval.core.runner import EvalReport, ItemReport @@ -86,8 +86,13 @@ def _dispatch_agentic( model_version_override: str | None, reasoning_effort: ReasoningEffort | None = None, agent_id: str | None = None, -) -> None: - """Call the appropriate evaluate_agentic_* function for the item's test_kind.""" +) -> AgenticEvalOutcome | list[str] | None: + """Call the appropriate evaluate_agentic_* function for the item's test_kind. + + Returns whatever that function returns -- alert_skill/metric_skill/conversation return + an AgenticEvalOutcome; the rest still return None + (unchanged). + """ kind = item.test_kind eo = item.expected_output lf_kw: _LfKw = { @@ -100,7 +105,7 @@ def _dispatch_agentic( } if kind in ("vis_agentic", "agentic_visualization"): - evaluate_agentic_visualization( + return evaluate_agentic_visualization( host=host, token=token, workspace_id=workspace_id, @@ -111,7 +116,7 @@ def _dispatch_agentic( **lf_kw, ) elif kind == "agentic_metric_skill": - evaluate_agentic_metric_skill( + return evaluate_agentic_metric_skill( host=host, token=token, workspace_id=workspace_id, @@ -122,7 +127,7 @@ def _dispatch_agentic( **lf_kw, ) elif kind == "agentic_alert_skill": - evaluate_agentic_alert_skill( + return evaluate_agentic_alert_skill( host=host, token=token, workspace_id=workspace_id, @@ -136,7 +141,7 @@ def _dispatch_agentic( eo_dict = eo if isinstance(eo, dict) else {} tool_call = eo_dict.get("tool_call", {}) expected_args = tool_call.get("function_arguments", eo_dict) - evaluate_agentic_search_tool( + return evaluate_agentic_search_tool( host=host, token=token, workspace_id=workspace_id, @@ -147,7 +152,7 @@ def _dispatch_agentic( **lf_kw, ) elif kind == "agentic_general_question": - evaluate_agentic_general_question( + return evaluate_agentic_general_question( host=host, token=token, workspace_id=workspace_id, @@ -158,7 +163,7 @@ def _dispatch_agentic( **lf_kw, ) elif kind == "agentic_guardrail": - evaluate_agentic_guardrail( + return evaluate_agentic_guardrail( host=host, token=token, workspace_id=workspace_id, @@ -180,7 +185,7 @@ def _dispatch_agentic( ) elif kind == "agentic_conversation": fixture_data = eo.get("fixture") or eo if isinstance(eo, dict) else {} - evaluate_agentic_conversation( + return evaluate_agentic_conversation( host=host, token=token, workspace_id=workspace_id, @@ -228,14 +233,26 @@ def run_agentic_items( ) t0 = time.perf_counter() try: - _dispatch_agentic( + outcome = _dispatch_agentic( item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort, agent_id ) + if isinstance(outcome, AgenticEvalOutcome): + reasoning_steps = outcome.reasoning_steps + conversation_id = outcome.conversation_id + response_id = outcome.response_id + else: + reasoning_steps, conversation_id, response_id = outcome, None, None item_report.pass_at_k = True item_report.runs = k + item_report.reasoning_steps = reasoning_steps or [] + item_report.conversation_id = conversation_id + item_report.response_id = response_id except AssertionError as exc: item_report.pass_at_k = False item_report.runs = k + item_report.reasoning_steps = getattr(exc, "reasoning_steps", None) or [] + item_report.conversation_id = getattr(exc, "conversation_id", None) + item_report.response_id = getattr(exc, "response_id", None) print(f"[agentic] {item.id} FAIL: {exc}", flush=True) except Exception as exc: item_report.error = f"{type(exc).__name__}: {exc}" diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index 11a0bbf71..fe26de79d 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -6,7 +6,7 @@ import json import os import re -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any from gooddata_sdk import GoodDataSdk @@ -14,7 +14,7 @@ from gooddata_eval.core.agentic._catalog import CatalogMetricAlert from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.models import ToolCallEvent +from gooddata_eval.core.models import AgenticEvalOutcome, ToolCallEvent try: from openai import OpenAI as _OpenAI @@ -333,6 +333,8 @@ class AlertRunResult: alert_id: str | None eval: AlertEvaluation actual_alert_arguments: dict + reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None @dataclass @@ -481,6 +483,8 @@ def _run_once(conv_id: str) -> AlertRunResult: alert_id: str | None = None actual_args: dict = {} tool_called = False + reasoning_steps: list[str] = [] + response_id: str | None = None # conversation_history stores prior turns for GPT-4o context. # Roles follow GPT-4o's perspective: "assistant"=agent text, "user"=sim-user reply. conversation_history: list = [] @@ -488,6 +492,8 @@ def _run_once(conv_id: str) -> AlertRunResult: for _iteration in range(max_iterations): chat_result = client.send_message(conv_id, current_question) + reasoning_steps.extend(chat_result.reasoning_steps or []) + response_id = chat_result.response_id or response_id alert_id, actual_args, tool_called = _extract_alert_call(chat_result.tool_call_events or []) if tool_called: alert_id_to_delete = alert_id @@ -523,6 +529,8 @@ def _run_once(conv_id: str) -> AlertRunResult: alert_id=alert_id, eval=ev, actual_alert_arguments=actual_args, + reasoning_steps=reasoning_steps, + response_id=response_id, ) finally: if alert_id_to_delete: @@ -573,6 +581,9 @@ class AlertSkillAssertionError(AssertionError): """Raised when an alert-skill evaluation fails.""" __tracebackhide__ = True + reasoning_steps: list[str] + conversation_id: str + response_id: str | None def evaluate_agentic_alert_skill( @@ -592,8 +603,16 @@ def evaluate_agentic_alert_skill( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, -) -> None: - """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure.""" +) -> AgenticEvalOutcome: + """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure. + + Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an + AgenticEvalOutcome on success; on failure the same three values are attached to the + raised exception as + ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors the + `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them + either way. + """ from datetime import datetime as _dt # noqa: PLC0415 from datetime import timezone as _tz # noqa: PLC0415 @@ -667,7 +686,7 @@ def evaluate_agentic_alert_skill( if not summary.pass_at_k: best = summary.best ev = best.eval - raise AlertSkillAssertionError( + exc = AlertSkillAssertionError( f"Alert skill assertion failed. strict_pass={ev.strict_pass}. " f"alert_created={ev.alert_created}, operator_correct={ev.operator_correct}, " f"threshold_correct={ev.threshold_correct}, trigger_correct={ev.trigger_correct}, " @@ -675,3 +694,12 @@ def evaluate_agentic_alert_skill( f"recipients_correct={ev.recipients_correct}. " f"Actual args: {best.actual_alert_arguments}" ) + exc.reasoning_steps = best.reasoning_steps + exc.conversation_id = best.conversation_id + exc.response_id = best.response_id + raise exc + return AgenticEvalOutcome( + reasoning_steps=summary.best.reasoning_steps, + conversation_id=summary.best.conversation_id, + response_id=summary.best.response_id, + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 1e50352c2..95821bb43 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -5,7 +5,7 @@ import json import re -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Literal from gooddata_sdk import GoodDataSdk @@ -15,7 +15,7 @@ from gooddata_eval.core.agentic.metric_skill import _delete_metric, _extract_created_metric_ids from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.models import ChatResult, ToolCallEvent +from gooddata_eval.core.models import AgenticEvalOutcome, ChatResult, ToolCallEvent from gooddata_eval.core.scoring import ( check_filters, check_viz_type, @@ -265,6 +265,8 @@ class ConversationResult: full_skill_coverage: bool conversation_success: bool total_clarification_turns: int + reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None def run_agentic_conversation( @@ -296,6 +298,8 @@ def run_agentic_conversation( # not persist in the (shared) workspace and get reused by a later test. Deferred to # the end — a later turn may $ref a metric an earlier turn created. created_metric_ids: list[str] = [] + reasoning_steps: list[str] = [] + response_id: str | None = None try: if initial_conversation_id is not None: @@ -332,6 +336,8 @@ def run_agentic_conversation( chat_result = client.send_message(conversation_id, current_message) final_result = chat_result all_tool_calls.extend(chat_result.tool_call_events or []) + reasoning_steps.extend(chat_result.reasoning_steps or []) + response_id = chat_result.response_id or response_id if _check_output_present(resolved_turn, chat_result): break @@ -395,6 +401,8 @@ def run_agentic_conversation( full_skill_coverage=full_skill_coverage, conversation_success=conversation_success, total_clarification_turns=total_clarification_turns, + reasoning_steps=reasoning_steps, + response_id=response_id, ) @@ -402,6 +410,9 @@ class ConversationAssertionError(AssertionError): """Raised when a conversation evaluation fails.""" __tracebackhide__ = True + reasoning_steps: list[str] + conversation_id: str + response_id: str | None def evaluate_agentic_conversation( @@ -419,8 +430,16 @@ def evaluate_agentic_conversation( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, -) -> None: - """Run conversation evaluation, log to Langfuse, and raise on failure.""" +) -> AgenticEvalOutcome: + """Run conversation evaluation, log to Langfuse, and raise on failure. + + Returns the conversation's outcome (reasoning_steps, conversation_id, response_id) as + an AgenticEvalOutcome on success; on failure the same three values are attached to the + raised exception as + ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors the + `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them + either way. + """ from datetime import datetime as _dt # noqa: PLC0415 from datetime import timezone as _tz # noqa: PLC0415 @@ -497,8 +516,17 @@ def evaluate_agentic_conversation( if not result.conversation_success: failed_turns = [tr for tr in result.turn_results if not tr.skill_success] - raise ConversationAssertionError( + exc = ConversationAssertionError( f"Conversation assertion failed. " f"full_skill_coverage={result.full_skill_coverage}. " f"Failed turns: {[t.turn_id for t in failed_turns]}" ) + exc.reasoning_steps = result.reasoning_steps + exc.conversation_id = result.conversation_id + exc.response_id = result.response_id + raise exc + return AgenticEvalOutcome( + reasoning_steps=result.reasoning_steps, + conversation_id=result.conversation_id, + response_id=result.response_id, + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index 34d3f0324..a699ebe63 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -5,14 +5,14 @@ import os import re -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any from gooddata_sdk import GoodDataSdk from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.models import ToolCallEvent +from gooddata_eval.core.models import AgenticEvalOutcome, ToolCallEvent try: from openai import OpenAI as _OpenAI @@ -129,6 +129,8 @@ class MetricRunResult: actual_maql: str maql_correct: bool total_turns: float + reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None @dataclass @@ -205,11 +207,15 @@ def _execute_single_metric_run( metric_id_to_delete: str | None = None turns = 0 current_question = question + reasoning_steps: list[str] = [] + response_id: str | None = None try: for _iteration in range(max_iterations): turns += 1 chat_result = client.send_message(conversation_id, current_question) + reasoning_steps.extend(chat_result.reasoning_steps or []) + response_id = chat_result.response_id or response_id candidate = _extract_metric_result(chat_result.tool_call_events or []) if candidate is not None: metric_result = candidate @@ -236,6 +242,8 @@ def _execute_single_metric_run( actual_maql=actual_maql, maql_correct=maql_correct, total_turns=float(turns), + reasoning_steps=reasoning_steps, + response_id=response_id, ) finally: if metric_id_to_delete: @@ -306,6 +314,9 @@ class MetricSkillAssertionError(AssertionError): """Raised when a metric-skill evaluation fails.""" __tracebackhide__ = True + reasoning_steps: list[str] + conversation_id: str + response_id: str | None def evaluate_agentic_metric_skill( @@ -325,8 +336,16 @@ def evaluate_agentic_metric_skill( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, -) -> None: - """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure.""" +) -> AgenticEvalOutcome: + """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure. + + Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an + AgenticEvalOutcome on success; on failure the same three values are attached to the + raised exception as + ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors the + `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them + either way. + """ from datetime import datetime as _dt # noqa: PLC0415 from datetime import timezone as _tz # noqa: PLC0415 @@ -391,9 +410,18 @@ def evaluate_agentic_metric_skill( best = summary.best expected_outputs_list: list[dict] = expected_output if isinstance(expected_output, list) else [expected_output] candidates_str = "; ".join(repr(c.get("maql", "")) for c in expected_outputs_list) - raise MetricSkillAssertionError( + exc = MetricSkillAssertionError( f"Metric skill assertion failed. " f"metric_created={best.metric_created}, maql_correct={best.maql_correct}. " f"Expected MAQL (candidates): {candidates_str}. " f"Actual MAQL: {best.actual_maql}." ) + exc.reasoning_steps = best.reasoning_steps + exc.conversation_id = best.conversation_id + exc.response_id = best.response_id + raise exc + return AgenticEvalOutcome( + reasoning_steps=summary.best.reasoning_steps, + conversation_id=summary.best.conversation_id, + response_id=summary.best.response_id, + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index d1e375c97..eb85a5f28 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -188,6 +188,7 @@ def _build_chat_result(acc: _SseAccumulator) -> ChatResult: "alertProposals": acc.alert_proposals, "toolCallEvents": acc.tool_call_events, "reasoningStepCount": len(acc.reasoning_steps), + "reasoningSteps": [step["summary"] for step in acc.reasoning_steps], } if acc.visualizations: payload["createdVisualizations"] = { diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index ee58dcc80..0c44cc114 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -98,6 +98,7 @@ class ChatResult(BaseModel): alert_proposals: list[dict] = Field(default_factory=list, alias="alertProposals") tool_call_events: list[ToolCallEvent] = Field(default_factory=list, alias="toolCallEvents") reasoning_step_count: int = Field(default=0, alias="reasoningStepCount") + reasoning_steps: list[str] = Field(default_factory=list, alias="reasoningSteps") conversation_id: str | None = Field(default=None, alias="conversationId") response_id: str | None = Field(default=None, alias="responseId") # True once gen-ai's response_ended event arrived. @@ -106,6 +107,14 @@ class ChatResult(BaseModel): turn_wall_clock_sec: float | None = None +class AgenticEvalOutcome(BaseModel): + """Reasoning trace and trace-lookup IDs returned by an evaluate_agentic_* call on success.""" + + reasoning_steps: list[str] = Field(default_factory=list) + conversation_id: str | None = None + response_id: str | None = None + + class SummaryInput(BaseModel): """Structured input for the `dashboard_summary` test kind. diff --git a/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py b/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py index d4c7b4a3e..1a28e0001 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py @@ -36,6 +36,7 @@ def _build_run_dict(report: EvalReport) -> dict: "detail": item.best_detail, "conversation_id": item.conversation_id, "response_id": item.response_id, + "reasoning": item.reasoning_steps, } for item in report.items }, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/runner.py b/packages/gooddata-eval/src/gooddata_eval/core/runner.py index 5bd56c9bf..0161cc3f8 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/runner.py @@ -33,6 +33,7 @@ class ItemReport: best_detail: dict = field(default_factory=dict) conversation_id: str | None = None response_id: str | None = None + reasoning_steps: list[str] = field(default_factory=list) @property def avg_latency_s(self) -> float: @@ -116,6 +117,7 @@ def _run_one_item( chat_result = backend.ask(item) report.conversation_id = getattr(chat_result, "conversation_id", None) or report.conversation_id report.response_id = getattr(chat_result, "response_id", None) or report.response_id + report.reasoning_steps = getattr(chat_result, "reasoning_steps", None) or report.reasoning_steps evaluation = evaluator.evaluate(item, chat_result) latency = time.perf_counter() - t0 report.runs += 1 diff --git a/packages/gooddata-eval/tests/test_agentic_alert_skill.py b/packages/gooddata-eval/tests/test_agentic_alert_skill.py index 1a5bc55b3..d2241b1f5 100644 --- a/packages/gooddata-eval/tests/test_agentic_alert_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_alert_skill.py @@ -2,14 +2,17 @@ # SPDX-License-Identifier: LicenseRef-GoodData-Enterprise from unittest.mock import MagicMock, patch +import pytest from gooddata_eval.core.agentic.alert_skill import ( AlertEvaluation, + AlertSkillAssertionError, _check_filters, _check_recipients, _check_trigger, _deep_subset, _normalize_expected_output, _to_number, + evaluate_agentic_alert_skill, generate_simulated_alert_response, render_alert_proposal, run_agentic_alert_skill, @@ -567,3 +570,119 @@ def test_run_agentic_alert_skill_answers_proposal_only_confirmation_turn(): assert "admin@gooddata.com" in agent_message assert summary.best.eval.alert_created is True assert summary.best.alert_id == "alert-1" + + +def test_run_agentic_alert_skill_accumulates_reasoning_steps_across_iterations(): + proposal_turn = ChatResult.model_validate( + { + "text_response": None, + "alertProposals": [_PROPOSAL], + "toolCallEvents": [ + {"functionName": "prepare_metric_alert_proposal", "functionArguments": "{}", "result": None} + ], + "reasoningSteps": ["step one"], + } + ) + created_turn = ChatResult.model_validate( + { + "text_response": "Alert created.", + "toolCallEvents": [ + { + "functionName": "create_metric_alert", + "functionArguments": '{"operator": "GREATER_THAN", "threshold": 500}', + "result": '{"id": "alert-1"}', + } + ], + "reasoningSteps": ["step two"], + } + ) + mock_client = MagicMock() + mock_client.send_message.side_effect = [proposal_turn, created_turn] + + with ( + patch("gooddata_eval.core.agentic.alert_skill.ChatClient", return_value=mock_client), + patch( + "gooddata_eval.core.agentic.alert_skill.generate_simulated_alert_response", + return_value="Yes, please proceed to create the alert.", + ), + patch("gooddata_eval.core.agentic.alert_skill._delete_alert"), + ): + summary = run_agentic_alert_skill( + host="http://host", + token="tok", + workspace_id="ws1", + question="Notify me whenever the number of orders goes above 500", + expected_output={"operator": "GREATER_THAN", "threshold": 500}, + k=1, + max_iterations=6, + initial_conversation_id="conv-1", + ) + + assert summary.best.reasoning_steps == ["step one", "step two"] + + +def test_evaluate_agentic_alert_skill_returns_reasoning_steps_on_pass(): + chat_result = ChatResult.model_validate( + { + "text_response": "Alert created.", + "toolCallEvents": [ + { + "functionName": "create_metric_alert", + "functionArguments": '{"operator": "GREATER_THAN", "threshold": 500}', + "result": '{"id": "alert-1"}', + } + ], + "reasoningSteps": ["thinking about it"], + } + ) + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = chat_result + + with ( + patch("gooddata_eval.core.agentic.alert_skill.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.alert_skill._delete_alert"), + ): + outcome = evaluate_agentic_alert_skill( + host="http://host", + token="tok", + workspace_id="ws1", + question="Notify me whenever the number of orders goes above 500", + expected_output={"operator": "GREATER_THAN", "threshold": 500}, + k=1, + max_iterations=1, + ) + + assert outcome.reasoning_steps == ["thinking about it"] + assert outcome.conversation_id == "conv-1" + assert outcome.response_id is None + + +def test_evaluate_agentic_alert_skill_attaches_reasoning_steps_to_exception_on_fail(): + chat_result = ChatResult.model_validate( + { + "text_response": "I cannot create the alert", + "toolCallEvents": [], + "reasoningSteps": ["confused thinking"], + } + ) + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = chat_result + + with ( + patch("gooddata_eval.core.agentic.alert_skill.ChatClient", return_value=mock_client), + pytest.raises(AlertSkillAssertionError) as exc_info, + ): + evaluate_agentic_alert_skill( + host="http://host", + token="tok", + workspace_id="ws1", + question="Create alert", + expected_output={"operator": "GREATER_THAN", "threshold": 100}, + k=1, + max_iterations=1, + ) + assert exc_info.value.reasoning_steps == ["confused thinking"] + assert exc_info.value.conversation_id == "conv-1" + assert exc_info.value.response_id is None diff --git a/packages/gooddata-eval/tests/test_agentic_conversation.py b/packages/gooddata-eval/tests/test_agentic_conversation.py index cb2970272..a6368683b 100644 --- a/packages/gooddata-eval/tests/test_agentic_conversation.py +++ b/packages/gooddata-eval/tests/test_agentic_conversation.py @@ -4,10 +4,12 @@ import pytest from gooddata_eval.core.agentic.conversation import ( + ConversationAssertionError, ConversationFixture, TurnDefinition, TurnResult, _resolve_refs, + evaluate_agentic_conversation, run_agentic_conversation, ) from gooddata_eval.core.models import ChatResult, ToolCallEvent @@ -467,3 +469,134 @@ def test_run_agentic_conversation_records_a_failed_turn_when_a_ref_cannot_be_res assert result.turn_results[1].no_error is False assert result.turn_results[2].skill_success is True assert result.conversation_success is False + + +def test_run_agentic_conversation_accumulates_reasoning_steps_across_turns(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + tc = MagicMock(spec=ToolCallEvent) + tc.function_name = "set_skills" + tc.parsed_arguments = lambda: {"skills": ["visualization"]} + + turn1_result = MagicMock() + turn1_result.text_response = "Here is your visualization" + turn1_result.created_visualizations = [MagicMock()] + turn1_result.tool_call_events = [tc] + turn1_result.reasoning_steps = ["turn one reasoning"] + + turn2_result = MagicMock() + turn2_result.text_response = "Here is another visualization" + turn2_result.created_visualizations = [MagicMock()] + turn2_result.tool_call_events = [tc] + turn2_result.reasoning_steps = ["turn two reasoning"] + + mock_client.send_message.side_effect = [turn1_result, turn2_result] + + fixture = ConversationFixture( + id="test-reasoning", + expected_skills=["visualization"], + turns=[ + TurnDefinition( + turn_id="t1", + message="Make a chart", + expected_skill="visualization", + expected_output_type="visualization", + ), + TurnDefinition( + turn_id="t2", + message="Make another chart", + expected_skill="visualization", + expected_output_type="visualization", + ), + ], + ) + with patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client): + result = run_agentic_conversation( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + fixture=fixture, + ) + + assert result.reasoning_steps == ["turn one reasoning", "turn two reasoning"] + + +def test_evaluate_agentic_conversation_returns_reasoning_steps_on_pass(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + tc = MagicMock(spec=ToolCallEvent) + tc.function_name = "set_skills" + tc.parsed_arguments = lambda: {"skills": ["visualization"]} + chat_result = MagicMock() + chat_result.text_response = "Here is your visualization" + chat_result.created_visualizations = [MagicMock()] + chat_result.tool_call_events = [tc] + chat_result.reasoning_steps = ["thinking about it"] + chat_result.response_id = "resp-1" + mock_client.send_message.return_value = chat_result + + fixture = ConversationFixture( + id="test-1", + expected_skills=["visualization"], + turns=[ + TurnDefinition( + turn_id="t1", + message="Make a chart", + expected_skill="visualization", + expected_output_type="visualization", + ) + ], + ) + with patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client): + outcome = evaluate_agentic_conversation( + host="http://host", + token="tok", + workspace_id="ws1", + fixture=fixture, + ) + assert outcome.reasoning_steps == ["thinking about it"] + assert outcome.conversation_id == "conv-1" + assert outcome.response_id == "resp-1" + + +def test_evaluate_agentic_conversation_attaches_reasoning_steps_to_exception_on_fail(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + tc = MagicMock(spec=ToolCallEvent) + tc.function_name = "set_skills" + tc.parsed_arguments = lambda: {"skills": ["other_skill"]} + chat_result = MagicMock() + chat_result.text_response = "Here is something else" + chat_result.created_visualizations = None + chat_result.tool_call_events = [tc] + chat_result.alert_proposals = [] + chat_result.reasoning_steps = ["confused thinking"] + chat_result.response_id = "resp-2" + mock_client.send_message.return_value = chat_result + + fixture = ConversationFixture( + id="test-1", + expected_skills=["visualization"], + turns=[ + TurnDefinition( + turn_id="t1", + message="Make a chart", + expected_skill="visualization", + expected_output_type="visualization", + ) + ], + ) + with ( + patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client), + pytest.raises(ConversationAssertionError) as exc_info, + ): + evaluate_agentic_conversation( + host="http://host", + token="tok", + workspace_id="ws1", + fixture=fixture, + max_clarification_turns=0, + ) + assert exc_info.value.reasoning_steps == ["confused thinking"] + assert exc_info.value.conversation_id == "conv-1" + assert exc_info.value.response_id == "resp-2" diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill.py b/packages/gooddata-eval/tests/test_agentic_metric_skill.py index 5f9dec88a..684436ee1 100644 --- a/packages/gooddata-eval/tests/test_agentic_metric_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_metric_skill.py @@ -9,9 +9,11 @@ from gooddata_eval.core.agentic.metric_skill import ( AgenticMetricSummary, MetricRunResult, + MetricSkillAssertionError, SimulatedResponseError, _delete_metric, _normalize_maql, + evaluate_agentic_metric_skill, generate_simulated_response, run_agentic_metric_skill, ) @@ -318,3 +320,104 @@ def test_run_agentic_metric_skill_fails_the_run_when_the_simulated_reply_cannot_ assert summary.best.total_turns == 1.0 mock_client.close.assert_called_once() mock_sim.assert_called_once_with("Which brand field should I count?", {"maql": "SELECT {metric/foo}"}) + + +def test_run_agentic_metric_skill_accumulates_reasoning_steps_across_iterations(): + clarify_turn = ChatResult.model_validate( + { + "textResponse": "Could you clarify which foo you mean?", + "toolCallEvents": [], + "reasoningSteps": ["step one"], + } + ) + created_turn = ChatResult.model_validate( + { + "textResponse": "done", + "toolCallEvents": [ + { + "functionName": "create_metric", + "functionArguments": "{}", + "result": '{"data": {"maql": "SELECT {metric/foo}"}}', + } + ], + "reasoningSteps": ["step two"], + } + ) + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.side_effect = [clarify_turn, created_turn] + + with ( + patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.metric_skill.generate_simulated_response", return_value="It's foo"), + ): + summary = run_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=1, + max_iterations=2, + ) + + assert summary.best.reasoning_steps == ["step one", "step two"] + + +def test_evaluate_agentic_metric_skill_returns_reasoning_steps_on_pass(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = ChatResult.model_validate( + { + "textResponse": "done", + "toolCallEvents": [ + { + "functionName": "create_metric", + "functionArguments": "{}", + "result": '{"data": {"maql": "SELECT {metric/foo}"}}', + } + ], + "reasoningSteps": ["thinking about it"], + } + ) + with patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client): + outcome = evaluate_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=1, + max_iterations=1, + ) + assert outcome.reasoning_steps == ["thinking about it"] + assert outcome.conversation_id == "conv-1" + assert outcome.response_id is None + + +def test_evaluate_agentic_metric_skill_attaches_reasoning_steps_to_exception_on_fail(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = ChatResult.model_validate( + { + "textResponse": "I will work on that.", + "toolCallEvents": [], + "reasoningSteps": ["confused thinking"], + } + ) + with ( + patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client), + pytest.raises(MetricSkillAssertionError) as exc_info, + ): + evaluate_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=1, + max_iterations=1, + ) + assert exc_info.value.reasoning_steps == ["confused thinking"] + assert exc_info.value.conversation_id == "conv-1" + assert exc_info.value.response_id is None diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index 3cbff3137..627afebd9 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -1,9 +1,11 @@ # (C) 2026 GoodData Corporation. All rights reserved. +# SPDX-License-Identifier: LicenseRef-GoodData-Enterprise from unittest.mock import patch import pytest -from gooddata_eval.cli.agentic_runner import _dispatch_agentic -from gooddata_eval.core.models import DatasetItem +from gooddata_eval.cli.agentic_runner import _dispatch_agentic, run_agentic_items +from gooddata_eval.core.agentic.alert_skill import AlertSkillAssertionError +from gooddata_eval.core.models import AgenticEvalOutcome, DatasetItem def test_dispatch_agentic_passes_agent_id_through_to_alert_skill(): @@ -91,3 +93,85 @@ def test_dispatch_agentic_passes_agent_id_through_for_every_kind(kind, expected_ agent_id="agent-1", ) assert mock_eval.call_args.kwargs["agent_id"] == "agent-1" + + +def _item(test_kind: str = "agentic_alert_skill") -> DatasetItem: + return DatasetItem( + id="item-1", + dataset_name="d", + test_kind=test_kind, + question="Alert me when revenue drops below 100.", + expected_output={"operator": "LESS_THAN", "threshold": 100}, + ) + + +def test_run_agentic_items_surfaces_reasoning_steps_on_pass(): + with patch( + "gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", + return_value=AgenticEvalOutcome( + reasoning_steps=["it created the alert"], conversation_id="conv-1", response_id="resp-1" + ), + ): + report = run_agentic_items( + [_item()], + host="http://host", + token="tok", + workspace_id="ws1", + run_ts="2026-01-01", + ) + assert report.items[0].pass_at_k is True + assert report.items[0].reasoning_steps == ["it created the alert"] + assert report.items[0].conversation_id == "conv-1" + assert report.items[0].response_id == "resp-1" + + +def test_run_agentic_items_surfaces_reasoning_steps_from_exception_on_fail(): + exc = AlertSkillAssertionError("nope") + exc.reasoning_steps = ["it got confused"] + exc.conversation_id = "conv-2" + exc.response_id = "resp-2" + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=exc): + report = run_agentic_items( + [_item()], + host="http://host", + token="tok", + workspace_id="ws1", + run_ts="2026-01-01", + ) + assert report.items[0].pass_at_k is False + assert report.items[0].reasoning_steps == ["it got confused"] + assert report.items[0].conversation_id == "conv-2" + assert report.items[0].response_id == "resp-2" + + +def test_run_agentic_items_defaults_reasoning_steps_to_empty_when_exception_has_none(): + with patch( + "gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", + side_effect=AlertSkillAssertionError("nope"), + ): + report = run_agentic_items( + [_item()], + host="http://host", + token="tok", + workspace_id="ws1", + run_ts="2026-01-01", + ) + assert report.items[0].reasoning_steps == [] + assert report.items[0].conversation_id is None + assert report.items[0].response_id is None + + +def test_run_agentic_items_defaults_reasoning_steps_to_empty_for_untouched_kinds(): + # general_question/guardrail/search_tool/visualization still return None -- unchanged. + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_guardrail", return_value=None): + report = run_agentic_items( + [_item(test_kind="agentic_guardrail")], + host="http://host", + token="tok", + workspace_id="ws1", + run_ts="2026-01-01", + ) + assert report.items[0].pass_at_k is True + assert report.items[0].reasoning_steps == [] + assert report.items[0].conversation_id is None + assert report.items[0].response_id is None diff --git a/packages/gooddata-eval/tests/test_reporting.py b/packages/gooddata-eval/tests/test_reporting.py index f191de38e..467fda7fb 100644 --- a/packages/gooddata-eval/tests/test_reporting.py +++ b/packages/gooddata-eval/tests/test_reporting.py @@ -23,6 +23,7 @@ def _report() -> EvalReport: pass_at_k=True, runs=2, latency_s=2.5, + reasoning_steps=["step one", "step two"], ), ItemReport( id="i2", @@ -48,6 +49,8 @@ def test_build_json_report_keyed_by_item_id(): assert data["items"]["i1"]["pass_at_k"] is True assert data["items"]["i1"]["latency_s"] == 2.5 assert data["items"]["i1"]["avg_latency_s"] == 1.25 + assert data["items"]["i1"]["reasoning"] == ["step one", "step two"] + assert data["items"]["i2"]["reasoning"] == [] def test_write_json_report_creates_file(tmp_path): diff --git a/packages/gooddata-eval/tests/test_runner.py b/packages/gooddata-eval/tests/test_runner.py index a3f1742b1..925c214a6 100644 --- a/packages/gooddata-eval/tests/test_runner.py +++ b/packages/gooddata-eval/tests/test_runner.py @@ -258,6 +258,41 @@ def ask(self, item: DatasetItem) -> ChatResult: assert "conversation_id" not in report.items[0].error +def test_run_items_carries_reasoning_steps_from_chat_result(): + """reasoning_steps from the ChatResult surfaces on the item report, same as conversation_id.""" + + class _ReasoningBackend: + def ask(self, item: DatasetItem) -> ChatResult: + return ChatResult.model_validate( + {"textResponse": "which metric?", "reasoningSteps": ["step one", "step two"]} + ) + + report = run_items([_item()], _ReasoningBackend(), runs=1) + assert report.items[0].reasoning_steps == ["step one", "step two"] + + +def test_run_items_reasoning_steps_empty_when_chat_result_has_none(): + backend = _FakeBackend([_empty_chat()]) + report = run_items([_item()], backend, runs=1) + assert report.items[0].reasoning_steps == [] + + +def test_run_items_reasoning_steps_keeps_earlier_run_when_later_run_is_empty(): + """A later run with no reasoning events must not clobber an earlier run's steps (runner.py:120's `or`).""" + + class _MixedReasoningBackend: + def __init__(self): + self.calls = 0 + + def ask(self, item: DatasetItem) -> ChatResult: + self.calls += 1 + steps = ["step one", "step two"] if self.calls == 1 else [] + return ChatResult.model_validate({"textResponse": "answer", "reasoningSteps": steps}) + + report = run_items([_item()], _MixedReasoningBackend(), runs=2) + assert report.items[0].reasoning_steps == ["step one", "step two"] + + def test_run_items_callback_exception_is_logged_not_swallowed(capsys): """A raising callback prints a traceback to stderr but the run continues.""" backend = _FakeBackend([_chat_with(_viz_obj())] * 2) diff --git a/packages/gooddata-eval/tests/test_sse_client.py b/packages/gooddata-eval/tests/test_sse_client.py index 6361410ba..c02b1466f 100644 --- a/packages/gooddata-eval/tests/test_sse_client.py +++ b/packages/gooddata-eval/tests/test_sse_client.py @@ -175,9 +175,17 @@ def test_parse_sse_lines_counts_reasoning_steps(): ] result = parse_sse_lines(lines) assert result.reasoning_step_count == 2 + assert result.reasoning_steps == ["step one", "step two"] assert result.text_response == "Done" +def test_parse_sse_lines_reasoning_steps_empty_when_no_reasoning_events(): + lines = ['data: {"item": {"role": "assistant", "content": {"type": "text", "text": "Done"}}}'] + result = parse_sse_lines(lines) + assert result.reasoning_step_count == 0 + assert result.reasoning_steps == [] + + def test_parse_sse_lines_prefers_multipart_viz_over_adhoc_fallback(): """Real multipart visualization takes priority over adhoc tool call stash."""