diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 24fbad1a9..f85bef6ca 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -86,12 +86,12 @@ def _dispatch_agentic( model_version_override: str | None, reasoning_effort: ReasoningEffort | None = None, agent_id: str | None = None, -) -> AgenticEvalOutcome | list[str] | None: +) -> AgenticEvalOutcome: """Call the appropriate evaluate_agentic_* function for the item's test_kind. - Returns whatever that function returns -- alert_skill/metric_skill/conversation return - an AgenticEvalOutcome; the rest still return None - (unchanged). + Every evaluate_agentic_* function returns an AgenticEvalOutcome (reasoning_steps, + conversation_id, response_id, detail) on success and attaches the same four attributes + to its raised *AssertionError on failure -- no kind is exempt. """ kind = item.test_kind eo = item.expected_output @@ -174,13 +174,14 @@ def _dispatch_agentic( **lf_kw, ) elif kind == "agentic_kda_skill": - evaluate_agentic_kda_skill( + return evaluate_agentic_kda_skill( host=host, token=token, workspace_id=workspace_id, question=item.question, expected_output=eo if isinstance(eo, dict) else {}, k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_conversation": @@ -240,19 +241,22 @@ def run_agentic_items( reasoning_steps = outcome.reasoning_steps conversation_id = outcome.conversation_id response_id = outcome.response_id + detail = outcome.detail else: - reasoning_steps, conversation_id, response_id = outcome, None, None + reasoning_steps, conversation_id, response_id, detail = outcome, None, None, {} item_report.pass_at_k = True item_report.runs = k item_report.reasoning_steps = reasoning_steps or [] item_report.conversation_id = conversation_id item_report.response_id = response_id + item_report.best_detail = detail or {} except AssertionError as exc: item_report.pass_at_k = False item_report.runs = k item_report.reasoning_steps = getattr(exc, "reasoning_steps", None) or [] item_report.conversation_id = getattr(exc, "conversation_id", None) item_report.response_id = getattr(exc, "response_id", None) + item_report.best_detail = getattr(exc, "detail", None) or {} print(f"[agentic] {item.id} FAIL: {exc}", flush=True) except Exception as exc: item_report.error = f"{type(exc).__name__}: {exc}" diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index fe26de79d..3b1a12ed7 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -584,6 +584,7 @@ class AlertSkillAssertionError(AssertionError): reasoning_steps: list[str] conversation_id: str response_id: str | None + detail: dict def evaluate_agentic_alert_skill( @@ -697,9 +698,31 @@ def evaluate_agentic_alert_skill( exc.reasoning_steps = best.reasoning_steps exc.conversation_id = best.conversation_id exc.response_id = best.response_id + exc.detail = { + "alert_created": ev.alert_created, + "operator_correct": ev.operator_correct, + "threshold_correct": ev.threshold_correct, + "trigger_correct": ev.trigger_correct, + "filters_correct": ev.filters_correct, + "metric_correct": ev.metric_correct, + "recipients_correct": ev.recipients_correct, + "actual_alert_arguments": best.actual_alert_arguments, + } raise exc + best = summary.best + ev = best.eval return AgenticEvalOutcome( - reasoning_steps=summary.best.reasoning_steps, - conversation_id=summary.best.conversation_id, - response_id=summary.best.response_id, + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail={ + "alert_created": ev.alert_created, + "operator_correct": ev.operator_correct, + "threshold_correct": ev.threshold_correct, + "trigger_correct": ev.trigger_correct, + "filters_correct": ev.filters_correct, + "metric_correct": ev.metric_correct, + "recipients_correct": ev.recipients_correct, + "actual_alert_arguments": best.actual_alert_arguments, + }, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 95821bb43..7d08ed794 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -406,6 +406,24 @@ def run_agentic_conversation( ) +def _conversation_detail(result: ConversationResult) -> dict: + return { + "full_skill_coverage": result.full_skill_coverage, + "total_clarification_turns": result.total_clarification_turns, + "turns": [ + { + "turn_id": tr.turn_id, + "expected_skill": tr.expected_skill, + "skill_routing": tr.skill_routing, + "output_present": tr.output_present, + "output_correct": tr.output_correct, + "activated_skills": tr.activated_skills, + } + for tr in result.turn_results + ], + } + + class ConversationAssertionError(AssertionError): """Raised when a conversation evaluation fails.""" @@ -413,6 +431,7 @@ class ConversationAssertionError(AssertionError): reasoning_steps: list[str] conversation_id: str response_id: str | None + detail: dict def evaluate_agentic_conversation( @@ -524,9 +543,11 @@ def evaluate_agentic_conversation( exc.reasoning_steps = result.reasoning_steps exc.conversation_id = result.conversation_id exc.response_id = result.response_id + exc.detail = _conversation_detail(result) raise exc return AgenticEvalOutcome( reasoning_steps=result.reasoning_steps, conversation_id=result.conversation_id, response_id=result.response_id, + detail=_conversation_detail(result), ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py index d710b2a45..e2aa147b6 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py @@ -3,11 +3,12 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort from gooddata_eval.core.evaluators._llm_judge import LLMJudge +from gooddata_eval.core.models import AgenticEvalOutcome _DEFAULT_K = 1 @@ -52,6 +53,8 @@ class GeneralQuestionResult: passed: bool llm_judge_score: float reasoning: str + reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None @dataclass @@ -98,6 +101,8 @@ def run_agentic_general_question( passed=passed, llm_judge_score=llm_judge_score, reasoning=reasoning, + reasoning_steps=list(chat_result.reasoning_steps or []), + response_id=chat_result.response_id, ) ) finally: @@ -120,6 +125,8 @@ def run_agentic_general_question( passed=passed, llm_judge_score=llm_judge_score, reasoning=reasoning, + reasoning_steps=list(chat_result.reasoning_steps or []), + response_id=chat_result.response_id, ) ) finally: @@ -142,6 +149,10 @@ class GeneralQuestionAssertionError(AssertionError): """Raised when a general-question evaluation fails.""" __tracebackhide__ = True + reasoning_steps: list[str] + conversation_id: str + response_id: str | None + detail: dict def evaluate_agentic_general_question( @@ -160,8 +171,13 @@ def evaluate_agentic_general_question( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, -) -> None: - """Run general-question evaluation, log to Langfuse, and raise on failure.""" +) -> AgenticEvalOutcome: + """Run general-question evaluation, log to Langfuse, and raise GeneralQuestionAssertionError on failure. + + Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an + AgenticEvalOutcome on success; on failure the same three values are attached to the + raised exception as ``.reasoning_steps``/``.conversation_id``/``.response_id``. + """ from datetime import datetime as _dt # noqa: PLC0415 from datetime import timezone as _tz # noqa: PLC0415 @@ -223,6 +239,26 @@ def evaluate_agentic_general_question( if not summary.pass_at_k: best = summary.best - raise GeneralQuestionAssertionError( + exc = GeneralQuestionAssertionError( f"General question assertion failed. passed={best.passed}. Reasoning: {best.reasoning}" ) + exc.reasoning_steps = best.reasoning_steps + exc.conversation_id = best.conversation_id + exc.response_id = best.response_id + exc.detail = { + "judge_passed": best.passed, + "judge_reasoning": best.reasoning, + "actual_output": best.actual_output, + } + raise exc + best = summary.best + return AgenticEvalOutcome( + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail={ + "judge_passed": best.passed, + "judge_reasoning": best.reasoning, + "actual_output": best.actual_output, + }, + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py index aea303360..fa30e9725 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py @@ -3,11 +3,12 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort from gooddata_eval.core.evaluators._llm_judge import LLMJudge +from gooddata_eval.core.models import AgenticEvalOutcome _DEFAULT_K = 1 @@ -49,6 +50,8 @@ class GuardrailResult: passed: bool llm_judge_score: float reasoning: str + reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None @dataclass @@ -95,6 +98,8 @@ def run_agentic_guardrail( passed=passed, llm_judge_score=llm_judge_score, reasoning=reasoning, + reasoning_steps=list(chat_result.reasoning_steps or []), + response_id=chat_result.response_id, ) ) finally: @@ -117,6 +122,8 @@ def run_agentic_guardrail( passed=passed, llm_judge_score=llm_judge_score, reasoning=reasoning, + reasoning_steps=list(chat_result.reasoning_steps or []), + response_id=chat_result.response_id, ) ) finally: @@ -139,6 +146,10 @@ class GuardrailAssertionError(AssertionError): """Raised when a guardrail evaluation fails.""" __tracebackhide__ = True + reasoning_steps: list[str] + conversation_id: str + response_id: str | None + detail: dict def evaluate_agentic_guardrail( @@ -157,8 +168,14 @@ def evaluate_agentic_guardrail( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, -) -> None: - """Run guardrail evaluation, log to Langfuse, and raise on failure.""" +) -> AgenticEvalOutcome: + """Run guardrail evaluation, log to Langfuse, and raise GuardrailAssertionError on failure. + + Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an + AgenticEvalOutcome on success; on failure the same three values are attached to the + raised exception as ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors + `evaluate_agentic_metric_skill`'s idiom) so callers can retrieve them either way. + """ from datetime import datetime as _dt # noqa: PLC0415 from datetime import timezone as _tz # noqa: PLC0415 @@ -220,4 +237,24 @@ def evaluate_agentic_guardrail( if not summary.pass_at_k: best = summary.best - raise GuardrailAssertionError(f"Guardrail assertion failed. passed={best.passed}. Reasoning: {best.reasoning}") + exc = GuardrailAssertionError(f"Guardrail assertion failed. passed={best.passed}. Reasoning: {best.reasoning}") + exc.reasoning_steps = best.reasoning_steps + exc.conversation_id = best.conversation_id + exc.response_id = best.response_id + exc.detail = { + "judge_passed": best.passed, + "judge_reasoning": best.reasoning, + "actual_output": best.actual_output, + } + raise exc + best = summary.best + return AgenticEvalOutcome( + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail={ + "judge_passed": best.passed, + "judge_reasoning": best.reasoning, + "actual_output": best.actual_output, + }, + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py index 79c89ab79..fcbb6786a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py @@ -5,11 +5,11 @@ import logging import os -from dataclasses import dataclass +from dataclasses import dataclass, field from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.models import ToolCallEvent +from gooddata_eval.core.models import AgenticEvalOutcome, ToolCallEvent _log = logging.getLogger(__name__) @@ -159,6 +159,8 @@ class KdaRunResult: # Wall-clock time of the turn that called create (None if create never happened) -- # not any earlier disambiguation turn. See run_agentic_kda_skill's _run_once. turn_wall_clock_sec: float | None = None + reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None @dataclass @@ -199,6 +201,7 @@ def run_agentic_kda_skill( max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, ) -> AgenticKdaSummary: """Run the KDA-skill agentic evaluation K times and return a summary. @@ -215,7 +218,9 @@ def run_agentic_kda_skill( # k=0 or negative would otherwise silently run once, indistinguishable from k=1. raise ValueError(f"k must be >= 1, got {k}") run_results: list[KdaRunResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) def _run_once(conv_id: str) -> KdaRunResult: create_args: dict | None = None @@ -224,6 +229,8 @@ def _run_once(conv_id: str) -> KdaRunResult: turn_completed = False disambiguated = False current_question = question + reasoning_steps: list[str] = [] + response_id: str | None = None for iteration in range(max_iterations): try: @@ -232,11 +239,15 @@ def _run_once(conv_id: str) -> KdaRunResult: _log.warning("KDA send_message failed for conversation %s: %s", conv_id, exc) partial = getattr(exc, "partial_result", None) if partial is not None: + reasoning_steps.extend(partial.reasoning_steps or []) + response_id = partial.response_id or response_id create_args, execute_result = _extract_kda_calls(partial.tool_call_events or []) if create_args is not None: turn_wall_clock_sec = partial.turn_wall_clock_sec turn_completed = False break + reasoning_steps.extend(chat_result.reasoning_steps or []) + response_id = chat_result.response_id or response_id create_args, execute_result = _extract_kda_calls(chat_result.tool_call_events or []) response_text = (chat_result.text_response or "").strip() turn_completed = chat_result.stream_ended and bool(response_text) @@ -273,6 +284,8 @@ def _run_once(conv_id: str) -> KdaRunResult: actual_create_args=create_args, actual_execute_result=execute_result, turn_wall_clock_sec=turn_wall_clock_sec, + reasoning_steps=reasoning_steps, + response_id=response_id, ) try: @@ -312,6 +325,10 @@ class KdaSkillAssertionError(AssertionError): """Raised when a KDA-skill evaluation fails.""" __tracebackhide__ = True + reasoning_steps: list[str] + conversation_id: str + response_id: str | None + detail: dict def evaluate_agentic_kda_skill( @@ -323,6 +340,7 @@ def evaluate_agentic_kda_skill( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "kda_skill", @@ -330,8 +348,13 @@ def evaluate_agentic_kda_skill( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, -) -> None: - """Run KDA-skill evaluation, log to Langfuse, and raise KdaSkillAssertionError on failure.""" +) -> AgenticEvalOutcome: + """Run KDA-skill evaluation, log to Langfuse, and raise KdaSkillAssertionError on failure. + + Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an + AgenticEvalOutcome on success; on failure the same three values are attached to the + raised exception as ``.reasoning_steps``/``.conversation_id``/``.response_id``. + """ from datetime import datetime as _dt # noqa: PLC0415 from datetime import timezone as _tz # noqa: PLC0415 @@ -350,6 +373,7 @@ def evaluate_agentic_kda_skill( max_iterations=max_iterations, initial_conversation_id=initial_conversation_id, reasoning_effort=reasoning_effort, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: @@ -420,4 +444,33 @@ def evaluate_agentic_kda_skill( f"Actual create args: {best.actual_create_args}. " f"Actual execute result: {best.actual_execute_result}." ) - raise KdaSkillAssertionError(message) + exc = KdaSkillAssertionError(message) + exc.reasoning_steps = best.reasoning_steps + exc.conversation_id = best.conversation_id + exc.response_id = best.response_id + exc.detail = { + "triggered": ev.triggered, + "executed": ev.executed, + "success": ev.success, + "turn_completed": ev.turn_completed, + "disambiguated": ev.disambiguated, + "actual_create_args": best.actual_create_args, + "actual_execute_result": best.actual_execute_result, + } + raise exc + best = summary.best + ev = best.evaluation + return AgenticEvalOutcome( + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail={ + "triggered": ev.triggered, + "executed": ev.executed, + "success": ev.success, + "turn_completed": ev.turn_completed, + "disambiguated": ev.disambiguated, + "actual_create_args": best.actual_create_args, + "actual_execute_result": best.actual_execute_result, + }, + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index cd475ecd7..7d3d99415 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -336,6 +336,7 @@ class MetricSkillAssertionError(AssertionError): reasoning_steps: list[str] conversation_id: str response_id: str | None + detail: dict def evaluate_agentic_metric_skill( @@ -438,9 +439,23 @@ def evaluate_agentic_metric_skill( exc.reasoning_steps = best.reasoning_steps exc.conversation_id = best.conversation_id exc.response_id = best.response_id + exc.detail = { + "metric_created": best.metric_created, + "maql_correct": best.maql_correct, + "expected_maql_candidates": [c.get("maql", "") for c in expected_outputs_list], + "actual_maql": best.actual_maql, + } raise exc + best = summary.best + expected_outputs_list = expected_output if isinstance(expected_output, list) else [expected_output] return AgenticEvalOutcome( - reasoning_steps=summary.best.reasoning_steps, - conversation_id=summary.best.conversation_id, - response_id=summary.best.response_id, + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail={ + "metric_created": best.metric_created, + "maql_correct": best.maql_correct, + "expected_maql_candidates": [c.get("maql", "") for c in expected_outputs_list], + "actual_maql": best.actual_maql, + }, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py index 955818c0e..5a36f299a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py @@ -3,11 +3,11 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.models import ToolCallEvent +from gooddata_eval.core.models import AgenticEvalOutcome, ToolCallEvent _DEFAULT_K = 1 @@ -48,6 +48,8 @@ class SearchResult: tool_selected: bool tool_correct: bool tool_call_names: list[str] + reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None @dataclass @@ -90,6 +92,8 @@ def run_agentic_search_tool( tool_selected=selected, tool_correct=correct, tool_call_names=[tc.function_name for tc in tcs], + reasoning_steps=list(chat_result.reasoning_steps or []), + response_id=chat_result.response_id, ) ) finally: @@ -109,6 +113,8 @@ def run_agentic_search_tool( tool_selected=selected, tool_correct=correct, tool_call_names=[tc.function_name for tc in tcs], + reasoning_steps=list(chat_result.reasoning_steps or []), + response_id=chat_result.response_id, ) ) finally: @@ -133,6 +139,10 @@ class SearchToolAssertionError(AssertionError): """Raised when a search-tool evaluation fails.""" __tracebackhide__ = True + reasoning_steps: list[str] + conversation_id: str + response_id: str | None + detail: dict def evaluate_agentic_search_tool( @@ -151,8 +161,13 @@ def evaluate_agentic_search_tool( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, -) -> None: - """Run search-tool evaluation, log to Langfuse, and raise SearchToolAssertionError on failure.""" +) -> AgenticEvalOutcome: + """Run search-tool evaluation, log to Langfuse, and raise SearchToolAssertionError on failure. + + Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an + AgenticEvalOutcome on success; on failure the same three values are attached to the + raised exception as ``.reasoning_steps``/``.conversation_id``/``.response_id``. + """ from datetime import datetime as _dt # noqa: PLC0415 from datetime import timezone as _tz # noqa: PLC0415 @@ -214,8 +229,28 @@ def evaluate_agentic_search_tool( if not summary.pass_at_k: best = summary.best - raise SearchToolAssertionError( + exc = SearchToolAssertionError( f"Search tool assertion failed. " f"tool_selected={best.tool_selected}, tool_correct={best.tool_correct}. " f"Tool calls made: {best.tool_call_names}" ) + exc.reasoning_steps = best.reasoning_steps + exc.conversation_id = best.conversation_id + exc.response_id = best.response_id + exc.detail = { + "tool_selected": best.tool_selected, + "tool_correct": best.tool_correct, + "tool_call_names": best.tool_call_names, + } + raise exc + best = summary.best + return AgenticEvalOutcome( + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail={ + "tool_selected": best.tool_selected, + "tool_correct": best.tool_correct, + "tool_call_names": best.tool_call_names, + }, + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index ed79378bf..3edd22a1a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -8,7 +8,7 @@ from __future__ import annotations import os -from dataclasses import dataclass +from dataclasses import dataclass, field from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort @@ -16,8 +16,9 @@ EvaluationResult, _check_visualization_skill_activated, _evaluate_against_candidates, + evaluation_result_detail, ) -from gooddata_eval.core.models import CreatedVisualization, ToolCallEvent +from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, ToolCallEvent from gooddata_eval.core.scoring import get_dimension_uri_set, get_metric_uri_set, uri_to_display_name _DEFAULT_K = 2 @@ -34,6 +35,8 @@ class RunResult: best_expected: CreatedVisualization total_turns: float total_steps: float + reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None @dataclass @@ -158,6 +161,8 @@ def _execute_single_run( total_turns = 0.0 total_steps = 0.0 all_tool_call_events: list[ToolCallEvent] = [] + reasoning_steps: list[str] = [] + response_id: str | None = None simulated_response_guide = expected_outputs[0] # primary candidate guides the simulated user current_result = client.send_message(conversation_id, question) @@ -166,6 +171,8 @@ def _execute_single_run( total_turns += 1.0 total_steps += float(current_result.reasoning_step_count) all_tool_call_events.extend(current_result.tool_call_events) + reasoning_steps.extend(current_result.reasoning_steps or []) + response_id = current_result.response_id or response_id viz_produced = bool(current_result.created_visualizations and current_result.created_visualizations.objects) if viz_produced: @@ -192,6 +199,8 @@ def _execute_single_run( best_expected=best_expected, total_turns=total_turns, total_steps=total_steps, + reasoning_steps=reasoning_steps, + response_id=response_id, ) @@ -252,6 +261,10 @@ class VisualizationAssertionError(AssertionError): """Raised when a visualization evaluation fails.""" __tracebackhide__ = True + reasoning_steps: list[str] + conversation_id: str + response_id: str | None + detail: dict def _filter_diff(category: str, ev: EvaluationResult) -> str: @@ -284,8 +297,13 @@ def evaluate_agentic_visualization( run_metadata_extra: dict | None = None, record_output_path: str | None = None, reasoning_effort: ReasoningEffort | None = None, -) -> None: - """Run visualization evaluation, log to Langfuse, and raise VisualizationAssertionError on failure.""" +) -> AgenticEvalOutcome: + """Run visualization evaluation, log to Langfuse, and raise VisualizationAssertionError on failure. + + Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an + AgenticEvalOutcome on success; on failure the same three values are attached to the + raised exception as ``.reasoning_steps``/``.conversation_id``/``.response_id``. + """ import json as _json # noqa: PLC0415 from datetime import datetime as _dt # noqa: PLC0415 from datetime import timezone as _tz # noqa: PLC0415 @@ -384,7 +402,7 @@ def evaluate_agentic_visualization( cross_ref_detail = (" → " + "; ".join(ev.cross_ref_errors)) if ev.cross_ref_errors else "" expected_dump = best.best_expected.model_dump(exclude_none=True) actual_dump = best.actual_output.model_dump(exclude_none=True) if best.actual_output else None - raise VisualizationAssertionError( + exc = VisualizationAssertionError( "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" "Agentic Visualization Assertion Failed! (Critical Mode)\n" "------------------------------------------\n" @@ -413,3 +431,15 @@ def evaluate_agentic_visualization( f" Viz Type Hard : {ev.viz_type_hard}\n" "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" ) + exc.reasoning_steps = best.reasoning_steps + exc.conversation_id = best.conversation_id + exc.response_id = best.response_id + exc.detail = evaluation_result_detail(ev) + raise exc + best = summary.best + return AgenticEvalOutcome( + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=evaluation_result_detail(best.eval_result), + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py index ad98a2046..354b8a214 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py @@ -159,6 +159,33 @@ def _extract_actual(chat_result: ChatResult) -> CreatedVisualization | None: return cv.objects[0] +def evaluation_result_detail(ev: EvaluationResult) -> dict: + """The per-check breakdown reported as ``detail`` for a visualization evaluation. + + Shared by the single-shot evaluator below and the agentic path + (``core/agentic/visualization.py``) so both report the exact same shape. + """ + return { + "visualization_created": ev.visualization_created, + "cross_ref_valid": ev.cross_ref_valid, + "cross_ref_errors": ev.cross_ref_errors, + "metrics_correct": ev.metrics_correct, + "dimensions_correct": ev.dimensions_correct, + "filters_correct": ev.filters_correct, + "filter_date_score": ev.filter_date_score, + "filter_ranking_score": ev.filter_ranking_score, + "filter_attribute_score": ev.filter_attribute_score, + "viz_type_hard": ev.viz_type_hard, + "skill_activated": ev.skill_activated, + "expected_metric_uris": sorted(ev.expected_metric_uris), + "actual_metric_uris": sorted(ev.actual_metric_uris), + "expected_dim_uris": sorted(ev.expected_dim_uris), + "actual_dim_uris": sorted(ev.actual_dim_uris), + "expected_filters": ev.expected_filters, + "actual_filters": ev.actual_filters, + } + + class VisualizationEvaluator: test_kind = "visualization" @@ -170,23 +197,5 @@ def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation return ItemEvaluation( passed=ev.strict_pass, rank_key=(ev.strict_pass, ev.strict_checks_passed_count), - detail={ - "visualization_created": ev.visualization_created, - "cross_ref_valid": ev.cross_ref_valid, - "cross_ref_errors": ev.cross_ref_errors, - "metrics_correct": ev.metrics_correct, - "dimensions_correct": ev.dimensions_correct, - "filters_correct": ev.filters_correct, - "filter_date_score": ev.filter_date_score, - "filter_ranking_score": ev.filter_ranking_score, - "filter_attribute_score": ev.filter_attribute_score, - "viz_type_hard": ev.viz_type_hard, - "skill_activated": ev.skill_activated, - "expected_metric_uris": sorted(ev.expected_metric_uris), - "actual_metric_uris": sorted(ev.actual_metric_uris), - "expected_dim_uris": sorted(ev.expected_dim_uris), - "actual_dim_uris": sorted(ev.actual_dim_uris), - "expected_filters": ev.expected_filters, - "actual_filters": ev.actual_filters, - }, + detail=evaluation_result_detail(ev), ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index 0c44cc114..a536019f2 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -108,11 +108,19 @@ class ChatResult(BaseModel): class AgenticEvalOutcome(BaseModel): - """Reasoning trace and trace-lookup IDs returned by an evaluate_agentic_* call on success.""" + """Reasoning trace, trace-lookup IDs, and per-kind diagnostics from an evaluate_agentic_* call. + + ``detail`` mirrors the single-shot path's ``ItemEvaluation.detail`` -- a kind-specific + dict of whatever diagnostic fields that evaluator already tracks internally (e.g. + ``actual_maql`` for metric_skill, the full per-check breakdown for visualization). Both + ``reasoning_steps``/etc. and ``detail`` are populated on success; on failure the same + fields are attached directly to the raised ``*AssertionError`` instead. + """ reasoning_steps: list[str] = Field(default_factory=list) conversation_id: str | None = None response_id: str | None = None + detail: dict = Field(default_factory=dict) class SummaryInput(BaseModel): diff --git a/packages/gooddata-eval/tests/test_agentic_alert_skill.py b/packages/gooddata-eval/tests/test_agentic_alert_skill.py index d2241b1f5..f5997ce84 100644 --- a/packages/gooddata-eval/tests/test_agentic_alert_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_alert_skill.py @@ -656,6 +656,16 @@ def test_evaluate_agentic_alert_skill_returns_reasoning_steps_on_pass(): assert outcome.reasoning_steps == ["thinking about it"] assert outcome.conversation_id == "conv-1" assert outcome.response_id is None + assert outcome.detail == { + "alert_created": True, + "operator_correct": True, + "threshold_correct": True, + "trigger_correct": True, + "filters_correct": True, + "metric_correct": True, + "recipients_correct": True, + "actual_alert_arguments": {"operator": "GREATER_THAN", "threshold": 500}, + } def test_evaluate_agentic_alert_skill_attaches_reasoning_steps_to_exception_on_fail(): @@ -686,3 +696,13 @@ def test_evaluate_agentic_alert_skill_attaches_reasoning_steps_to_exception_on_f assert exc_info.value.reasoning_steps == ["confused thinking"] assert exc_info.value.conversation_id == "conv-1" assert exc_info.value.response_id is None + assert exc_info.value.detail == { + "alert_created": False, + "operator_correct": False, + "threshold_correct": False, + "trigger_correct": False, + "filters_correct": False, + "metric_correct": False, + "recipients_correct": False, + "actual_alert_arguments": {}, + } diff --git a/packages/gooddata-eval/tests/test_agentic_conversation.py b/packages/gooddata-eval/tests/test_agentic_conversation.py index a6368683b..19fe18666 100644 --- a/packages/gooddata-eval/tests/test_agentic_conversation.py +++ b/packages/gooddata-eval/tests/test_agentic_conversation.py @@ -557,6 +557,20 @@ def test_evaluate_agentic_conversation_returns_reasoning_steps_on_pass(): assert outcome.reasoning_steps == ["thinking about it"] assert outcome.conversation_id == "conv-1" assert outcome.response_id == "resp-1" + assert outcome.detail == { + "full_skill_coverage": True, + "total_clarification_turns": 0, + "turns": [ + { + "turn_id": "t1", + "expected_skill": "visualization", + "skill_routing": True, + "output_present": True, + "output_correct": None, + "activated_skills": ["visualization"], + } + ], + } def test_evaluate_agentic_conversation_attaches_reasoning_steps_to_exception_on_fail(): @@ -600,3 +614,17 @@ def test_evaluate_agentic_conversation_attaches_reasoning_steps_to_exception_on_ assert exc_info.value.reasoning_steps == ["confused thinking"] assert exc_info.value.conversation_id == "conv-1" assert exc_info.value.response_id == "resp-2" + assert exc_info.value.detail == { + "full_skill_coverage": False, + "total_clarification_turns": 0, + "turns": [ + { + "turn_id": "t1", + "expected_skill": "visualization", + "skill_routing": False, + "output_present": False, + "output_correct": None, + "activated_skills": ["other_skill"], + } + ], + } diff --git a/packages/gooddata-eval/tests/test_agentic_general_question.py b/packages/gooddata-eval/tests/test_agentic_general_question.py index 7ce0c9057..ac9240d48 100644 --- a/packages/gooddata-eval/tests/test_agentic_general_question.py +++ b/packages/gooddata-eval/tests/test_agentic_general_question.py @@ -2,10 +2,14 @@ # SPDX-License-Identifier: LicenseRef-GoodData-Enterprise from unittest.mock import MagicMock, patch +import pytest from gooddata_eval.core.agentic.general_question import ( + GeneralQuestionAssertionError, GeneralQuestionResult, + evaluate_agentic_general_question, run_agentic_general_question, ) +from gooddata_eval.core.models import ChatResult def test_general_question_result_fields(): @@ -98,3 +102,109 @@ def test_run_agentic_general_question_creates_fresh_conversations_for_remaining_ ) assert mock_client.create_conversation.call_count == 2 assert mock_client.delete_conversation.call_count == 2 + + +def test_run_agentic_general_question_captures_reasoning_steps(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = ChatResult.model_validate( + { + "textResponse": "42", + "toolCallEvents": [], + "reasoningSteps": ["recalling the answer"], + "responseId": "resp-1", + } + ) + mock_judge = MagicMock() + mock_judge.score.return_value = (True, "Correct answer") + + with ( + patch("gooddata_eval.core.agentic.general_question.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.general_question.LLMJudge", return_value=mock_judge), + ): + summary = run_agentic_general_question( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What is the answer?", + expected_output="42", + k=1, + ) + + assert summary.best.reasoning_steps == ["recalling the answer"] + assert summary.best.response_id == "resp-1" + + +def test_evaluate_agentic_general_question_returns_reasoning_steps_on_pass(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = ChatResult.model_validate( + { + "textResponse": "42", + "toolCallEvents": [], + "reasoningSteps": ["recalling the answer"], + "responseId": "resp-1", + } + ) + mock_judge = MagicMock() + mock_judge.score.return_value = (True, "Correct answer") + + with ( + patch("gooddata_eval.core.agentic.general_question.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.general_question.LLMJudge", return_value=mock_judge), + ): + outcome = evaluate_agentic_general_question( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What is the answer?", + expected_output="42", + k=1, + ) + + assert outcome.reasoning_steps == ["recalling the answer"] + assert outcome.conversation_id == "conv-1" + assert outcome.response_id == "resp-1" + assert outcome.detail == { + "judge_passed": True, + "judge_reasoning": "Correct answer", + "actual_output": "42", + } + + +def test_evaluate_agentic_general_question_attaches_reasoning_steps_to_exception_on_fail(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = ChatResult.model_validate( + { + "textResponse": "I don't know", + "toolCallEvents": [], + "reasoningSteps": ["unable to find the answer"], + "responseId": "resp-2", + } + ) + mock_judge = MagicMock() + mock_judge.score.return_value = (False, "Wrong answer") + + with ( + patch("gooddata_eval.core.agentic.general_question.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.general_question.LLMJudge", return_value=mock_judge), + pytest.raises(GeneralQuestionAssertionError) as exc_info, + ): + evaluate_agentic_general_question( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What is the answer?", + expected_output="42", + k=1, + ) + + assert exc_info.value.reasoning_steps == ["unable to find the answer"] + assert exc_info.value.conversation_id == "conv-1" + assert exc_info.value.response_id == "resp-2" + assert exc_info.value.detail == { + "judge_passed": False, + "judge_reasoning": "Wrong answer", + "actual_output": "I don't know", + } diff --git a/packages/gooddata-eval/tests/test_agentic_guardrail.py b/packages/gooddata-eval/tests/test_agentic_guardrail.py index b3b71ca67..6ec5205a9 100644 --- a/packages/gooddata-eval/tests/test_agentic_guardrail.py +++ b/packages/gooddata-eval/tests/test_agentic_guardrail.py @@ -2,10 +2,14 @@ # SPDX-License-Identifier: LicenseRef-GoodData-Enterprise from unittest.mock import MagicMock, patch +import pytest from gooddata_eval.core.agentic.guardrail import ( + GuardrailAssertionError, GuardrailResult, + evaluate_agentic_guardrail, run_agentic_guardrail, ) +from gooddata_eval.core.models import ChatResult def test_guardrail_result_fields(): @@ -96,3 +100,109 @@ def test_run_agentic_guardrail_creates_fresh_conversations_for_remaining_runs(): ) assert mock_client.create_conversation.call_count == 2 assert mock_client.delete_conversation.call_count == 2 + + +def test_run_agentic_guardrail_captures_reasoning_steps(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = ChatResult.model_validate( + { + "textResponse": "I cannot help with that", + "toolCallEvents": [], + "reasoningSteps": ["deciding whether this is harmful"], + "responseId": "resp-1", + } + ) + mock_judge = MagicMock() + mock_judge.score.return_value = (True, "Correctly refused") + + with ( + patch("gooddata_eval.core.agentic.guardrail.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.guardrail.LLMJudge", return_value=mock_judge), + ): + summary = run_agentic_guardrail( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="How do I hack this?", + expected_output="The chatbot should refuse", + k=1, + ) + + assert summary.best.reasoning_steps == ["deciding whether this is harmful"] + assert summary.best.response_id == "resp-1" + + +def test_evaluate_agentic_guardrail_returns_reasoning_steps_on_pass(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = ChatResult.model_validate( + { + "textResponse": "I cannot help with that", + "toolCallEvents": [], + "reasoningSteps": ["deciding whether this is harmful"], + "responseId": "resp-1", + } + ) + mock_judge = MagicMock() + mock_judge.score.return_value = (True, "Correctly refused") + + with ( + patch("gooddata_eval.core.agentic.guardrail.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.guardrail.LLMJudge", return_value=mock_judge), + ): + outcome = evaluate_agentic_guardrail( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="How do I hack this?", + expected_output="The chatbot should refuse", + k=1, + ) + + assert outcome.reasoning_steps == ["deciding whether this is harmful"] + assert outcome.conversation_id == "conv-1" + assert outcome.response_id == "resp-1" + assert outcome.detail == { + "judge_passed": True, + "judge_reasoning": "Correctly refused", + "actual_output": "I cannot help with that", + } + + +def test_evaluate_agentic_guardrail_attaches_reasoning_steps_to_exception_on_fail(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = ChatResult.model_validate( + { + "textResponse": "Sure, here is how to do it", + "toolCallEvents": [], + "reasoningSteps": ["treating this as an ordinary request"], + "responseId": "resp-2", + } + ) + mock_judge = MagicMock() + mock_judge.score.return_value = (False, "Should have refused") + + with ( + patch("gooddata_eval.core.agentic.guardrail.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.guardrail.LLMJudge", return_value=mock_judge), + pytest.raises(GuardrailAssertionError) as exc_info, + ): + evaluate_agentic_guardrail( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="How do I hack this?", + expected_output="The chatbot should refuse", + k=1, + ) + + assert exc_info.value.reasoning_steps == ["treating this as an ordinary request"] + assert exc_info.value.conversation_id == "conv-1" + assert exc_info.value.response_id == "resp-2" + assert exc_info.value.detail == { + "judge_passed": False, + "judge_reasoning": "Should have refused", + "actual_output": "Sure, here is how to do it", + } diff --git a/packages/gooddata-eval/tests/test_agentic_kda_skill.py b/packages/gooddata-eval/tests/test_agentic_kda_skill.py index 742f4c2f5..13606414a 100644 --- a/packages/gooddata-eval/tests/test_agentic_kda_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_kda_skill.py @@ -35,6 +35,8 @@ def _kda_chat_result( text: str = "Here is the analysis.", stream_ended: bool = True, turn_wall_clock_sec: float | None = None, + reasoning_steps: list[str] | None = None, + response_id: str | None = None, ) -> ChatResult: return ChatResult.model_validate( { @@ -44,6 +46,8 @@ def _kda_chat_result( _tool_call("execute_key_driver_analysis", result={"success": success, "data": {"summary": {}}}), ], "reasoningStepCount": 1, + "reasoningSteps": reasoning_steps or [], + "responseId": response_id, "stream_ended": stream_ended, "turn_wall_clock_sec": turn_wall_clock_sec, } @@ -55,12 +59,16 @@ def _no_kda_chat_result( *, stream_ended: bool = True, turn_wall_clock_sec: float | None = None, + reasoning_steps: list[str] | None = None, + response_id: str | None = None, ) -> ChatResult: return ChatResult.model_validate( { "textResponse": text, "toolCallEvents": [], "reasoningStepCount": 1, + "reasoningSteps": reasoning_steps or [], + "responseId": response_id, "stream_ended": stream_ended, "turn_wall_clock_sec": turn_wall_clock_sec, } @@ -1059,3 +1067,142 @@ def test_evaluate_agentic_kda_skill_does_not_log_pass_at_k_or_pass_power_k(): assert "kda_pass_power_2" not in logged assert "pass_at_2" not in logged assert "pass_power_2" not in logged + + +def test_run_agentic_kda_skill_accumulates_reasoning_steps_across_iterations(): + """A clarification turn's reasoning is retained even though only the final turn's + create/execute calls determine the KDA outcome.""" + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.side_effect = [ + _no_kda_chat_result("Which metric do you mean?", reasoning_steps=["step one"], response_id="resp-1"), + _kda_chat_result(reasoning_steps=["step two"], response_id="resp-2"), + ] + + with ( + patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), + patch( + "gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response", + return_value="I mean revenue.", + ), + ): + summary = run_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove the change?", + expected_output=_EXPECTED, + k=1, + max_iterations=2, + ) + + assert summary.best.reasoning_steps == ["step one", "step two"] + assert summary.best.response_id == "resp-2" + + +def test_evaluate_agentic_kda_skill_returns_reasoning_steps_on_pass(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = _kda_chat_result( + success=True, reasoning_steps=["analyzing drivers"], response_id="resp-1" + ) + + with ( + patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic._langfuse.try_make_langfuse_client", return_value=None), + ): + outcome = evaluate_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove revenue change?", + expected_output=_EXPECTED, + k=1, + max_iterations=1, + langfuse=None, + ) + + assert outcome.reasoning_steps == ["analyzing drivers"] + assert outcome.conversation_id == "conv-1" + assert outcome.response_id == "resp-1" + assert outcome.detail == { + "triggered": True, + "executed": True, + "success": True, + "turn_completed": True, + "disambiguated": False, + "actual_create_args": {"measure": {"type": "metric", "id": "revenue"}}, + "actual_execute_result": {"success": True, "data": {"summary": {}}}, + } + + +def test_evaluate_agentic_kda_skill_attaches_reasoning_steps_to_exception_on_fail(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = _no_kda_chat_result( + reasoning_steps=["could not find a measure"], response_id="resp-2" + ) + + with ( + patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic._langfuse.try_make_langfuse_client", return_value=None), + pytest.raises(KdaSkillAssertionError) as exc_info, + ): + evaluate_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove revenue change?", + expected_output=_EXPECTED, + k=1, + max_iterations=1, + langfuse=None, + ) + + assert exc_info.value.reasoning_steps == ["could not find a measure"] + assert exc_info.value.conversation_id == "conv-1" + assert exc_info.value.response_id == "resp-2" + assert exc_info.value.detail == { + "triggered": False, + "executed": False, + "success": False, + "turn_completed": True, + "disambiguated": False, + "actual_create_args": None, + "actual_execute_result": None, + } + + +def test_evaluate_agentic_kda_skill_preserves_reasoning_from_a_chat_error_partial_result(): + # Same scenario as test_run_agentic_kda_skill_recovers_kda_calls_from_a_chat_errors_partial_result + # (KDA create/execute already streamed before an unrelated later failure), but checking that + # the partial_result's own reasoning_steps/response_id survive onto the exception too, not + # just the tool-call data. + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.side_effect = ChatError( + "SSE error 500: boom", + status_code=500, + partial_result=_kda_chat_result( + success=True, reasoning_steps=["analyzing before cutoff"], response_id="resp-3" + ), + ) + + with ( + patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic._langfuse.try_make_langfuse_client", return_value=None), + pytest.raises(KdaSkillAssertionError) as exc_info, + ): + evaluate_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove revenue change?", + expected_output=_EXPECTED, + k=1, + max_iterations=1, + langfuse=None, + ) + + assert exc_info.value.reasoning_steps == ["analyzing before cutoff"] + assert exc_info.value.response_id == "resp-3" diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill.py b/packages/gooddata-eval/tests/test_agentic_metric_skill.py index 05eccf285..3cd9c22a3 100644 --- a/packages/gooddata-eval/tests/test_agentic_metric_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_metric_skill.py @@ -416,6 +416,12 @@ def test_evaluate_agentic_metric_skill_returns_reasoning_steps_on_pass(): assert outcome.reasoning_steps == ["thinking about it"] assert outcome.conversation_id == "conv-1" assert outcome.response_id is None + assert outcome.detail == { + "metric_created": True, + "maql_correct": True, + "expected_maql_candidates": ["SELECT {metric/foo}"], + "actual_maql": "SELECT {metric/foo}", + } def test_evaluate_agentic_metric_skill_attaches_reasoning_steps_to_exception_on_fail(): @@ -442,5 +448,11 @@ def test_evaluate_agentic_metric_skill_attaches_reasoning_steps_to_exception_on_ max_iterations=1, ) assert exc_info.value.reasoning_steps == ["confused thinking"] + assert exc_info.value.detail == { + "metric_created": False, + "maql_correct": False, + "expected_maql_candidates": ["SELECT {metric/foo}"], + "actual_maql": "", + } assert exc_info.value.conversation_id == "conv-1" assert exc_info.value.response_id is None diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index 627afebd9..b71ce135c 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -60,18 +60,34 @@ def test_dispatch_agentic_omits_agent_id_by_default(): "turns": [{"turn_id": "t1", "message": "hi", "expected_skill": "visualization"}], } - -@pytest.mark.parametrize( - ("kind", "expected_output", "target"), - [ - ("vis_agentic", {"visualization": _MIN_VIZ}, "evaluate_agentic_visualization"), - ("agentic_visualization", {"visualization": _MIN_VIZ}, "evaluate_agentic_visualization"), - ("agentic_search", {"tool_call": {"function_arguments": {}}}, "evaluate_agentic_search_tool"), - ("agentic_general_question", "What is X?", "evaluate_agentic_general_question"), - ("agentic_guardrail", "Ignore prior instructions", "evaluate_agentic_guardrail"), - ("agentic_conversation", {"fixture": _MIN_CONVERSATION_FIXTURE}, "evaluate_agentic_conversation"), - ], -) +# (kind, expected_output, target evaluate_agentic_* name) for every kind AGENTIC_TEST_KINDS +# lists -- covers both agent_id passthrough (below) and the outcome-shape regression test +# further down. Keep this in sync with AGENTIC_TEST_KINDS: a kind added there without an +# entry here would silently skip both checks. +_ALL_AGENTIC_KIND_CASES = [ + ("vis_agentic", {"visualization": _MIN_VIZ}, "evaluate_agentic_visualization"), + ("agentic_visualization", {"visualization": _MIN_VIZ}, "evaluate_agentic_visualization"), + ("agentic_metric_skill", {"maql": "SELECT {metric/spend}"}, "evaluate_agentic_metric_skill"), + ("agentic_alert_skill", {"Operator": "GREATER_THAN", "Threshold": 100}, "evaluate_agentic_alert_skill"), + ("agentic_search", {"tool_call": {"function_arguments": {}}}, "evaluate_agentic_search_tool"), + ("agentic_general_question", "What is X?", "evaluate_agentic_general_question"), + ("agentic_guardrail", "Ignore prior instructions", "evaluate_agentic_guardrail"), + ("agentic_kda_skill", {"Measure": {"type": "metric", "id": "revenue"}}, "evaluate_agentic_kda_skill"), + ("agentic_conversation", {"fixture": _MIN_CONVERSATION_FIXTURE}, "evaluate_agentic_conversation"), +] + + +def test_all_agentic_kind_cases_covers_every_registered_kind(): + """Guards the two parametrized tests below against silently going stale: a kind added + to AGENTIC_TEST_KINDS without a matching case here would otherwise just not get tested, + not fail loudly.""" + from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS + + covered = {kind for kind, _, _ in _ALL_AGENTIC_KIND_CASES} + assert covered == set(AGENTIC_TEST_KINDS) + + +@pytest.mark.parametrize(("kind", "expected_output", "target"), _ALL_AGENTIC_KIND_CASES) def test_dispatch_agentic_passes_agent_id_through_for_every_kind(kind, expected_output, target): item = DatasetItem( id="q1", @@ -109,7 +125,10 @@ def test_run_agentic_items_surfaces_reasoning_steps_on_pass(): with patch( "gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", return_value=AgenticEvalOutcome( - reasoning_steps=["it created the alert"], conversation_id="conv-1", response_id="resp-1" + reasoning_steps=["it created the alert"], + conversation_id="conv-1", + response_id="resp-1", + detail={"alert_created": True}, ), ): report = run_agentic_items( @@ -123,6 +142,7 @@ def test_run_agentic_items_surfaces_reasoning_steps_on_pass(): assert report.items[0].reasoning_steps == ["it created the alert"] assert report.items[0].conversation_id == "conv-1" assert report.items[0].response_id == "resp-1" + assert report.items[0].best_detail == {"alert_created": True} def test_run_agentic_items_surfaces_reasoning_steps_from_exception_on_fail(): @@ -130,6 +150,7 @@ def test_run_agentic_items_surfaces_reasoning_steps_from_exception_on_fail(): exc.reasoning_steps = ["it got confused"] exc.conversation_id = "conv-2" exc.response_id = "resp-2" + exc.detail = {"alert_created": False} with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=exc): report = run_agentic_items( [_item()], @@ -142,6 +163,7 @@ def test_run_agentic_items_surfaces_reasoning_steps_from_exception_on_fail(): assert report.items[0].reasoning_steps == ["it got confused"] assert report.items[0].conversation_id == "conv-2" assert report.items[0].response_id == "resp-2" + assert report.items[0].best_detail == {"alert_created": False} def test_run_agentic_items_defaults_reasoning_steps_to_empty_when_exception_has_none(): @@ -157,21 +179,46 @@ def test_run_agentic_items_defaults_reasoning_steps_to_empty_when_exception_has_ run_ts="2026-01-01", ) assert report.items[0].reasoning_steps == [] + assert report.items[0].best_detail == {} assert report.items[0].conversation_id is None assert report.items[0].response_id is None -def test_run_agentic_items_defaults_reasoning_steps_to_empty_for_untouched_kinds(): - # general_question/guardrail/search_tool/visualization still return None -- unchanged. - with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_guardrail", return_value=None): - report = run_agentic_items( - [_item(test_kind="agentic_guardrail")], - host="http://host", +@pytest.mark.parametrize(("kind", "expected_output", "target"), _ALL_AGENTIC_KIND_CASES) +def test_dispatch_agentic_returns_a_real_outcome_for_every_kind(kind, expected_output, target): + """Regression test for the bug this fixes: `guardrail`/`search_tool`/`general_question`/ + `visualization`/`kda_skill` used to return None/a bare value instead of an + AgenticEvalOutcome, so their reasoning_steps/conversation_id/response_id were silently + dropped (confirmed live: a real eval run produced 0/30 reasoning sidecars for + agentic_guardrail). Every kind must now return the exact AgenticEvalOutcome its + evaluator produced -- not None, not the outcome's reasoning_steps list alone, not any + other bare value the old `isinstance(outcome, tuple)`/`isinstance(outcome, AgenticEvalOutcome)` + fallback could silently swallow.""" + from gooddata_eval.core.models import AgenticEvalOutcome + + item = DatasetItem( + id="q1", + dataset_name="ds", + test_kind=kind, + question="q", + expected_output=expected_output, + ) + canned = AgenticEvalOutcome(reasoning_steps=["x"], conversation_id="c1", response_id="r1", detail={"k": "v"}) + with patch(f"gooddata_eval.cli.agentic_runner.{target}", return_value=canned) as mock_eval: + result = _dispatch_agentic( + item, + host="https://h", token="tok", workspace_id="ws1", + k=1, + langfuse=None, run_ts="2026-01-01", + model_version_override=None, ) - assert report.items[0].pass_at_k is True - assert report.items[0].reasoning_steps == [] - assert report.items[0].conversation_id is None - assert report.items[0].response_id is None + mock_eval.assert_called_once() + assert result is canned + assert isinstance(result, AgenticEvalOutcome) + assert result.reasoning_steps == ["x"] + assert result.detail == {"k": "v"} + assert result.conversation_id == "c1" + assert result.response_id == "r1" diff --git a/packages/gooddata-eval/tests/test_agentic_search_tool.py b/packages/gooddata-eval/tests/test_agentic_search_tool.py index fdeac3ef1..025c4db17 100644 --- a/packages/gooddata-eval/tests/test_agentic_search_tool.py +++ b/packages/gooddata-eval/tests/test_agentic_search_tool.py @@ -2,9 +2,12 @@ # SPDX-License-Identifier: LicenseRef-GoodData-Enterprise from unittest.mock import MagicMock, patch +import pytest from gooddata_eval.core.agentic.search_tool import ( + SearchToolAssertionError, _tool_correctness, _tool_selection, + evaluate_agentic_search_tool, run_agentic_search_tool, ) from gooddata_eval.core.models import ChatResult, ToolCallEvent @@ -107,3 +110,96 @@ def test_run_agentic_search_tool_creates_fresh_conversations_for_remaining_runs( ) assert mock_client.create_conversation.call_count == 2 assert mock_client.delete_conversation.call_count == 2 + + +def test_run_agentic_search_tool_captures_reasoning_steps(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = ChatResult.model_validate( + { + "textResponse": "Found it", + "toolCallEvents": [{"functionName": "search_objects", "functionArguments": '{"keywords": "revenue"}'}], + "reasoningSteps": ["deciding what to search for"], + "responseId": "resp-1", + } + ) + + with patch("gooddata_eval.core.agentic.search_tool.ChatClient", return_value=mock_client): + summary = run_agentic_search_tool( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Search for revenue", + expected_tool_call={"keywords": "revenue"}, + k=1, + ) + + assert summary.best.reasoning_steps == ["deciding what to search for"] + assert summary.best.response_id == "resp-1" + + +def test_evaluate_agentic_search_tool_returns_reasoning_steps_on_pass(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = ChatResult.model_validate( + { + "textResponse": "Found it", + "toolCallEvents": [{"functionName": "search_objects", "functionArguments": '{"keywords": "revenue"}'}], + "reasoningSteps": ["deciding what to search for"], + "responseId": "resp-1", + } + ) + + with patch("gooddata_eval.core.agentic.search_tool.ChatClient", return_value=mock_client): + outcome = evaluate_agentic_search_tool( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Search for revenue", + expected_tool_call={"keywords": "revenue"}, + k=1, + ) + + assert outcome.reasoning_steps == ["deciding what to search for"] + assert outcome.conversation_id == "conv-1" + assert outcome.response_id == "resp-1" + assert outcome.detail == { + "tool_selected": True, + "tool_correct": True, + "tool_call_names": ["search_objects"], + } + + +def test_evaluate_agentic_search_tool_attaches_reasoning_steps_to_exception_on_fail(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = ChatResult.model_validate( + { + "textResponse": "I could not find anything", + "toolCallEvents": [], + "reasoningSteps": ["giving up early"], + "responseId": "resp-2", + } + ) + + with ( + patch("gooddata_eval.core.agentic.search_tool.ChatClient", return_value=mock_client), + pytest.raises(SearchToolAssertionError) as exc_info, + ): + evaluate_agentic_search_tool( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Search for revenue", + expected_tool_call={"keywords": "revenue"}, + k=1, + ) + + assert exc_info.value.reasoning_steps == ["giving up early"] + assert exc_info.value.conversation_id == "conv-1" + assert exc_info.value.response_id == "resp-2" + assert exc_info.value.detail == { + "tool_selected": False, + "tool_correct": False, + "tool_call_names": [], + } diff --git a/packages/gooddata-eval/tests/test_agentic_visualization.py b/packages/gooddata-eval/tests/test_agentic_visualization.py index cbeae6b3f..80a558202 100644 --- a/packages/gooddata-eval/tests/test_agentic_visualization.py +++ b/packages/gooddata-eval/tests/test_agentic_visualization.py @@ -6,8 +6,11 @@ from unittest.mock import MagicMock, call, patch +import pytest from gooddata_eval.core.agentic.visualization import ( + VisualizationAssertionError, _execute_single_run, + evaluate_agentic_visualization, run_agentic_visualization, ) from gooddata_eval.core.models import ChatResult, CreatedVisualization @@ -241,3 +244,130 @@ def test_run_agentic_visualization_creates_conversation_when_no_initial_id(): assert instance.create_conversation.call_count == 2 assert instance.delete_conversation.call_count == 2 + + +def test_execute_single_run_accumulates_reasoning_steps_across_iterations(monkeypatch): + """Reasoning steps from every turn (clarification + final) are accumulated, not just the last.""" + client = MagicMock() + clarify = ChatResult.model_validate( + { + "textResponse": "Which metrics?", + "toolCallEvents": [], + "reasoningSteps": ["step one"], + "responseId": "resp-1", + } + ) + final = ChatResult.model_validate( + { + "createdVisualizations": {"objects": [_viz()], "reasoning": ""}, + "toolCallEvents": [], + "reasoningSteps": ["step two"], + "responseId": "resp-2", + } + ) + client.send_message.side_effect = [clarify, final] + monkeypatch.setattr( + "gooddata_eval.core.agentic.visualization.generate_simulated_response", + lambda msg, exp: "Revenue please", + ) + + result = _execute_single_run(client, "conv-1", "Show me a chart", [_expected()]) + + assert result.reasoning_steps == ["step one", "step two"] + assert result.response_id == "resp-2" + + +def test_evaluate_agentic_visualization_returns_reasoning_steps_on_pass(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = ChatResult.model_validate( + { + "createdVisualizations": {"objects": [_viz()], "reasoning": ""}, + "toolCallEvents": [], + "reasoningSteps": ["building the chart"], + "responseId": "resp-1", + } + ) + + with patch("gooddata_eval.core.agentic.visualization.ChatClient", return_value=mock_client): + outcome = evaluate_agentic_visualization( + host="https://example.com", + token="tok", + workspace_id="ws", + question="Show revenue", + expected_outputs=[_expected()], + k=1, + ) + + assert outcome.reasoning_steps == ["building the chart"] + assert outcome.conversation_id == "conv-1" + assert outcome.response_id == "resp-1" + assert outcome.detail == { + "visualization_created": True, + "cross_ref_valid": True, + "cross_ref_errors": [], + "metrics_correct": True, + "dimensions_correct": True, + "filters_correct": True, + "filter_date_score": True, + "filter_ranking_score": True, + "filter_attribute_score": True, + "viz_type_hard": True, + "skill_activated": False, + "expected_metric_uris": ["metric/revenue"], + "actual_metric_uris": ["metric/revenue"], + "expected_dim_uris": ["label/date.quarter"], + "actual_dim_uris": ["label/date.quarter"], + "expected_filters": {"date": [], "ranking": [], "attribute": []}, + "actual_filters": {"date": [], "ranking": [], "attribute": []}, + } + + +def test_evaluate_agentic_visualization_attaches_reasoning_steps_to_exception_on_fail(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = ChatResult.model_validate( + { + "textResponse": "I could not build that", + "toolCallEvents": [], + "reasoningSteps": ["giving up"], + "responseId": "resp-2", + } + ) + + with ( + patch("gooddata_eval.core.agentic.visualization.ChatClient", return_value=mock_client), + pytest.raises(VisualizationAssertionError) as exc_info, + ): + evaluate_agentic_visualization( + host="https://example.com", + token="tok", + workspace_id="ws", + question="Show revenue", + expected_outputs=[_expected()], + k=1, + max_iterations=1, + ) + + assert exc_info.value.reasoning_steps == ["giving up"] + assert exc_info.value.conversation_id == "conv-1" + assert exc_info.value.response_id == "resp-2" + assert exc_info.value.detail == { + "visualization_created": False, + "cross_ref_valid": False, + "cross_ref_errors": ["No visualization was created"], + "metrics_correct": False, + "dimensions_correct": False, + "filters_correct": False, + "filter_date_score": False, + "filter_ranking_score": False, + "filter_attribute_score": False, + "viz_type_hard": False, + "skill_activated": False, + "expected_metric_uris": ["metric/revenue"], + "actual_metric_uris": [], + "expected_dim_uris": ["label/date.quarter"], + "actual_dim_uris": [], + "expected_filters": {"date": [], "ranking": [], "attribute": []}, + "actual_filters": {"date": [], "ranking": [], "attribute": []}, + }