feat: Revision third-party eval metrics adapter (DeepEval + Autoevals)

.gitignore

@@ -229,3 +229,4 @@ local_settings.py
 Dockerfile
 CLAUDE.md
 .omc/
+.deepeval/

pyproject.toml

@@ -173,3 +173,9 @@ simulation = [
 datasets = [
     "requests>=2.31.0",
 ]
+deepeval = [
+    "deepeval>=2.0.0",
+]
+autoevals = [
+    "autoevals>=0.0.50",
+]

src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/__init__.py

@@ -0,0 +1,5 @@
+"""Third-party evaluation adapters for AgentCore code-based evaluators."""
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.base import BaseAdapter
+
+__all__ = ["BaseAdapter"]

src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/autoevals/__init__.py

@@ -0,0 +1,5 @@
+"""Autoevals adapter for AgentCore code-based evaluators."""
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals.adapter import AutoevalsAdapter
+
+__all__ = ["AutoevalsAdapter"]

src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/autoevals/adapter.py

@@ -0,0 +1,86 @@
+"""Autoevals adapter for AgentCore code-based evaluators."""
+
+import logging
+from typing import Any, Callable, Dict, Optional
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.models import EvaluatorInput, EvaluatorOutput
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.base import BaseAdapter
+
+logger = logging.getLogger(__name__)
+
+
+class AutoevalsAdapter(BaseAdapter):
+    """Adapter that runs an Autoevals scorer against AgentCore evaluation events.
+
+    Example (default span mapping)::
+
+        from autoevals import Factuality
+        from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoevalsAdapter
+
+        scorer = Factuality()
+        adapter = AutoevalsAdapter(scorer=scorer)
+
+    Example (customer mapper returning eval kwargs)::
+
+        adapter = AutoevalsAdapter(
+            scorer=Factuality(),
+            customer_mapper=lambda ev: {
+                "input": ev.session_spans[0]["attributes"]["question"],
+                "output": ev.session_spans[0]["attributes"]["answer"],
+                "expected": "the expected answer",
+            },
+        )
+    """
+
+    def __init__(
+        self,
+        scorer: Any,
+        customer_mapper: Optional[Callable[[EvaluatorInput], Dict[str, Any]]] = None,
+        threshold: float = 0.5,
+    ):
+        """Initialize the adapter.
+
+        Args:
+            scorer: An Autoevals scorer instance (e.g. Factuality(), ClosedQA()).
+            customer_mapper: Optional callable that receives the EvaluatorInput and
+                returns a dict of kwargs for scorer.eval(). Bypasses default span
+                mapping when provided. Expected keys: input, output, expected (optional).
+            threshold: Score threshold for Pass/Fail determination. Defaults to 0.5.
+        """
+        self.scorer = scorer
+        self.customer_mapper = customer_mapper
+        self.threshold = threshold
+
+    def _run(self, evaluator_input: EvaluatorInput) -> EvaluatorOutput:
+        """Run the Autoevals scorer pipeline."""
+        if self.customer_mapper is not None:
+            kwargs = self.customer_mapper(evaluator_input)
+        else:
+            result = self._default_extract(evaluator_input)
+            if not result.input or not result.actual_output:
+                missing = []
+                if not result.input:
+                    missing.append("input")
+                if not result.actual_output:
+                    missing.append("actual_output")
+                scorer_name = type(self.scorer).__name__
+                return EvaluatorOutput(
+                    label="Error",
+                    errorCode="MISSING_REQUIRED_FIELD",
+                    errorMessage=f"Field(s) {missing} required by {scorer_name} but not found in evaluation event. "
+                    f"Provide a customer_mapper or ensure spans contain the necessary data.",
+                )
+            kwargs: Dict[str, Any] = {
+                "input": result.input,
+                "output": result.actual_output,
+            }
+            if result.expected_output:
+                kwargs["expected"] = result.expected_output
+
+        eval_result = self.scorer.eval(**kwargs)
+
+        score = eval_result.score
+        label = "Pass" if score is not None and score >= self.threshold else "Fail"
+        explanation = getattr(eval_result, "metadata", {}).get("rationale", "") if hasattr(eval_result, "metadata") else ""
+
+        return EvaluatorOutput(value=score, label=label, explanation=explanation)

src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/base.py

@@ -0,0 +1,77 @@
+"""Base adapter for third-party evaluation framework integrations."""
+
+import abc
+import logging
+from typing import Any, Dict, List, Optional
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.models import EvaluatorInput, EvaluatorOutput
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.span_mappers import (
+    SpanMapResult,
+    map_spans,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class BaseAdapter(abc.ABC):
+    """Base adapter for third-party evaluation framework integrations.
+
+    Accepts an EvaluatorInput (from the code_based_evaluators flow),
+    extracts fields from spans using the built-in mapper layer, runs the
+    evaluation via execute(), and returns an EvaluatorOutput.
+
+    Never raises unhandled exceptions — always returns a valid EvaluatorOutput.
+    """
+
+    def __call__(self, evaluator_input: EvaluatorInput, context: Any = None) -> EvaluatorOutput:
+        """Handle an evaluation invocation.
+
+        Args:
+            evaluator_input: Parsed EvaluatorInput from the code-based evaluator flow.
+            context: Lambda context object (unused).
+
+        Returns:
+            EvaluatorOutput with score, label, and explanation or error fields.
+        """
+        try:
+            return self._run(evaluator_input)
+        except ValueError as e:
+            logger.error("Field extraction failed: %s", e)
+            return EvaluatorOutput(
+                label="Error",
+                errorCode="FIELD_EXTRACTION_ERROR",
+                errorMessage=str(e),
+            )
+        except Exception as e:
+            logger.error("Execution failed: %s", e, exc_info=True)
+            return EvaluatorOutput(
+                label="Error",
+                errorCode="METRIC_ERROR",
+                errorMessage=f"{type(self).__name__} failed: {e}",
+            )
+
+    @abc.abstractmethod
+    def _run(self, evaluator_input: EvaluatorInput) -> EvaluatorOutput:
+        """Run the full evaluation pipeline. Subclasses implement this."""
+
+    def _default_extract(self, evaluator_input: EvaluatorInput) -> SpanMapResult:
+        """Extract fields using the built-in span mapper layer."""
+        spans = self._filter_spans_by_target(evaluator_input)
+        return map_spans(spans, evaluator_input.reference_inputs)
+
+    def _filter_spans_by_target(self, evaluator_input: EvaluatorInput) -> List[Dict]:
+        """Filter session spans based on evaluationLevel and evaluationTarget.
+
+        Per the AgentCore code-based evaluator contract:
+        - TRACE: only spans matching target_trace_id
+        - TOOL_CALL: only the span matching target_span_id
+        - SESSION: all spans (no filtering)
+        """
+        spans = evaluator_input.session_spans
+
+        if evaluator_input.evaluation_level == "TRACE" and evaluator_input.target_trace_id:
+            spans = [s for s in spans if s.get("traceId") == evaluator_input.target_trace_id]
+        elif evaluator_input.evaluation_level == "TOOL_CALL" and evaluator_input.target_span_id:
+            spans = [s for s in spans if s.get("spanId") == evaluator_input.target_span_id]
+
+        return spans

src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/deepeval/__init__.py

@@ -0,0 +1,5 @@
+"""DeepEval adapter for AgentCore code-based evaluators."""
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval.adapter import DeepEvalAdapter
+
+__all__ = ["DeepEvalAdapter"]

src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/deepeval/adapter.py

@@ -0,0 +1,101 @@
+"""DeepEval adapter for AgentCore code-based evaluators."""
+
+import logging
+from typing import Any, Callable, Optional
+
+from deepeval.metrics import BaseMetric
+from deepeval.test_case import LLMTestCase
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.models import EvaluatorInput, EvaluatorOutput
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.base import BaseAdapter
+
+logger = logging.getLogger(__name__)
+
+
+class DeepEvalAdapter(BaseAdapter):
+    """Adapter that runs a DeepEval metric against AgentCore evaluation events.
+
+    Example (default span mapping)::
+
+        from deepeval.metrics import AnswerRelevancyMetric
+        from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter
+
+        metric = AnswerRelevancyMetric(threshold=0.7)
+        adapter = DeepEvalAdapter(metric=metric)
+
+    Example (customer mapper returning LLMTestCase)::
+
+        adapter = DeepEvalAdapter(
+            metric=AnswerRelevancyMetric(threshold=0.7),
+            customer_mapper=lambda ev: LLMTestCase(
+                input=ev.session_spans[0]["attributes"]["user_query"],
+                actual_output=ev.session_spans[0]["attributes"]["response"],
+            ),
+        )
+    """
+
+    def __init__(
+        self,
+        metric: BaseMetric,
+        customer_mapper: Optional[Callable[[EvaluatorInput], LLMTestCase]] = None,
+        model: Optional[Any] = None,
+    ):
+        """Initialize the adapter.
+
+        Args:
+            metric: A DeepEval BaseMetric instance (e.g. AnswerRelevancyMetric).
+            customer_mapper: Optional callable that receives the EvaluatorInput and
+                returns a LLMTestCase. Bypasses default span mapping when provided.
+            model: Optional model override for the metric's LLM.
+        """
+        self.metric = metric
+        self.customer_mapper = customer_mapper
+        if model is not None:
+            self.metric.model = model
+
+    def _run(self, evaluator_input: EvaluatorInput) -> EvaluatorOutput:
+        """Run the DeepEval metric pipeline."""
+        if self.customer_mapper is not None:
+            test_case = self.customer_mapper(evaluator_input)
+        else:
+            result = self._default_extract(evaluator_input)
+            if not result.input or not result.actual_output:
+                missing = []
+                if not result.input:
+                    missing.append("input")
+                if not result.actual_output:
+                    missing.append("actual_output")
+                metric_name = type(self.metric).__name__
+                return EvaluatorOutput(
+                    label="Error",
+                    errorCode="MISSING_REQUIRED_FIELD",
+                    errorMessage=f"Field(s) {missing} required by {metric_name} but not found in evaluation event. "
+                    f"Provide a customer_mapper or ensure spans contain the necessary data.",
+                )
+            test_case = LLMTestCase(
+                input=result.input,
+                actual_output=result.actual_output,
+                expected_output=result.expected_output,
+                context=result.context,
+                retrieval_context=result.retrieval_context,
+            )
+
+        try:
+            self.metric.measure(test_case)
+        except Exception as e:
+            if type(e).__name__ == "MissingTestCaseParamsError":
+                return EvaluatorOutput(
+                    label="Error",
+                    errorCode="MISSING_REQUIRED_FIELD",
+                    errorMessage=f"{type(self.metric).__name__} requires fields not extracted from spans: {e}. "
+                    f"Provide a customer_mapper to supply custom fields from your trace data.",
+                )
+            raise
+
+        score = self.metric.score
+        reason = getattr(self.metric, "reason", None) or ""
+        threshold = getattr(self.metric, "threshold", 0.5)
+        success = getattr(self.metric, "success", score is not None and score >= threshold)
+        label = "Pass" if success else "Fail"
+
+        return EvaluatorOutput(value=score, label=label, explanation=reason)

src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/span_mappers/__init__.py

@@ -0,0 +1,13 @@
+"""Span mappers for extracting evaluation fields from Agent SDK trace formats."""
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.span_mappers.base import (
+    map_spans,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.span_mappers.common import (
+    SpanMapResult,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.span_mappers.strands import (
+    StrandsSpanMapper,
+)
+
+__all__ = ["SpanMapResult", "map_spans", "StrandsSpanMapper"]

src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/span_mappers/base.py

@@ -0,0 +1,51 @@
+"""Span mapping orchestration."""
+
+import logging
+from typing import Any, Dict, List, Optional
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.span_mappers.common import (
+    SpanMapResult,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.span_mappers.strands import (
+    StrandsSpanMapper,
+)
+
+logger = logging.getLogger(__name__)
+
+_strands_mapper = StrandsSpanMapper()
+
+
+def map_spans(
+    session_spans: List[Dict[str, Any]],
+    reference_inputs: Optional[List[Any]] = None,
+) -> SpanMapResult:
+    """Map session spans to evaluation fields.
+
+    Currently supports Strands Agent SDK spans (scope.name == "strands.telemetry.tracer").
+    Additional framework support can be added when real span data is available.
+
+    Args:
+        session_spans: Raw ADOT span dicts from the evaluation service.
+        reference_inputs: Optional ReferenceInput list for expected_output.
+
+    Returns:
+        SpanMapResult with extracted fields.
+
+    Raises:
+        ValueError: If no mapper can extract data from the spans.
+    """
+    result = _strands_mapper.map(session_spans)
+    if result is not None:
+        if reference_inputs:
+            ref = reference_inputs[0]
+            expected = getattr(ref, "expected_response_text", None)
+            if expected:
+                result.expected_output = expected
+        return result
+
+    raise ValueError(
+        "Could not extract evaluation fields from spans. "
+        "No Strands agent span (scope.name=='strands.telemetry.tracer' with "
+        "gen_ai.operation.name=='invoke_agent') found. "
+        "Provide a customer_mapper for custom or unsupported span formats."
+    )