feat: Third-party eval metrics adapter (DeepEval + Autoevals) with strands-evals mappers

.gitignore

@@ -229,3 +229,5 @@ local_settings.py
 Dockerfile
 CLAUDE.md
 .omc/
+.deepeval/
+e2e_tests/

pyproject.toml

@@ -150,7 +150,7 @@ dev = [
     "websockets>=14.1",
     "wheel>=0.45.1",
     "strands-agents>=1.20.0",
-    "strands-agents-evals>=0.1.0",
+    "strands-agents-evals>=1.0.0,<2.0.0",
     "a2a-sdk[http-server]>=0.3,<1.0",
     "ag-ui-protocol>=0.1.10",
     "mcp-proxy-for-aws>=0.1.0",
@@ -164,12 +164,18 @@ strands-agents = [
     "mcp>=1.23.0,<2.0.0",
 ]
 strands-agents-evals = [
-    "strands-agents-evals>=0.1.0"
+    "strands-agents-evals>=1.0.0,<2.0.0"
 ]
 simulation = [
     "jinja2>=3.1.0",
-    "strands-agents-evals>=0.1.0",
+    "strands-agents-evals>=1.0.0,<2.0.0",
 ]
 datasets = [
     "requests>=2.31.0",
 ]
+deepeval = [
+    "deepeval>=2.0.0",
+]
+autoevals = [
+    "autoevals>=0.0.50",
+]

src/bedrock_agentcore/evaluation/custom_code_based_evaluators/models.py

@@ -80,8 +80,9 @@ class EvaluatorOutput(BaseModel):
 
     @model_validator(mode="after")
     def _require_label_or_error_code(self) -> "EvaluatorOutput":
-        if not self.errorCode and self.label is None:
+        if not self.errorCode and self.label is None and self.value is None:
             raise ValueError(
-                "label is required for success responses; set errorCode to return an error response without a label"
+                "Either label, value, or errorCode must be set; "
+                "set errorCode to return an error response without a label"
             )
         return self

src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/__init__.py

@@ -0,0 +1,5 @@
+"""Third-party evaluation adapters for AgentCore code-based evaluators."""
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.base import BaseAdapter
+
+__all__ = ["BaseAdapter"]

src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/autoevals/__init__.py

@@ -0,0 +1,5 @@
+"""Autoevals adapter for AgentCore code-based evaluators."""
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals.adapter import AutoevalsAdapter
+
+__all__ = ["AutoevalsAdapter"]

src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/autoevals/adapter.py

@@ -0,0 +1,98 @@
+"""Autoevals adapter for AgentCore code-based evaluators."""
+
+import logging
+from typing import Any, Callable, Dict, Optional
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.models import EvaluatorInput, EvaluatorOutput
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.base import BaseAdapter
+
+logger = logging.getLogger(__name__)
+
+
+class AutoevalsAdapter(BaseAdapter):
+    """Adapter that runs an Autoevals scorer against AgentCore evaluation events.
+
+    Example (default span mapping)::
+
+        from autoevals import Factuality
+        from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoevalsAdapter
+
+        scorer = Factuality()
+        adapter = AutoevalsAdapter(metric=scorer)
+
+    Example (custom mapper returning eval kwargs)::
+
+        from typing import Dict, Any
+
+        def my_mapper(ev: EvaluatorInput) -> Dict[str, Any]:
+            return {
+                "input": ev.session_spans[0]["attributes"]["question"],
+                "output": ev.session_spans[0]["attributes"]["answer"],
+                "expected": "the expected answer",
+            }
+
+        adapter = AutoevalsAdapter(
+            metric=Factuality(),
+            custom_mapper=my_mapper,
+        )
+    """
+
+    def __init__(
+        self,
+        metric: Any,
+        custom_mapper: Optional[Callable[[EvaluatorInput], Dict[str, Any]]] = None,
+        threshold: Optional[float] = None,
+    ):
+        """Initialize the adapter.
+
+        Args:
+            metric: An Autoevals scorer instance (e.g. Factuality(), ClosedQA()).
+            custom_mapper: Optional callable that receives the EvaluatorInput and
+                returns a dict of kwargs for metric.eval(). Bypasses default span
+                mapping when provided. Expected keys: input, output, expected (optional).
+            threshold: Optional score threshold for Pass/Fail label. If None, label
+                is omitted from the output.
+        """
+        self.metric = metric
+        self.custom_mapper = custom_mapper
+        self.threshold = threshold
+
+    def _run(self, evaluator_input: EvaluatorInput) -> EvaluatorOutput:
+        """Run the Autoevals scorer pipeline."""
+        if self.custom_mapper is not None:
+            kwargs = self.custom_mapper(evaluator_input)
+        else:
+            result = self._default_extract(evaluator_input)
+            if not result.input or not result.actual_output:
+                missing = []
+                if not result.input:
+                    missing.append("input")
+                if not result.actual_output:
+                    missing.append("actual_output")
+                metric_name = type(self.metric).__name__
+                return EvaluatorOutput(
+                    label="Error",
+                    errorCode="MISSING_REQUIRED_FIELD",
+                    errorMessage=f"Field(s) {missing} required by {metric_name} but not found in evaluation event. "
+                    f"Provide a custom_mapper or ensure spans contain the necessary data.",
+                )
+            kwargs: Dict[str, Any] = {
+                "input": result.input,
+                "output": result.actual_output,
+            }
+            if result.expected_output:
+                kwargs["expected"] = result.expected_output
+            elif result.assertions:
+                kwargs["expected"] = "\n".join(result.assertions)
+            if result.retrieval_context:
+                kwargs["context"] = "\n".join(result.retrieval_context)
+
+        eval_result = self.metric.eval(**kwargs)
+
+        score = eval_result.score
+        label = None
+        if self.threshold is not None:
+            label = "Pass" if score is not None and score >= self.threshold else "Fail"
+        explanation = getattr(eval_result, "metadata", {}).get("rationale", "") if hasattr(eval_result, "metadata") else ""
+
+        return EvaluatorOutput(value=score, label=label, explanation=explanation)

src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/base.py

@@ -0,0 +1,84 @@
+"""Base adapter for third-party evaluation framework integrations."""
+
+import abc
+import logging
+from typing import Any, Dict, List, Optional
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.models import EvaluatorInput, EvaluatorOutput
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.span_mappers import (
+    SpanMapResult,
+    map_spans,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class BaseAdapter(abc.ABC):
+    """Base adapter for third-party evaluation framework integrations.
+
+    Accepts an EvaluatorInput (from the code_based_evaluators flow),
+    extracts fields from spans using the built-in mapper layer, runs the
+    evaluation via execute(), and returns an EvaluatorOutput.
+
+    Never raises unhandled exceptions — always returns a valid EvaluatorOutput.
+    """
+
+    def __call__(self, evaluator_input: EvaluatorInput, context: Any = None) -> EvaluatorOutput:
+        """Handle an evaluation invocation.
+
+        Args:
+            evaluator_input: Parsed EvaluatorInput from the code-based evaluator flow.
+            context: Lambda context object (unused).
+
+        Returns:
+            EvaluatorOutput with score, label, and explanation or error fields.
+        """
+        try:
+            return self._run(evaluator_input)
+        except ValueError as e:
+            logger.error("Field extraction failed: %s", e)
+            return EvaluatorOutput(
+                label="Error",
+                errorCode="FIELD_EXTRACTION_ERROR",
+                errorMessage=str(e),
+            )
+        except Exception as e:
+            logger.error("Execution failed: %s", e, exc_info=True)
+            return EvaluatorOutput(
+                label="Error",
+                errorCode="METRIC_ERROR",
+                errorMessage=f"{type(self).__name__} failed: {e}",
+            )
+
+    @abc.abstractmethod
+    def _run(self, evaluator_input: EvaluatorInput) -> EvaluatorOutput:
+        """Run the full evaluation pipeline. Subclasses implement this."""
+
+    def _default_extract(self, evaluator_input: EvaluatorInput) -> SpanMapResult:
+        """Extract fields using the built-in span mapper layer."""
+        spans = self._filter_spans_by_target(evaluator_input)
+        return map_spans(spans, evaluator_input.reference_inputs)
+
+    def _filter_spans_by_target(self, evaluator_input: EvaluatorInput) -> List[Dict]:
+        """Filter session spans based on evaluationLevel and evaluationTarget.
+
+        The service passes ALL session spans in every Lambda invocation without
+        pre-filtering. It fans out one Lambda call per evaluation target (one per
+        trace at TRACE level, one per span at TOOL_CALL level) and provides the
+        target ID so the Lambda can scope its evaluation. We filter here because
+        the service-side _invoke_single_eval_target passes session_spans directly
+        into the payload without filtering.
+
+        Levels:
+        - SESSION: all spans (no filtering) — one call for the entire session
+        - TRACE: only spans matching target_trace_id (service sends exactly one)
+        - TOOL_CALL: only the span matching target_span_id (service sends exactly one)
+        """
+        spans = evaluator_input.session_spans
+
+        if evaluator_input.evaluation_level == "TRACE" and evaluator_input.target_trace_id:
+            spans = [s for s in spans if s.get("traceId") == evaluator_input.target_trace_id]
+        elif evaluator_input.evaluation_level == "TOOL_CALL" and evaluator_input.target_span_id:
+            spans = [s for s in spans if s.get("spanId") == evaluator_input.target_span_id]
+
+        return spans

src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/deepeval/__init__.py

@@ -0,0 +1,5 @@
+"""DeepEval adapter for AgentCore code-based evaluators."""
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval.adapter import DeepEvalAdapter
+
+__all__ = ["DeepEvalAdapter"]

src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/deepeval/adapter.py

@@ -0,0 +1,121 @@
+"""DeepEval adapter for AgentCore code-based evaluators."""
+
+import logging
+from typing import Any, Callable, Dict, List, Optional
+
+from deepeval.metrics import BaseMetric
+from deepeval.test_case import LLMTestCase, ToolCall
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.models import EvaluatorInput, EvaluatorOutput
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.base import BaseAdapter
+
+logger = logging.getLogger(__name__)
+
+
+class DeepEvalAdapter(BaseAdapter):
+    """Adapter that runs a DeepEval metric against AgentCore evaluation events.
+
+    Example (default span mapping)::
+
+        from deepeval.metrics import AnswerRelevancyMetric
+        from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter
+
+        metric = AnswerRelevancyMetric(threshold=0.7)
+        adapter = DeepEvalAdapter(metric=metric)
+
+    Example (custom mapper returning LLMTestCase)::
+
+        from deepeval.test_case import LLMTestCase
+
+        def my_mapper(ev: EvaluatorInput) -> LLMTestCase:
+            return LLMTestCase(
+                input=ev.session_spans[0]["attributes"]["user_query"],
+                actual_output=ev.session_spans[0]["attributes"]["response"],
+            )
+
+        adapter = DeepEvalAdapter(
+            metric=AnswerRelevancyMetric(threshold=0.7),
+            custom_mapper=my_mapper,
+        )
+    """
+
+    def __init__(
+        self,
+        metric: BaseMetric,
+        custom_mapper: Optional[Callable[[EvaluatorInput], LLMTestCase]] = None,
+    ):
+        """Initialize the adapter.
+
+        Args:
+            metric: A DeepEval BaseMetric instance (e.g. AnswerRelevancyMetric).
+            custom_mapper: Optional callable that receives the EvaluatorInput and
+                returns a LLMTestCase. Bypasses default span mapping when provided.
+        """
+        self.metric = metric
+        self.custom_mapper = custom_mapper
+
+    def _run(self, evaluator_input: EvaluatorInput) -> EvaluatorOutput:
+        """Run the DeepEval metric pipeline."""
+        if self.custom_mapper is not None:
+            test_case = self.custom_mapper(evaluator_input)
+        else:
+            result = self._default_extract(evaluator_input)
+            if not result.input or not result.actual_output:
+                missing = []
+                if not result.input:
+                    missing.append("input")
+                if not result.actual_output:
+                    missing.append("actual_output")
+                metric_name = type(self.metric).__name__
+                return EvaluatorOutput(
+                    label="Error",
+                    errorCode="MISSING_REQUIRED_FIELD",
+                    errorMessage=f"Field(s) {missing} required by {metric_name} but not found in evaluation event. "
+                    f"Provide a custom_mapper or ensure spans contain the necessary data.",
+                )
+            # assertions from reference_inputs take precedence over span-extracted context
+            context = result.assertions if result.assertions else result.context
+            test_case = LLMTestCase(
+                input=result.input,
+                actual_output=result.actual_output,
+                expected_output=result.expected_output,
+                context=context,
+                retrieval_context=result.retrieval_context,
+                tools_called=self._build_tool_calls(result.tools_called) if result.tools_called else None,
+                expected_tools=self._build_tool_calls(result.expected_tools) if result.expected_tools else None,
+            )
+
+        try:
+            self.metric.measure(test_case)
+        except Exception as e:
+            if type(e).__name__ == "MissingTestCaseParamsError":
+                return EvaluatorOutput(
+                    label="Error",
+                    errorCode="MISSING_REQUIRED_FIELD",
+                    errorMessage=f"{type(self.metric).__name__} requires fields not extracted from spans: {e}. "
+                    f"Provide a custom_mapper to supply custom fields from your trace data.",
+                )
+            raise
+
+        score = self.metric.score
+        reason = getattr(self.metric, "reason", None) or ""
+        threshold = getattr(self.metric, "threshold", 0.5)
+        success = getattr(self.metric, "success", score is not None and score >= threshold)
+        label = "Pass" if success else "Fail"
+
+        return EvaluatorOutput(value=score, label=label, explanation=reason)
+
+    @staticmethod
+    def _build_tool_calls(tools_called: List[Dict[str, Any]]) -> List[ToolCall]:
+        """Convert extracted tool call dicts to DeepEval ToolCall objects."""
+        result: List[ToolCall] = []
+        for tc in tools_called:
+            name = tc.get("name", "")
+            if not name:
+                continue
+            result.append(ToolCall(
+                name=name,
+                input_parameters=tc.get("input_parameters"),
+                output=tc.get("output"),
+            ))
+        return result

src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/span_mappers/__init__.py

@@ -0,0 +1,17 @@
+"""Span mappers for extracting evaluation fields from Agent SDK trace formats.
+
+Uses strands-evals mappers for span format auto-detection and extraction,
+bridged to SpanMapResult for adapter consumption.
+"""
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.span_mappers.common import (
+    SpanMapResult,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.span_mappers.registry import (
+    map_spans,
+)
+
+__all__ = [
+    "SpanMapResult",
+    "map_spans",
+]