Add AWS/Bedrock user simulator (Nova Sonic) + optional Bedrock agent STT/TTS & judges

.env.example

@@ -39,8 +39,12 @@ AWS_ACCESS_KEY_ID=your_aws_access_key_id_here
 AWS_SECRET_ACCESS_KEY=your_aws_secret_access_key_here
 
 # --- Alternative providers (optional) ---
-# If you only have an OpenAI key you can skip AWS and set JUDGE_MODEL=gpt-5.2
-# to override all text judges. Audio judge metrics still require Gemini.
+# JUDGE_MODEL overrides all text judges (validation + scoring). Set it to a model_name in
+# EVA_MODEL_LIST — e.g. a Bedrock deployment for an all-AWS run:
+#   JUDGE_MODEL=us.anthropic.claude-opus-4-6-v1
+# or an OpenAI model if that's all you have: JUDGE_MODEL=gpt-5.2
+# Audio judge metrics (user_speech_fidelity) need an audio-capable model (e.g. Gemini) — Bedrock has
+# none, so that judge is automatically skipped when its model isn't a deployment in EVA_MODEL_LIST.
 
 #i Azure OpenAI key (alternative to direct OpenAI).
 #d secret
@@ -69,7 +73,7 @@ EVA_MODEL__LLM=gpt-5.2
 # --- LLM mode: STT ---
 #i STT provider for the voice pipeline.
 #d enum
-#e assemblyai,cartesia,cartesia-multilingual,deepgram,deepgram-flux,elevenlabs,nvidia,nvidia-baseten,openai,smallest
+#e assemblyai,aws_transcribe,cartesia,cartesia-multilingual,deepgram,deepgram-flux,elevenlabs,nvidia,nvidia-baseten,openai,smallest
 #x pipeline_mode=LLM
 EVA_MODEL__STT=cartesia
 
@@ -87,7 +91,7 @@ EVA_MODEL__STT_PARAMS='{"api_key": "your_cartesia_api_key", "model": "ink-2"}'
 # --- TTS (LLM and AudioLLM modes) ---
 #i TTS provider for the voice pipeline.
 #d enum
-#e cartesia,chatterbox,elevenlabs,gemini,kokoro,nvidia-baseten,openai,smallest,xtts
+#e aws_polly,cartesia,chatterbox,elevenlabs,gemini,kokoro,nvidia-baseten,openai,smallest,xtts
 #x pipeline_mode=LLM,AudioLLM
 EVA_MODEL__TTS=cartesia
 
@@ -97,6 +101,15 @@ EVA_MODEL__TTS=cartesia
 #x pipeline_mode=LLM,AudioLLM
 EVA_MODEL__TTS_PARAMS='{"api_key": "your_cartesia_api_key", "model": "sonic"}'
 
+# --- All-AWS cascade agent (Amazon Transcribe STT + Bedrock LLM + Amazon Polly TTS) ---
+# Runs the agent-under-test entirely on AWS. Credentials + region come from the standard AWS env
+# chain (AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY[/AWS_SESSION_TOKEN] + AWS_REGION), so *_PARAMS are
+# optional (region/voice/engine tuning only). The LLM uses a bedrock/... entry in EVA_MODEL_LIST.
+#   EVA_MODEL__STT=aws_transcribe
+#   EVA_MODEL__TTS=aws_polly
+#   EVA_MODEL__TTS_PARAMS='{"voice_id": "Joanna", "engine": "neural"}'
+#   EVA_MODEL__LLM=us.anthropic.claude-opus-4-6   # a model_name whose litellm_params.model is bedrock/...
+
 # --- LLM mode: cascade latency optimizations ---
 #i Prompt a model-generated lead-in before a tool call: 'off' or 'auto'.
 #d enum
@@ -261,7 +274,7 @@ EVA_MODEL_LIST='[
 # --- User simulator provider ---
 #i Caller provider. ElevenLabs is the backward-compatible default.
 #d enum
-#e elevenlabs,openai_realtime
+#e elevenlabs,openai_realtime,bedrock_s2s
 EVA_USER_SIMULATOR__PROVIDER=elevenlabs
 
 #i OpenAI Realtime caller model. Used only when provider=openai_realtime.
@@ -279,6 +292,31 @@ EVA_USER_SIMULATOR__PROVIDER=elevenlabs
 #x EVA_USER_SIMULATOR__PROVIDER=openai_realtime
 #v EVA_USER_SIMULATOR__MALE_VOICE=cedar
 
+# --- Bedrock Nova Sonic (audio-native S2S) caller ---
+# Requires the optional dependency on Python >=3.12: pip install 'eva[bedrock-s2s]'
+# and AWS creds in the env (AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY[/AWS_SESSION_TOKEN]).
+#i Nova Sonic model id. Used only when provider=bedrock_s2s.
+#d string
+#x EVA_USER_SIMULATOR__PROVIDER=bedrock_s2s
+#v EVA_USER_SIMULATOR__MODEL_ID=amazon.nova-2-sonic-v1:0
+
+#i AWS region for the Bedrock bidirectional stream.
+#d string
+#x EVA_USER_SIMULATOR__PROVIDER=bedrock_s2s
+#v EVA_USER_SIMULATOR__REGION=us-east-1
+
+#i Nova Sonic voiceId for the female / male caller personas.
+#d string
+#x EVA_USER_SIMULATOR__PROVIDER=bedrock_s2s
+#v EVA_USER_SIMULATOR__FEMALE_VOICE=tiffany
+#v EVA_USER_SIMULATOR__MALE_VOICE=matthew
+
+#i Turn-detection endpointing sensitivity (HIGH|MEDIUM|LOW).
+#d enum
+#e HIGH,MEDIUM,LOW
+#x EVA_USER_SIMULATOR__PROVIDER=bedrock_s2s
+#v EVA_USER_SIMULATOR__ENDPOINTING_SENSITIVITY=HIGH
+
 # --- Language (mutually exclusive with Accent and Behavior) ---
 #i ISO 639-1 language code for the user simulator. Datasets must exist for the selected language. Pattern for the agent ID pairs below: EVA_{LANG}_USER_{F|M}.
 #d enum

pyproject.toml

@@ -21,6 +21,9 @@ license = "MIT"
 dependencies = [
     "pydantic>=2.0",
     "pydantic-settings>=2.0",
+    # AWS Transcribe STT / Polly TTS assistant services (pipecat.services.aws.*) run on `websockets`
+    # + `aiobotocore` (brought in by `aioboto3` below) — no extra needed. Not using pipecat's [aws]
+    # extra because it pins aiobotocore>=3, which conflicts with aioboto3's aiobotocore 2.x.
     "pipecat-ai>=1.4.0",
     "elevenlabs>=2.53.0",
     "openai>=2.36.0",
@@ -56,6 +59,12 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
+# Audio-native Nova Sonic (bedrock_s2s) caller. Isolated because the bidirectional
+# streaming client is experimental (pre-1.0) and requires Python >=3.12, while EVA
+# itself supports 3.11+. Install with: pip install 'eva[bedrock-s2s]' (Python >=3.12).
+bedrock-s2s = [
+    "aws-sdk-bedrock-runtime>=0.7.0; python_version >= '3.12'",
+]
 dev = [
     "pytest>=7.0",
     "pytest-asyncio>=0.21",
@@ -104,3 +113,17 @@ ignore = ["D203", "D206", "D213", "D400", "D401", "D413", "D415", "E1", "E501"]
 [tool.mypy]
 python_version = "3.11"
 strict = true
+
+# The optional 'bedrock-s2s' extra pulls the experimental, untyped aws-sdk-bedrock-runtime
+# and its smithy deps, which are Python >=3.12-only (they use `type X = ...` syntax mypy
+# rejects under python_version 3.11). Skip following into them; the imports are lazy and
+# guarded at the factory. Only relevant when the extra is installed.
+[[tool.mypy.overrides]]
+module = [
+    "aws_sdk_bedrock_runtime.*",
+    "smithy_aws_core.*",
+    "smithy_core.*",
+    "smithy_http.*",
+]
+follow_imports = "skip"
+ignore_missing_imports = true

src/eva/__init__.py

@@ -7,7 +7,7 @@
 
 # Bump simulation_version when changes affect benchmark outputs (agent code,
 # user simulator, orchestrator, simulation prompts, agent configs, tool mocks).
-simulation_version = "2.0.1"
+simulation_version = "2.0.5"
 
 # Bump metrics_version when changes affect metric computation (metrics code,
 # judge prompts, pricing tables, postprocessor).

src/eva/assistant/pipeline/services.py

@@ -5,6 +5,7 @@
 
 import dataclasses
 import datetime
+import os
 from collections.abc import AsyncGenerator
 from typing import Any
 
@@ -110,6 +111,20 @@ def _to_language_enum(tag: str) -> Language:
 _audio_llm_url_counter: int = 0
 
 
+def _aws_credentials(params: dict[str, Any]) -> dict[str, Any]:
+    """Resolve AWS creds/region for the pipecat AWS services from params, then the env chain.
+
+    pipecat's AWS services take the secret access key as ``api_key``. Any value left as None
+    lets the underlying SDK fall back to its own default resolution.
+    """
+    return {
+        "api_key": params.get("aws_secret_access_key") or os.environ.get("AWS_SECRET_ACCESS_KEY"),
+        "aws_access_key_id": params.get("aws_access_key_id") or os.environ.get("AWS_ACCESS_KEY_ID"),
+        "aws_session_token": params.get("aws_session_token") or os.environ.get("AWS_SESSION_TOKEN"),
+        "region": params.get("region") or os.environ.get("AWS_REGION", "us-east-1"),
+    }
+
+
 def create_stt_service(
     model: str | None,
     params: dict[str, Any] | None = None,
@@ -313,9 +328,19 @@ def create_stt_service(
             ),
         )
 
+    elif model_lower in {"aws", "aws_transcribe", "amazon_transcribe", "transcribe"}:
+        from pipecat.services.aws.stt import AWSTranscribeSTTService, AWSTranscribeSTTSettings
+
+        logger.info("Using AWS Transcribe STT")
+        return AWSTranscribeSTTService(
+            sample_rate=params.get("sample_rate", SAMPLE_RATE),
+            settings=AWSTranscribeSTTSettings(language=_to_language_enum(language_code)),
+            **_aws_credentials(params),
+        )
+
     else:
         raise ValueError(
-            f"Unknown STT model: {model}. Available: assemblyai, cartesia, cartesia-multilingual, cohere, deepgram, deepgram-flux, elevenlabs, nvidia, nvidia-baseten, openai, smallest, xai"
+            f"Unknown STT model: {model}. Available: assemblyai, aws_transcribe, cartesia, cartesia-multilingual, cohere, deepgram, deepgram-flux, elevenlabs, nvidia, nvidia-baseten, openai, smallest, xai"
         )
 
 
@@ -591,9 +616,31 @@ def create_tts_service(
         xtts_tts._settings.language = language_code
         return xtts_tts
 
+    elif model_lower in {"aws", "aws_polly", "amazon_polly", "polly"}:
+        from pipecat.services.aws.tts import AWSPollyTTSService, AWSPollyTTSSettings
+
+        logger.info("Using AWS Polly TTS")
+        # Forward optional Polly tuning (engine, pitch, rate, volume, lexicon_names); voice and
+        # language are passed explicitly below.
+        polly_settings_kwargs = {
+            k: params[k]
+            for f in dataclasses.fields(AWSPollyTTSSettings)
+            if (k := f.name) in params and k not in {"voice", "language", "model"}
+        }
+        polly_settings_kwargs.setdefault("engine", "neural")
+        return AWSPollyTTSService(
+            sample_rate=SAMPLE_RATE,
+            settings=AWSPollyTTSSettings(
+                voice=params.get("voice_id", "Joanna"),
+                language=_to_language_enum(language_code),
+                **polly_settings_kwargs,
+            ),
+            **_aws_credentials(params),
+        )
+
     else:
         raise ValueError(
-            f"Unknown TTS model: {model}. Available: cartesia, chatterbox, deepgram, elevenlabs, gemini, kokoro, nvidia-baseten, openai, smallest, voxtral, xai, xtts"
+            f"Unknown TTS model: {model}. Available: aws_polly, cartesia, chatterbox, deepgram, elevenlabs, gemini, kokoro, nvidia-baseten, openai, smallest, voxtral, xai, xtts"
         )
 
 

src/eva/models/config.py

@@ -51,9 +51,14 @@ def _get_all_metrics() -> list[str]:
     return [m for m in get_global_registry().list_metrics() if m not in _VALIDATION_METRIC_NAMES]
 
 
-def _param_alias(params: dict[str, Any]) -> str:
-    """Return the display alias from a params dict."""
-    return params.get("alias") or params["model"]
+def _param_alias(params: dict[str, Any] | None, fallback: str = "") -> str:
+    """Return the display alias from a params dict (``alias`` > ``model``), else ``fallback``.
+
+    AWS Transcribe/Polly have no ``model``/params, so callers pass the provider name as fallback.
+    """
+    if not params:
+        return fallback
+    return params.get("alias") or params.get("model") or fallback
 
 
 _elevenlabs_agent_cache: dict[str, dict[str, str]] = {}
@@ -101,6 +106,13 @@ def _fetch_elevenlabs_agent_models(s2s_params: dict[str, Any]) -> dict[str, str]
         return {"stt": "unknown", "llm": "unknown", "tts": "unknown"}
 
 
+# STT/TTS providers that authenticate via the standard AWS credential chain (AWS_ACCESS_KEY_ID /
+# AWS_SECRET_ACCESS_KEY[/AWS_SESSION_TOKEN] + AWS_REGION) instead of an api_key + model in *_PARAMS.
+_AWS_ENV_CREDENTIALED_STT_TTS: frozenset[str] = frozenset(
+    {"aws", "aws_transcribe", "amazon_transcribe", "transcribe", "aws_polly", "amazon_polly", "polly"}
+)
+
+
 class ModelConfig(BaseModel):
     """Flat model configuration covering all pipeline modes.
 
@@ -217,8 +229,8 @@ def pipeline_parts(self) -> dict[str, str]:
         match self.pipeline_type:
             case PipelineType.AUDIO_LLM:
                 return {
-                    "audio_llm": _param_alias(self.audio_llm_params),
-                    "tts": _param_alias(self.tts_params),
+                    "audio_llm": _param_alias(self.audio_llm_params, self.audio_llm or ""),
+                    "tts": _param_alias(self.tts_params, self.tts or ""),
                 }
             case PipelineType.S2S:
                 if self.s2s == "elevenlabs":
@@ -230,9 +242,9 @@ def pipeline_parts(self) -> dict[str, str]:
                 return {"s2s": _param_alias(self.s2s_params)}
             case PipelineType.CASCADE:
                 return {
-                    "stt": _param_alias(self.stt_params),
+                    "stt": _param_alias(self.stt_params, self.stt or ""),
                     "llm": self.llm,
-                    "tts": _param_alias(self.tts_params),
+                    "tts": _param_alias(self.tts_params, self.tts or ""),
                 }
 
     @model_validator(mode="before")
@@ -430,8 +442,53 @@ class OpenAIRealtimeSimulatorConfig(BaseModel):
     male_voice: str = Field("cedar", description="Voice used for male caller personas.")
 
 
+class BedrockS2SSimulatorConfig(BaseModel):
+    """Settings for the native Amazon Bedrock speech-to-speech user simulator.
+
+    EVA plays the caller with **Amazon Nova Sonic**, an audio-native S2S model, over
+    Bedrock's bidirectional streaming API. The agent's audio is streamed straight to
+    Nova Sonic and the caller's spoken reply streams back — no separate STT/LLM/TTS
+    cascade. Nova Sonic also emits both transcripts itself (agent ASR + caller text).
+
+    Requires the optional ``eva[bedrock-s2s]`` dependency (the experimental
+    ``aws-sdk-bedrock-runtime`` client, Python >=3.12) and AWS credentials in the
+    environment (``AWS_ACCESS_KEY_ID`` / ``AWS_SECRET_ACCESS_KEY`` /
+    ``AWS_SESSION_TOKEN``) plus ``AWS_REGION``.
+    """
+
+    provider: Literal["bedrock_s2s"] = "bedrock_s2s"
+    model_id: str = Field(
+        "amazon.nova-2-sonic-v1:0",
+        description="Nova Sonic model id (e.g. 'amazon.nova-2-sonic-v1:0' or 'amazon.nova-sonic-v1:0').",
+    )
+    region: str = Field("us-east-1", description="AWS region for the Bedrock bidirectional stream.")
+    female_voice: str = Field("tiffany", description="Nova Sonic voiceId used for female caller personas.")
+    male_voice: str = Field("matthew", description="Nova Sonic voiceId used for male caller personas.")
+    temperature: float = Field(0.7, description="Sampling temperature for the caller model.")
+    top_p: float = Field(0.9, description="Nucleus sampling top-p for the caller model.")
+    max_tokens: int = Field(1024, description="Max tokens per caller response (sessionStart inferenceConfiguration).")
+    endpointing_sensitivity: Literal["HIGH", "MEDIUM", "LOW"] = Field(
+        "HIGH",
+        description=(
+            "Nova Sonic turn-detection endpointing sensitivity. Higher = quicker to decide the agent "
+            "finished its turn (more responsive, but may cut in on a pausey agent). Lower = waits longer."
+        ),
+    )
+    output_sample_rate: Literal[8000, 16000, 24000] = Field(
+        16000,
+        description=(
+            "Nova Sonic audio output rate (Hz). 16000 matches the audio bridge input rate so no "
+            "resampling is needed on the caller-audio hop."
+        ),
+    )
+    playback_drain_seconds: float = Field(
+        15.0,
+        description="Max seconds to wait for the caller's audio to finish playing before ending its turn.",
+    )
+
+
 UserSimulatorConfig = Annotated[
-    ElevenLabsSimulatorConfig | OpenAIRealtimeSimulatorConfig,
+    ElevenLabsSimulatorConfig | OpenAIRealtimeSimulatorConfig | BedrockS2SSimulatorConfig,
     Field(discriminator="provider"),
 ]
 
@@ -730,6 +787,11 @@ def _validate_service_params(
             loc = ("model", service.lower())
             yield InitErrorDetails(type=PydanticCustomError("missing_service", message), loc=loc, input=provider)
 
+        # AWS Transcribe/Polly authenticate via the standard AWS credential chain (env vars), not an
+        # api_key, and have no "model" param — so their *_PARAMS are optional (region/voice/engine only).
+        if provider and provider.lower() in _AWS_ENV_CREDENTIALED_STT_TTS:
+            return
+
         required_keys = ["api_key", "model"]
         missing = [key for key in required_keys if key not in params] if params else required_keys
         if missing:

src/eva/orchestrator/runner.py

@@ -159,14 +159,15 @@ async def run(self, records: list[EvaluationRecord]) -> RunResult:
 
         # Resolve exact models used (captures defaults from services.py + any alias labels)
         if self.config.model.pipeline_type == PipelineType.CASCADE:
-            stt_params = self.config.model.stt_params
-            tts_params = self.config.model.tts_params
+            # AWS Transcribe/Polly have no model/params; fall back to the provider name.
+            stt_params = self.config.model.stt_params or {}
+            tts_params = self.config.model.tts_params or {}
             self.config.resolved_models = {
                 "stt_provider": self.config.model.stt,
-                "stt_model": stt_params["model"],
+                "stt_model": stt_params.get("model", self.config.model.stt),
                 "stt_alias": stt_params.get("alias"),
                 "tts_provider": self.config.model.tts,
-                "tts_model": tts_params["model"],
+                "tts_model": tts_params.get("model", self.config.model.tts),
                 "tts_alias": tts_params.get("alias"),
                 "llm": self.config.model.llm,
             }