diff --git a/src/eva/assistant/pipecat_server.py b/src/eva/assistant/pipecat_server.py index 1b52be12..03319b11 100644 --- a/src/eva/assistant/pipecat_server.py +++ b/src/eva/assistant/pipecat_server.py @@ -568,11 +568,25 @@ async def _on_turn_end(_service, transcript: str) -> None: def _create_transport(self, websocket) -> FastAPIWebsocketTransport: """Create the WebSocket transport with Twilio frame serialization.""" + audio_in_filter_cfg = self.pipeline_config.audio_in_filter.lower() + if audio_in_filter_cfg == "none": + audio_in_filter = None + elif audio_in_filter_cfg == "krisp_viva": + from pipecat.audio.filters.krisp_viva_filter import KrispVivaFilter + + audio_in_filter = KrispVivaFilter(**self.pipeline_config.audio_in_filter_params) + logger.info("Using Krisp VIVA audio-in filter") + else: + raise ValueError( + f"Unsupported audio_in_filter: {audio_in_filter_cfg}. Supported types: 'krisp_viva', 'none'" + ) + return FastAPIWebsocketTransport( websocket=websocket, params=FastAPIWebsocketParams( audio_in_enabled=True, audio_out_enabled=True, + audio_in_filter=audio_in_filter, add_wav_header=False, serializer=TwilioFrameSerializer( self.conversation_id, diff --git a/src/eva/assistant/pipeline/turn_config.py b/src/eva/assistant/pipeline/turn_config.py index 7208f638..d4e0be53 100644 --- a/src/eva/assistant/pipeline/turn_config.py +++ b/src/eva/assistant/pipeline/turn_config.py @@ -49,8 +49,13 @@ def create_vad_analyzer(vad_type: str, vad_params: dict[str, Any]) -> VADAnalyze # Create VADParams, respecting existing defaults if no params specified params = VADParams(**vad_params) if vad_params else None return SileroVADAnalyzer(params=params) + elif vad_type_lower == "krisp_viva": + from pipecat.audio.vad.krisp_viva_vad import KrispVivaVadAnalyzer + + params = VADParams(**vad_params) if vad_params else None + return KrispVivaVadAnalyzer(params=params) else: - raise ValueError(f"Unsupported VAD type: {vad_type}. Supported types: 'silero', 'none'") + raise ValueError(f"Unsupported VAD type: {vad_type}. Supported types: 'silero', 'none', 'krisp_viva'") def create_turn_start_strategy( diff --git a/src/eva/models/config.py b/src/eva/models/config.py index 6ea2461d..cf960cd8 100644 --- a/src/eva/models/config.py +++ b/src/eva/models/config.py @@ -188,7 +188,7 @@ class ModelConfig(BaseModel): vad: str = Field( "silero", description=( - "VAD analyzer type: 'silero' or 'none'. Defaults to 'silero' (SileroVADAnalyzer). Use 'none' with external turn strategies (e.g. deepgram-flux) to skip local VAD. Set via EVA_MODEL__VAD." + "VAD analyzer type: 'silero', 'krisp_viva' or 'none'. Defaults to 'silero' (SileroVADAnalyzer). Use 'none' with external turn strategies (e.g. deepgram-flux) to skip local VAD. Set via EVA_MODEL__VAD." ), ) vad_params: dict[str, Any] = Field( @@ -198,6 +198,19 @@ class ModelConfig(BaseModel): ), ) + # Transport-level audio input filter + audio_in_filter: str = Field( + "none", + description=( + "Audio input filter applied to the transport: 'krisp_viva' or 'none'. Defaults to " + "'none'. Set via EVA_MODEL__AUDIO_IN_FILTER." + ), + ) + audio_in_filter_params: dict[str, Any] = Field( + {}, + description="Audio input filter parameters (JSON). Set via EVA_MODEL__AUDIO_IN_FILTER_PARAMS.", + ) + # CASCADE-only latency controls. pre_tool_speech: str = Field( "off", diff --git a/tests/unit/assistant/test_pipecat_server.py b/tests/unit/assistant/test_pipecat_server.py index f0efd8e0..e2dfc3a0 100644 --- a/tests/unit/assistant/test_pipecat_server.py +++ b/tests/unit/assistant/test_pipecat_server.py @@ -33,7 +33,6 @@ def _make_server(tmp_path: Path): srv._server_task = None srv._runner = None srv._metrics_observer = None - srv._latency_measurements = [] srv.num_seconds = 0 srv.pipeline_config = MagicMock() srv.conversation_id = "test-conv" diff --git a/vendor/krisp/README.md b/vendor/krisp/README.md index 7693fd51..f8953109 100644 --- a/vendor/krisp/README.md +++ b/vendor/krisp/README.md @@ -22,7 +22,9 @@ Result (wheel version and `.kef` set will vary by download): vendor/krisp/ ├── README.md (tracked) ├── krisp_audio-1.10.0-cp311-cp311-linux_x86_64.whl (git-ignored) -└── krisp-viva-tp-v3.kef (git-ignored) +├── krisp-viva-tp-v3.kef (git-ignored) +├── krisp-viva-vad-v2.kef (git-ignored) +└── krisp-viva-vi-tel-v2.1.kef (git-ignored) ``` > [!WARNING] @@ -35,6 +37,8 @@ In the Docker container: - The `eva` command (which is actually `scripts/docker_eva_wrapper.sh`) installs the `.whl`, if present. - The `KRISP_VIVA_TURN_MODEL_PATH` env var points at the `.kef` file directly on that mount, e.g. `vendor/krisp/krisp-viva-tp-v3.kef`. - The `KRISP_VIVA_API_KEY` env var provides the API key. +- If using the Krisp VAD, set `EVA_MODEL__VAD` to `krisp_viva` and point `KRISP_VIVA_VAD_MODEL_PATH` to the `krisp-viva-vad-v2.kef` file +- If using the Krisp audio input filter, set `EVA_MODEL__AUDIO_IN_FILTER` to `krisp_viva` and point `KRISP_VIVA_AUDIO_IN_FILTER_MODEL_PATH` to the `krisp-viva-vi-tel-v2.1.kef` file ## Runtime requirement