From b55b419cc230b17791bc0b3de5623b942fa1c2d3 Mon Sep 17 00:00:00 2001 From: Katrina Date: Tue, 25 Aug 2026 19:27:40 -0400 Subject: [PATCH 1/3] add support for krisp vad and voice isolation --- src/eva/assistant/pipecat_server.py | 13 +++++++++++++ src/eva/assistant/pipeline/turn_config.py | 6 +++++- src/eva/models/config.py | 15 ++++++++++++++- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/eva/assistant/pipecat_server.py b/src/eva/assistant/pipecat_server.py index 1b52be12..7525aa92 100644 --- a/src/eva/assistant/pipecat_server.py +++ b/src/eva/assistant/pipecat_server.py @@ -11,6 +11,7 @@ import uvicorn from fastapi import FastAPI, WebSocket +from pipecat.audio.filters.krisp_viva_filter import KrispVivaFilter from pipecat.frames.frames import ( CancelFrame, LLMRunFrame, @@ -568,11 +569,23 @@ async def _on_turn_end(_service, transcript: str) -> None: def _create_transport(self, websocket) -> FastAPIWebsocketTransport: """Create the WebSocket transport with Twilio frame serialization.""" + audio_in_filter_cfg = self.pipeline_config.audio_in_filter.lower() + if audio_in_filter_cfg == "none": + audio_in_filter = None + elif audio_in_filter_cfg == "krisp": + audio_in_filter = KrispVivaFilter(**self.pipeline_config.audio_in_filter_params) + logger.info("Using Krisp VIVA audio-in filter") + else: + raise ValueError( + f"Unsupported audio_in_filter: {audio_in_filter_cfg}. Supported types: 'krisp_viva', 'none'" + ) + return FastAPIWebsocketTransport( websocket=websocket, params=FastAPIWebsocketParams( audio_in_enabled=True, audio_out_enabled=True, + audio_in_filter=audio_in_filter, add_wav_header=False, serializer=TwilioFrameSerializer( self.conversation_id, diff --git a/src/eva/assistant/pipeline/turn_config.py b/src/eva/assistant/pipeline/turn_config.py index 7208f638..11afb843 100644 --- a/src/eva/assistant/pipeline/turn_config.py +++ b/src/eva/assistant/pipeline/turn_config.py @@ -8,6 +8,7 @@ from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.krisp_viva_vad import KrispVivaVadAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams from pipecat.turns.user_start import ( @@ -49,8 +50,11 @@ def create_vad_analyzer(vad_type: str, vad_params: dict[str, Any]) -> VADAnalyze # Create VADParams, respecting existing defaults if no params specified params = VADParams(**vad_params) if vad_params else None return SileroVADAnalyzer(params=params) + elif vad_type_lower == "krisp_viva": + params = VADParams(**vad_params) if vad_params else None + return KrispVivaVadAnalyzer(params=params) else: - raise ValueError(f"Unsupported VAD type: {vad_type}. Supported types: 'silero', 'none'") + raise ValueError(f"Unsupported VAD type: {vad_type}. Supported types: 'silero', 'none', 'krisp_viva'") def create_turn_start_strategy( diff --git a/src/eva/models/config.py b/src/eva/models/config.py index 6ea2461d..cf960cd8 100644 --- a/src/eva/models/config.py +++ b/src/eva/models/config.py @@ -188,7 +188,7 @@ class ModelConfig(BaseModel): vad: str = Field( "silero", description=( - "VAD analyzer type: 'silero' or 'none'. Defaults to 'silero' (SileroVADAnalyzer). Use 'none' with external turn strategies (e.g. deepgram-flux) to skip local VAD. Set via EVA_MODEL__VAD." + "VAD analyzer type: 'silero', 'krisp_viva' or 'none'. Defaults to 'silero' (SileroVADAnalyzer). Use 'none' with external turn strategies (e.g. deepgram-flux) to skip local VAD. Set via EVA_MODEL__VAD." ), ) vad_params: dict[str, Any] = Field( @@ -198,6 +198,19 @@ class ModelConfig(BaseModel): ), ) + # Transport-level audio input filter + audio_in_filter: str = Field( + "none", + description=( + "Audio input filter applied to the transport: 'krisp_viva' or 'none'. Defaults to " + "'none'. Set via EVA_MODEL__AUDIO_IN_FILTER." + ), + ) + audio_in_filter_params: dict[str, Any] = Field( + {}, + description="Audio input filter parameters (JSON). Set via EVA_MODEL__AUDIO_IN_FILTER_PARAMS.", + ) + # CASCADE-only latency controls. pre_tool_speech: str = Field( "off", From d6c3a1ef83611d4250eb45e174417a221c5e7222 Mon Sep 17 00:00:00 2001 From: Katrina Date: Wed, 26 Aug 2026 10:09:17 -0400 Subject: [PATCH 2/3] only import krisp if krisp is selected for vad or audio-in filter --- src/eva/assistant/pipecat_server.py | 3 ++- src/eva/assistant/pipeline/turn_config.py | 3 ++- tests/unit/assistant/test_pipecat_server.py | 1 - vendor/krisp/README.md | 6 +++++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/eva/assistant/pipecat_server.py b/src/eva/assistant/pipecat_server.py index 7525aa92..af2f1258 100644 --- a/src/eva/assistant/pipecat_server.py +++ b/src/eva/assistant/pipecat_server.py @@ -11,7 +11,6 @@ import uvicorn from fastapi import FastAPI, WebSocket -from pipecat.audio.filters.krisp_viva_filter import KrispVivaFilter from pipecat.frames.frames import ( CancelFrame, LLMRunFrame, @@ -573,6 +572,8 @@ def _create_transport(self, websocket) -> FastAPIWebsocketTransport: if audio_in_filter_cfg == "none": audio_in_filter = None elif audio_in_filter_cfg == "krisp": + from pipecat.audio.filters.krisp_viva_filter import KrispVivaFilter + audio_in_filter = KrispVivaFilter(**self.pipeline_config.audio_in_filter_params) logger.info("Using Krisp VIVA audio-in filter") else: diff --git a/src/eva/assistant/pipeline/turn_config.py b/src/eva/assistant/pipeline/turn_config.py index 11afb843..d4e0be53 100644 --- a/src/eva/assistant/pipeline/turn_config.py +++ b/src/eva/assistant/pipeline/turn_config.py @@ -8,7 +8,6 @@ from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 -from pipecat.audio.vad.krisp_viva_vad import KrispVivaVadAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams from pipecat.turns.user_start import ( @@ -51,6 +50,8 @@ def create_vad_analyzer(vad_type: str, vad_params: dict[str, Any]) -> VADAnalyze params = VADParams(**vad_params) if vad_params else None return SileroVADAnalyzer(params=params) elif vad_type_lower == "krisp_viva": + from pipecat.audio.vad.krisp_viva_vad import KrispVivaVadAnalyzer + params = VADParams(**vad_params) if vad_params else None return KrispVivaVadAnalyzer(params=params) else: diff --git a/tests/unit/assistant/test_pipecat_server.py b/tests/unit/assistant/test_pipecat_server.py index f0efd8e0..e2dfc3a0 100644 --- a/tests/unit/assistant/test_pipecat_server.py +++ b/tests/unit/assistant/test_pipecat_server.py @@ -33,7 +33,6 @@ def _make_server(tmp_path: Path): srv._server_task = None srv._runner = None srv._metrics_observer = None - srv._latency_measurements = [] srv.num_seconds = 0 srv.pipeline_config = MagicMock() srv.conversation_id = "test-conv" diff --git a/vendor/krisp/README.md b/vendor/krisp/README.md index 7693fd51..f8953109 100644 --- a/vendor/krisp/README.md +++ b/vendor/krisp/README.md @@ -22,7 +22,9 @@ Result (wheel version and `.kef` set will vary by download): vendor/krisp/ ├── README.md (tracked) ├── krisp_audio-1.10.0-cp311-cp311-linux_x86_64.whl (git-ignored) -└── krisp-viva-tp-v3.kef (git-ignored) +├── krisp-viva-tp-v3.kef (git-ignored) +├── krisp-viva-vad-v2.kef (git-ignored) +└── krisp-viva-vi-tel-v2.1.kef (git-ignored) ``` > [!WARNING] @@ -35,6 +37,8 @@ In the Docker container: - The `eva` command (which is actually `scripts/docker_eva_wrapper.sh`) installs the `.whl`, if present. - The `KRISP_VIVA_TURN_MODEL_PATH` env var points at the `.kef` file directly on that mount, e.g. `vendor/krisp/krisp-viva-tp-v3.kef`. - The `KRISP_VIVA_API_KEY` env var provides the API key. +- If using the Krisp VAD, set `EVA_MODEL__VAD` to `krisp_viva` and point `KRISP_VIVA_VAD_MODEL_PATH` to the `krisp-viva-vad-v2.kef` file +- If using the Krisp audio input filter, set `EVA_MODEL__AUDIO_IN_FILTER` to `krisp_viva` and point `KRISP_VIVA_AUDIO_IN_FILTER_MODEL_PATH` to the `krisp-viva-vi-tel-v2.1.kef` file ## Runtime requirement From 622199c3eb2b1b442fef27dc66909da38a8d74ea Mon Sep 17 00:00:00 2001 From: Katrina Date: Thu, 27 Aug 2026 10:46:15 -0400 Subject: [PATCH 3/3] update audio in filter name to krisp_viva --- src/eva/assistant/pipecat_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eva/assistant/pipecat_server.py b/src/eva/assistant/pipecat_server.py index af2f1258..03319b11 100644 --- a/src/eva/assistant/pipecat_server.py +++ b/src/eva/assistant/pipecat_server.py @@ -571,7 +571,7 @@ def _create_transport(self, websocket) -> FastAPIWebsocketTransport: audio_in_filter_cfg = self.pipeline_config.audio_in_filter.lower() if audio_in_filter_cfg == "none": audio_in_filter = None - elif audio_in_filter_cfg == "krisp": + elif audio_in_filter_cfg == "krisp_viva": from pipecat.audio.filters.krisp_viva_filter import KrispVivaFilter audio_in_filter = KrispVivaFilter(**self.pipeline_config.audio_in_filter_params)