From 79f63e1adc034187e698b7a44abcf863312923ca Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 6 Aug 2026 19:28:06 -0400 Subject: [PATCH 01/65] remove unused Backend/Role scaffolding eva.backend and eva.role were added as scaffolding for a symmetric provider abstraction that was rejected in favor of a tick-based cascade design. Nothing imports them. --- src/eva/backend/__init__.py | 22 --- src/eva/backend/base.py | 279 -------------------------------- src/eva/backend/capabilities.py | 48 ------ src/eva/backend/factory.py | 55 ------- src/eva/role/__init__.py | 17 -- src/eva/role/assistant.py | 132 --------------- src/eva/role/base.py | 159 ------------------ src/eva/role/user.py | 83 ---------- 8 files changed, 795 deletions(-) delete mode 100644 src/eva/backend/__init__.py delete mode 100644 src/eva/backend/base.py delete mode 100644 src/eva/backend/capabilities.py delete mode 100644 src/eva/backend/factory.py delete mode 100644 src/eva/role/__init__.py delete mode 100644 src/eva/role/assistant.py delete mode 100644 src/eva/role/base.py delete mode 100644 src/eva/role/user.py diff --git a/src/eva/backend/__init__.py b/src/eva/backend/__init__.py deleted file mode 100644 index b9513915..00000000 --- a/src/eva/backend/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Provider-agnostic ``Backend`` abstraction (design-only, Step 1 of the refactor). - -This package defines the contracts described in ``docs/refactor-step1.md``: -pure API/session objects (``Backend``) that know nothing about role -(assistant vs. user), plus a factory to construct them. Nothing in this -package is wired into the existing ``eva.assistant`` / ``eva.user_simulator`` -code yet -- these are new, additive, currently-unused types. -""" - -from eva.backend.base import Backend, BackendEvent, BackendEventType, ToolCallRequest, ToolCallResult -from eva.backend.capabilities import BackendCapabilities -from eva.backend.factory import BackendFactory - -__all__ = [ - "Backend", - "BackendCapabilities", - "BackendEvent", - "BackendEventType", - "BackendFactory", - "ToolCallRequest", - "ToolCallResult", -] diff --git a/src/eva/backend/base.py b/src/eva/backend/base.py deleted file mode 100644 index 63bbd278..00000000 --- a/src/eva/backend/base.py +++ /dev/null @@ -1,279 +0,0 @@ -"""Abstract ``Backend`` contract: pure API/session exchange, no role knowledge. - -DESIGN ONLY (Step 1 of the refactor, see docs/refactor-step1.md). This module -defines shapes, not behavior -- every method body is a stub. Nothing in -``eva.assistant`` or ``eva.user_simulator`` depends on this yet. - -A ``Backend`` wraps exactly one provider integration (OpenAI Realtime, Gemini -Live, ElevenLabs Agents, a cascade STT->LLM->TTS pipeline, ...) and exposes a -uniform send/receive surface for exchanging audio, text, and tool-call -traffic with that provider. It has **no opinion about role** -- it does not -know whether it is being driven by an ``AssistantRole`` or a ``UserRole``, -and it does not decide *what* prompt or tools to use (the ``Role`` supplies -those at open-time and owns tool execution). - -Symmetry note (per the design doc): a ``Backend`` must not assume it is "the -network server side" or "the client side" of a connection. Today, -assistant-side backends happen to be reached by an inbound WebSocket -connection (Twilio-framed) and user-side backends happen to dial out to a -provider or to the assistant's socket. Both are just implementation details -of a concrete subclass's ``open()``/``send()``/``receive()`` -- the abstract -contract itself is direction-agnostic so that a later mediator can sit -between two ``Backend`` instances (Backend <-> mediator <-> Backend) without -requiring either side to be "the server." -""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from collections.abc import AsyncIterator -from dataclasses import dataclass, field -from enum import StrEnum -from typing import Any - -from eva.backend.capabilities import BackendCapabilities - - -class BackendEventType(StrEnum): - """Kinds of events a ``Backend`` can surface via ``receive()``. - - Not every ``Backend`` implementation will emit every event type -- a thin, - end-to-end backend (e.g. ElevenLabs Agents) may only ever emit - ``AUDIO_OUTPUT``, ``TRANSCRIPT``, ``TURN_END``, and ``ERROR``, because it - has no separable tool-calling seam of its own that the caller can observe - (tool calls, if any, happen inside the provider and are not surfaced). - Consumers must treat unhandled event types as ignorable, not as errors. - """ - - AUDIO_OUTPUT = "audio_output" - """A chunk of output audio from the backend (assistant speech, or the - simulated user's speech, depending on which role's Backend this is).""" - - TRANSCRIPT = "transcript" - """A (possibly partial) transcript of something spoken -- either the - backend's own output or, for backends that provide it, the other party's - input as heard by this backend's ASR.""" - - TOOL_CALL_REQUEST = "tool_call_request" - """The backend's model wants to invoke a tool. Only emitted by backends - that expose a separable tool-calling seam (native S2S realtime APIs, - cascade LLM backends). The owning ``Role`` is responsible for executing - the tool and returning the result via ``send(tool_result=...)`` -- the - ``Backend`` never executes tools itself (see docs/refactor-step1.md, - "tool execution stays role-side").""" - - TURN_END = "turn_end" - """The backend's model has finished its current turn (end-of-utterance / - end-of-response signal).""" - - ERROR = "error" - """A provider-level error occurred (connection drop, API error, etc.).""" - - -@dataclass -class ToolCallRequest: - """A tool invocation requested by a backend's underlying model. - - Surfaced via a ``BackendEvent`` of type ``TOOL_CALL_REQUEST``. The owning - ``Role`` executes the tool (via its own ``ToolExecutor``) and reports the - outcome back to the backend with ``Backend.send(tool_result=...)`` so the - provider's tool-calling loop can continue. - """ - - call_id: str - """Provider-assigned identifier correlating the request to its result.""" - - name: str - """Tool name as requested by the model.""" - - arguments: dict[str, Any] - """Parsed tool call arguments.""" - - -@dataclass -class ToolCallResult: - """The outcome of executing a ``ToolCallRequest``. - - To be sent back to the backend so its underlying model can continue the - tool-calling loop. - """ - - call_id: str - """Must match the ``call_id`` of the originating ``ToolCallRequest``.""" - - result: Any - """JSON-serializable tool result payload.""" - - -@dataclass -class BackendEvent: - """A single event surfaced by ``Backend.receive()``. - - Exactly one of the optional payload fields is populated, matching - ``event_type``. This is intentionally a loose envelope (rather than a - tagged union of dataclasses) so that thin backends can populate only the - fields they support without needing empty placeholder subclasses. - """ - - event_type: BackendEventType - audio: bytes | None = None - transcript: str | None = None - tool_call_request: ToolCallRequest | None = None - error: str | None = None - metadata: dict[str, Any] = field(default_factory=dict) - """Provider-specific extras (e.g. raw event name, timestamps) that don't - warrant a first-class field. Consumers should not rely on specific keys - being present across providers. - - Convention (not enforced by this contract): a backend that proactively - re-engages after a dropped user turn (the turn-end fallback; see - ``AssistantRole``'s ``turn_end_fallback_seconds`` and the shipped - ``eva.assistant.pipeline.fallback``) tags the ``AUDIO_OUTPUT``/ - ``TRANSCRIPT`` event it emits for that turn so callers can distinguish a - fallback nudge from an ordinary model turn (e.g. for audit logging and so - downstream metrics can zero it). The shipped feature records the transcript - marker with ``message_type="turn_fallback"``; a backend surfacing the same - turn here should carry an equivalent flag in ``metadata`` (e.g. - ``metadata["turn_fallback"] = True``). This is *not* a new event type -- a - nudge is just an ordinary turn from the backend's model, triggered by the - backend noticing that a user turn was never detected within the fallback - window rather than by new input; it flows through the same ``receive()`` - surface as anything else.""" - - -class Backend(ABC): - """Pure API/session exchange with one provider. No role knowledge. - - Lifecycle: ``open()`` establishes the session, ``send()`` pushes audio / - text / tool results to the provider, ``receive()`` yields events back, - and ``close()`` tears the session down. A ``Role`` (see - ``eva.role.base``) owns one ``Backend`` instance and drives it. - - Implementations are expected to fall along a spectrum: - - - **Native speech-to-speech** (OpenAI Realtime, Gemini Live): a single - persistent duplex session; ``send(audio=...)`` streams mic audio in, - ``receive()`` yields interleaved ``AUDIO_OUTPUT``/``TRANSCRIPT``/ - ``TOOL_CALL_REQUEST``/``TURN_END`` events as the provider produces them. - - **Cascade** (STT -> LLM -> TTS, e.g. a Pipecat pipeline): internally - composed of separate provider calls, but from the caller's perspective - still just one ``Backend`` -- it decides internally when to run STT, - call the LLM, and synthesize TTS, and surfaces the same event shape. - - **End-to-end / thin** (ElevenLabs Agents): the provider handles - everything (ASR, dialogue policy, TTS) opaquely. Such a ``Backend`` - may only ever emit ``AUDIO_OUTPUT``/``TRANSCRIPT``/``TURN_END``/ - ``ERROR`` and may treat ``send(tool_result=...)`` as a no-op or raise - ``NotImplementedError`` -- callers must consult ``capabilities`` and - not assume every method does something on every backend. - - Symmetry: this contract says nothing about which side dials out and - which side is dialed into -- see the module docstring. - """ - - @property - @abstractmethod - def capabilities(self) -> BackendCapabilities: - """Static capability flags for this backend (see ``BackendCapabilities``). - - Must be available even before ``open()`` is called (i.e. it describes - the provider integration, not live session state). - """ - ... - - @abstractmethod - async def open(self, *, system_prompt: str, tools: list[dict[str, Any]] | None, config: dict[str, Any]) -> None: - """Establish the provider session. - - Args: - system_prompt: Fully-built system prompt for this session, as - assembled by the owning ``Role`` (``Role.build_prompt()``). - A thin end-to-end backend still receives this even if it - maps it onto a different provider concept (e.g. ElevenLabs - agent overrides). - tools: Tool schemas to expose to the provider's model, in - whatever wire format the concrete backend needs to translate - from the agent's tool definitions. ``None`` or ``[]`` for - backends/roles that don't expose tool calling (e.g. a - ``UserRole`` that only needs an ``end_call`` tool would still - pass that single tool here; a backend with no tool-calling - seam at all may simply ignore this argument). - config: Provider-specific configuration blob (model name, voice, - sample rate, turn-detection parameters, etc.). Deliberately - untyped here -- each concrete ``Backend`` defines and - validates its own config shape; the abstract contract does - not prescribe one, since a native S2S config and a cascade - config share little structure. An ``AssistantRole`` backend - configured for the turn-end fallback (see - ``AssistantRole.turn_end_fallback_seconds``) reads its - threshold from this blob (e.g. a - ``config["turn_end_fallback_seconds"]`` key) the same way -- - the fallback needs no dedicated typed parameter or new - ``Backend`` method, since the resulting nudge is just an - ordinary outbound turn (see ``BackendEvent.metadata``). - - Must be safe to call exactly once per ``Backend`` instance. Must not - block on the other party being ready to exchange data -- readiness to - *accept* traffic is enough (mirrors today's - ``AbstractAssistantServer.start()`` contract: non-blocking, returns - once ready). - """ - ... - - @abstractmethod - async def send( - self, - *, - audio: bytes | None = None, - text: str | None = None, - tool_result: ToolCallResult | None = None, - ) -> None: - """Push data to the provider. Exactly one of the keyword args is set. - - Args: - audio: Raw input audio chunk (format/sample-rate is whatever this - backend's ``open(config=...)`` declared; format conversion is - the caller's responsibility via the shared audio utilities, - not this method's). - text: A text turn to inject directly (e.g. a starting utterance, - or a cascade backend's synthesized user/assistant text before - TTS). Backends that are audio-only end-to-end (no text - injection seam) may raise ``NotImplementedError``. - tool_result: The result of executing a previously-surfaced - ``ToolCallRequest``, to be relayed back into the provider's - tool-calling loop so it can continue. Backends with no - tool-calling seam (see ``capabilities``) may raise - ``NotImplementedError``. - - This is intentionally the single, symmetric outbound method for both - "network-server-like" and "client-like" backends -- see the module - docstring on symmetry. A future mediator sitting between two - ``Backend`` instances would call this same method on each side. - """ - ... - - @abstractmethod - def receive(self) -> AsyncIterator[BackendEvent]: - """Yield events from the provider as they arrive. - - The single, symmetric inbound stream for both "network-server-like" - and "client-like" backends. Must be an async generator (or return an - object implementing ``__aiter__``/``__anext__``) that yields until - the session ends (``close()`` is called, the provider disconnects, - or a terminal ``ERROR``/``TURN_END``-with-hangup event occurs -- - exact termination semantics are provider-specific and left to each - concrete backend). - """ - ... - - @abstractmethod - async def close(self) -> None: - """Tear down the provider session. - - Must be safe to call even if ``open()`` was never called or the - session already ended on its own (idempotent). Concrete backends are - responsible for their own provider-specific teardown (closing - websockets, cancelling tasks, flushing buffers); this method does not - itself define audio/output persistence -- that remains a ``Role`` - concern (see ``eva.role.base``). - """ - ... diff --git a/src/eva/backend/capabilities.py b/src/eva/backend/capabilities.py deleted file mode 100644 index 7561872b..00000000 --- a/src/eva/backend/capabilities.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Capability flags describing what a ``Backend`` implementation can do. - -These flags exist so that a future mediator/turn-taking layer can branch on -backend shape without every ``Backend`` implementation needing to expose the -same granular seams. Per docs/refactor-step1.md, they are declared now but -must remain UNUSED in this step -- no turn-taking logic should read them yet. -""" - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class BackendCapabilities: - """Static, provider-declared capability flags for a ``Backend``. - - Each concrete ``Backend`` subclass sets these once (typically as a class - attribute or constructed in ``__init__``) to describe its streaming shape. - They are informational only in this step -- nothing consumes them yet. - - Attributes: - emits_continuous_audio: True if the backend produces a continuous - audio stream once speaking starts (native speech-to-speech models - such as OpenAI Realtime or Gemini Live, and end-to-end providers - such as ElevenLabs Agents). False for cascade backends - (STT -> LLM -> TTS) that emit discrete, chunked utterances with - gaps between TTS renders. This is the flag a future mediator uses - to decide whether "audio is still arriving" is a meaningful - signal on its own, or whether it must also track discrete - utterance boundaries. - supports_streaming_interruption: True if the underlying provider API - supports being told "stop talking now" mid-utterance and reacting - immediately (e.g. OpenAI/Gemini Realtime `response.cancel`-style - semantics). False if interruption can only be approximated by the - caller (e.g. stop forwarding audio, drop the rest of a queued TTS - buffer) rather than being a first-class provider feature. Declared - for later interruption-policy work; not consumed in this step. - owns_playout_clock: True if the backend itself is responsible for - audio playout pacing (e.g. a cascade backend streaming TTS chunks - at wall-clock rate), which is required for the barge-in work - planned for a later phase. False if playout pacing is delegated - to the caller/mediator, or if the backend has no continuous - playout concept at all (e.g. a fully end-to-end provider that - hands back a finished audio blob). - """ - - emits_continuous_audio: bool - supports_streaming_interruption: bool - owns_playout_clock: bool diff --git a/src/eva/backend/factory.py b/src/eva/backend/factory.py deleted file mode 100644 index d0fa61b2..00000000 --- a/src/eva/backend/factory.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Factory interface for constructing ``Backend`` instances by name. - -DESIGN ONLY (Step 1 of the refactor). Mirrors the shape of today's -``eva.user_simulator.factory.create_user_simulator`` (lazy per-provider -imports keyed off config type) but is provider-and-role-agnostic: the same -factory is meant to be usable to build a backend for either an -``AssistantRole`` or a ``UserRole``, since a ``Backend`` has no role -knowledge (that's the whole point of the split -- see docs/refactor-step1.md, -"lets any backend act as either role"). -""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import Any - -from eva.backend.base import Backend - - -class BackendFactory(ABC): - """Constructs a ``Backend`` for a named provider from a config blob. - - A concrete implementation is expected to hold (or look up) a registry - mapping provider name -> ``Backend`` subclass, analogous to today's - ``create_user_simulator`` / assistant-server construction in - ``orchestrator/runner.py``, and to import each provider module lazily so - that unused providers' SDKs need not be installed/imported. - """ - - @abstractmethod - def create(self, name: str, config: dict[str, Any]) -> Backend: - """Construct and return a not-yet-opened ``Backend``. - - Args: - name: Provider identifier (e.g. ``"openai_realtime"``, - ``"gemini_live"``, ``"elevenlabs"``, ``"cascade"``). The set - of valid names is defined by the concrete factory's registry, - not by this interface. - config: Provider-specific configuration understood by that - backend's ``open()`` (see ``Backend.open``). This factory - does not validate the shape of ``config`` beyond dispatching - on ``name`` -- each ``Backend`` subclass is responsible for - validating its own config. - - Returns: - A constructed ``Backend`` instance. The returned backend has not - had ``open()`` called on it yet -- construction and session - establishment are separate steps so a ``Role`` can construct its - backend early (e.g. at record setup) and open the session later - (e.g. once the other party is ready). - - Raises: - ValueError: if ``name`` does not match a known provider. - """ - ... diff --git a/src/eva/role/__init__.py b/src/eva/role/__init__.py deleted file mode 100644 index c005c0a7..00000000 --- a/src/eva/role/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Provider-agnostic ``Role`` abstraction (design-only, Step 1 of the refactor). - -A ``Role`` owns everything that today is duplicated across the assistant and -user-simulator stacks per-provider: prompt construction, tool ownership, and -goal/persona/agent-config data. Each ``Role`` holds exactly one -``eva.backend.Backend`` instance, created at runtime via a -``eva.backend.BackendFactory``. - -Nothing in this package is wired into the existing ``eva.assistant`` / -``eva.user_simulator`` code yet. -""" - -from eva.role.assistant import AssistantRole -from eva.role.base import Role -from eva.role.user import UserRole - -__all__ = ["AssistantRole", "Role", "UserRole"] diff --git a/src/eva/role/assistant.py b/src/eva/role/assistant.py deleted file mode 100644 index 897dba14..00000000 --- a/src/eva/role/assistant.py +++ /dev/null @@ -1,132 +0,0 @@ -"""``AssistantRole`` contract: the business-side answering role. - -DESIGN ONLY (Step 1 of the refactor, see docs/refactor-step1.md). Method -bodies are stubs; nothing here is wired into the existing code path yet. - -Plug-in point (where this will eventually replace existing code): - Today the assistant side is a concrete ``AbstractAssistantServer`` - subclass selected by ``eva.orchestrator.worker._get_server_class(framework)`` - (worker.py) and constructed + started inside - ``ConversationWorker._start_assistant()`` (worker.py), which calls - ``server_cls(...).start()``. Its outputs are flushed by - ``ConversationWorker._cleanup()`` via ``server.stop()`` (which internally - calls ``save_outputs()``), and ``get_conversation_stats()`` / - ``get_final_scenario_db()`` are read back in ``ConversationWorker.run()``. - - In a later phase, ``_start_assistant()`` becomes the construction site for - an ``AssistantRole`` (framework string -> ``backend_name`` passed to the - ``BackendFactory``), and the worker drives ``role.run()`` / - ``role.save_outputs()`` / ``role.get_final_scenario_db()`` instead of the - server's own lifecycle methods. The provider-specific server subclasses - collapse into ``Backend`` implementations behind the factory; the - role-agnostic orchestration in ``ConversationWorker`` stays put. This - module is deliberately separate from ``eva.role.user`` so that migration - can land assistant-side first without touching the user-side diff. -""" - -from __future__ import annotations - -from abc import abstractmethod -from typing import Any - -from eva.backend.factory import BackendFactory -from eva.role.base import Role - - -class AssistantRole(Role): - """Role that answers on behalf of the business (today's "assistant server"). - - Carries agent configuration and tool catalog; owns a ``ToolExecutor`` - (constructed by subclasses, not by this contract) to fulfill - ``handle_tool_call_request``. - - Turn-end fallback (self-nudge): the assistant's backstop for a *dropped - user turn*. When VAD / turn detection silently fails to fire for a real - user utterance, the call would otherwise hang until the provider's - inactivity timeout ends it. After the assistant stops speaking, if no user - turn is detected within ``turn_end_fallback_seconds``, the assistant - proactively re-engages with a nudge (acknowledge-and-answer if partial - user speech/audio was captured, otherwise ask the caller to repeat). This - is the seam already shipped as the pipeline-side ``TurnEndFallbackTimer`` - (see ``eva.assistant.pipeline.fallback`` and ``EVA_TURN_END_FALLBACK_TIME``); - it works for both cascade and audio-LLM pipelines. - - Two policies the backend owns, carried over from the shipped feature: - - Give up after a small number of *consecutive* nudges without a real user - turn resetting the count (``MAX_CONSECUTIVE_FALLBACK_NUDGES``), then let - the provider's inactivity backstop end the call. - - Never nudge once the call is ending (a nudge during teardown produces a - phantom assistant turn after the conversation is logically closed). - - Unlike the tool-call/idle-detection seams elsewhere in this contract, the - fallback needs no new ``Role`` method and no new ``Backend`` event type: - the nudge is just an ordinary outbound turn that this role's backend - produces on its own after the timeout, using the same - ``system_prompt``/instructions already established at ``open()`` time (see - ``Backend.open``'s ``config`` docstring). It is surfaced through the normal - ``receive()`` stream and tagged so downstream metrics can identify and zero - it (the shipped feature records the transcript marker with - ``message_type="turn_fallback"``; see ``BackendEvent.metadata``). Whether - the *other* side (a ``UserRole``) needs to do anything special upon - receiving it, versus just treating it as an ordinary assistant turn through - its existing ``run()`` loop, is left open -- see docs/refactor-step1.md - discussion; nothing here requires ``UserRole`` changes to handle it today. - """ - - def __init__( - self, - *, - backend_factory: BackendFactory, - backend_name: str, - backend_config: dict[str, Any], - agent_config_path: str, - scenario_db_path: str, - current_date_time: str, - turn_end_fallback_seconds: float | None = None, - ) -> None: - """Initialize the assistant role. - - Args: - backend_factory: Factory used to construct the backend. - backend_name: Key passed to the factory to select a backend. - backend_config: Provider-specific configuration for the backend. - agent_config_path: Path to the agent YAML (role, instructions, - tool schemas) -- mirrors ``AbstractAssistantServer.agent`` / - ``agent_config_path``. - scenario_db_path: Path to the per-record scenario database JSON - consumed by tool execution -- mirrors - ``AbstractAssistantServer.scenario_db_path``. - current_date_time: Current date/time string threaded into both - prompt construction and tool execution (mirrors existing - ``current_date_time`` plumbing throughout the assistant - stack). - turn_end_fallback_seconds: How long after the assistant stops - speaking to wait for a user turn before firing a turn-end - fallback nudge, or ``None`` to disable the fallback entirely - (preserving the old behavior of waiting for the provider's - inactivity timeout). Mirrors the shipped - ``EVA_TURN_END_FALLBACK_TIME`` knob. This is an - ``AssistantRole``-level tuning value, not a - ``BackendCapabilities`` flag (capabilities describe what a - backend *can* do, statically). Wiring it into the constructed - ``self.backend``'s own config (via ``backend_config`` / - ``Backend.open(config=...)``) is left to the concrete - subclass's constructor, same as elsewhere in this contract -- - a ``Role`` does not otherwise reach into backend config after - construction. A backend with no notion of idle timing (e.g. a - thin end-to-end backend that relies on its own provider - backstop) may simply ignore this value. - """ - super().__init__(backend_factory=backend_factory, backend_name=backend_name, backend_config=backend_config) - self.agent_config_path = agent_config_path - self.scenario_db_path = scenario_db_path - self.current_date_time = current_date_time - self.turn_end_fallback_seconds = turn_end_fallback_seconds - - @abstractmethod - def get_final_scenario_db(self) -> dict[str, Any]: - """Return the (possibly mutated) scenario database state, for metrics. - - Mirrors ``AbstractAssistantServer.get_final_scenario_db()``. - """ - ... diff --git a/src/eva/role/base.py b/src/eva/role/base.py deleted file mode 100644 index a75b4fa1..00000000 --- a/src/eva/role/base.py +++ /dev/null @@ -1,159 +0,0 @@ -"""Abstract ``Role`` base contract: prompt/tools/goal ownership over a ``Backend``. - -DESIGN ONLY (Step 1 of the refactor, see docs/refactor-step1.md). Every -method body here is a stub -- this module defines shapes, not behavior, and -is not imported by any existing code path. - -This module holds only the shared ``Role`` base. The two concrete roles live -in sibling modules -- ``AssistantRole`` in ``eva.role.assistant`` and -``UserRole`` in ``eva.role.user`` -- since each carries a meaningfully -different data payload and plug-in point (see those modules' docstrings) and -is expected to grow its own implementation in later phases; keeping them in -separate files keeps each phase's diff scoped to one role. - -Design choice -- one ``Role`` base with ``AssistantRole``/``UserRole`` -subclasses, rather than two unrelated ABCs: - Both roles share an identical *control loop* shape: construct a backend - via ``BackendFactory``, ``build_prompt()`` before opening it, drive - ``backend.receive()`` and dispatch tool-call requests to - ``handle_tool_call_request()``, and record recorded audio/transcript for - output. What differs between them is only the *data* they carry (agent - config + tool catalog for the assistant; goal + persona + starting - utterance for the user) and how they decide the conversation is over. - That's a difference in constructor args and a couple of abstract methods, - not in control flow -- so one shared base with two thin subclasses avoids - duplicating the event loop, while still keeping tool-ownership and - prompt-building role-specific via abstract methods. If the two roles' - control loops diverge significantly in a later phase, splitting them - apart is a mechanical extraction of ``Role`` into two ABCs -- nothing - here should make that harder. -""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from pathlib import Path -from typing import Any - -from eva.backend.base import Backend, ToolCallRequest, ToolCallResult -from eva.backend.factory import BackendFactory - - -class Role(ABC): - """Owns prompt, tools/goal, and a runtime-created ``Backend``. - - A ``Role`` is the thing that used to be split across - ``AbstractAssistantServer`` (assistant side) and ``AbstractUserSimulator`` - (user side): everything that is *not* pure provider API exchange lives - here instead of in ``Backend``. In particular: - - - Tool execution stays role-side (per docs/refactor-step1.md): a ``Role`` - is responsible for turning a ``ToolCallRequest`` surfaced by its - backend into a ``ToolCallResult``, using whatever execution engine is - appropriate for that role (``ToolExecutor`` for ``AssistantRole``; a - trivial/no-op handler for ``UserRole``, which today only exposes a - synthetic ``end_call`` tool). ``Backend`` implementations never - execute tools themselves. - - Prompt construction stays role-side: ``build_prompt()`` replaces both - ``AbstractAssistantServer._build_system_prompt()`` / - ``AgenticSystem``'s prompt building and - ``AbstractUserSimulator._build_prompt()``. - - Audio recording / output persistence is a role-side concern shared by - both subclasses (see ``docs/refactor-step1.md`` point 5, "consolidate - audio recording / output-saving into one shared helper") -- this base - class declares the seam (``record_audio`` / ``save_outputs``) but does - not implement the shared helper itself; that helper is later work. - """ - - def __init__(self, *, backend_factory: BackendFactory, backend_name: str, backend_config: dict[str, Any]) -> None: - """Construct the role's backend (but do not open its session yet). - - Args: - backend_factory: Factory used to construct ``self.backend``. - backend_name: Provider name passed through to - ``BackendFactory.create``. - backend_config: Provider-specific config passed through to - ``BackendFactory.create`` (not to be confused with the - ``config`` argument of ``Backend.open``, which is also - provider-specific but may be augmented by the role at - open-time, e.g. with a resolved sample rate). - """ - self.backend: Backend = backend_factory.create(backend_name, backend_config) - - @abstractmethod - def build_prompt(self) -> str: - """Build the full system prompt / instructions for this role. - - For ``AssistantRole`` this replaces - ``AbstractAssistantServer._build_system_prompt()`` and - ``AgenticSystem``'s inline prompt construction. For ``UserRole`` this - replaces ``AbstractUserSimulator._build_prompt()``. Called before - ``Backend.open()`` so the result can be passed as its - ``system_prompt`` argument. - """ - ... - - @abstractmethod - async def handle_tool_call_request(self, request: ToolCallRequest) -> ToolCallResult: - """Execute a tool call surfaced by this role's backend and return the result. - - This is the single place tool execution happens for this role -- - ``Backend`` implementations must never execute tools directly (see - class docstring). Implementations should log the call/result (e.g. - to an audit log) as part of executing it. - """ - ... - - @abstractmethod - async def run(self) -> str: - """Drive the conversation for this role until it reaches a terminal state. - - Expected shape (left to subclasses to implement, not prescribed in - detail here since the exact loop depends on the backend's - capabilities -- see ``BackendCapabilities``): - 1. ``await self.backend.open(system_prompt=self.build_prompt(), ...)`` - 2. Iterate ``self.backend.receive()``, dispatching - ``TOOL_CALL_REQUEST`` events to ``handle_tool_call_request`` and - feeding the ``ToolCallResult`` back via - ``self.backend.send(tool_result=...)``. - 3. Record audio/transcript events as they arrive (see - ``record_audio``). - 4. On a terminal event (hangup, timeout, transfer, error), call - ``await self.backend.close()`` and return an end-reason string. - - Returns: - A short end-reason string (e.g. ``"goodbye"``, ``"transfer"``, - ``"timeout"``, ``"error"``) -- mirrors the return contract of - today's ``AbstractUserSimulator.run_conversation()``. - """ - ... - - @abstractmethod - def record_audio(self, source: str, audio_data: bytes) -> None: - """Accumulate a chunk of audio for later persistence. - - Args: - source: Role-defined stream label (e.g. ``"user"``, - ``"assistant"``, or a cleaned/pre-perturbation variant). - Mirrors ``AbstractUserSimulator._record_audio`` / - ``AbstractAssistantServer``'s audio-buffer fields; the exact - set of valid labels is left to subclasses/shared helper, not - fixed by this contract. - audio_data: Raw PCM16 bytes at this role's recording sample rate. - """ - ... - - @abstractmethod - async def save_outputs(self, output_dir: Path) -> None: - """Persist this role's output artifacts to ``output_dir``. - - For ``AssistantRole`` this covers ``audit_log.json``, - ``transcript.jsonl``, scenario DB snapshots (mirrors - ``AbstractAssistantServer.save_outputs``). For ``UserRole`` this - covers ``user_simulator_events.jsonl`` (mirrors the event logger in - ``AbstractUserSimulator``). Audio WAV files are expected to be - written by the shared audio-recording helper referenced in - ``record_audio``, not necessarily by this method -- exact division of - labor is left to the later implementation phase. - """ - ... diff --git a/src/eva/role/user.py b/src/eva/role/user.py deleted file mode 100644 index 415d8b7a..00000000 --- a/src/eva/role/user.py +++ /dev/null @@ -1,83 +0,0 @@ -"""``UserRole`` contract: the simulated-caller role. - -DESIGN ONLY (Step 1 of the refactor, see docs/refactor-step1.md). Method -bodies are stubs; nothing here is wired into the existing code path yet. - -Plug-in point (where this will eventually replace existing code): - Today the user side is a concrete ``AbstractUserSimulator`` subclass - selected by ``eva.user_simulator.factory.create_user_simulator(config, ...)`` - and constructed inside ``ConversationWorker._start_user_simulator()`` - (worker.py), which passes it ``server_url=f"ws://localhost:{port}/ws"`` - to reach the assistant server. The conversation is driven by - ``ConversationWorker._run_conversation()`` calling - ``user_simulator.run_conversation()``, whose returned end-reason string - becomes the conversation result. - - In a later phase, ``_start_user_simulator()`` becomes the construction - site for a ``UserRole`` (simulator config -> ``backend_name`` + - ``backend_config`` for the ``BackendFactory``), and the worker drives - ``role.run()`` (returning the same end-reason string via - ``get_end_reason()``) instead of ``run_conversation()``. Note the - ``server_url`` handoff is a *transport* detail that today's user side owns - directly; per docs/refactor-step1.md the ``Backend`` contract is kept - direction-agnostic precisely so this WS-connect concern can move into a - ``Backend`` implementation (or, later, a mediator) without the role - caring. This module is deliberately separate from ``eva.role.assistant`` - so the user-side migration can land as its own scoped diff. -""" - -from __future__ import annotations - -from abc import abstractmethod -from typing import Any - -from eva.backend.factory import BackendFactory -from eva.role.base import Role - - -class UserRole(Role): - """Role that simulates the human caller (today's "user simulator"). - - Carries goal/persona instead of agent config/tools -- its tool surface, - if any, is limited to caller-side affordances like ``end_call`` (see - ``END_CALL_DESCRIPTION`` in today's ``eva.user_simulator.base``), not a - business tool catalog. - """ - - def __init__( - self, - *, - backend_factory: BackendFactory, - backend_name: str, - backend_config: dict[str, Any], - goal: dict[str, Any], - persona_config: dict[str, Any], - current_date_time: str, - ) -> None: - """Initialize the user role. - - Args: - backend_factory: Factory used to construct the backend. - backend_name: Key passed to the factory to select a backend. - backend_config: Provider-specific configuration for the backend. - goal: User goal / decision-tree data -- mirrors - ``AbstractUserSimulator.goal``. - persona_config: Persona/voice/behavior configuration -- mirrors - ``AbstractUserSimulator.persona_config``. - current_date_time: Threaded into prompt construction, mirroring - existing plumbing. - """ - super().__init__(backend_factory=backend_factory, backend_name=backend_name, backend_config=backend_config) - self.goal = goal - self.persona_config = persona_config - self.current_date_time = current_date_time - - @abstractmethod - def get_end_reason(self) -> str: - """Return the terminal end-reason for this conversation. - - Mirrors the return value of today's - ``AbstractUserSimulator.run_conversation()`` (``"goodbye"``, - ``"transfer"``, ``"timeout"``, ``"error"``, ...). - """ - ... From edcbda8e30faf985c46ff279bb205230f3c75c02 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 6 Aug 2026 19:33:17 -0400 Subject: [PATCH 02/65] add CascadeSimulatorConfig --- src/eva/models/config.py | 17 ++++++++++++++++- tests/unit/models/test_config_models.py | 23 +++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/eva/models/config.py b/src/eva/models/config.py index 315bf4eb..cdc3242a 100644 --- a/src/eva/models/config.py +++ b/src/eva/models/config.py @@ -472,8 +472,23 @@ class OpenAIRealtimeSimulatorConfig(BaseModel): male_voice: str = Field("cedar", description="Voice used for male caller personas.") +class CascadeSimulatorConfig(BaseModel): + """Self-hosted STT/LLM/TTS caller pipeline with tick-based turn-taking.""" + + provider: Literal["cascade"] = "cascade" + + stt: str = Field("elevenlabs", description="Streaming STT provider for transcribing assistant audio.") + stt_params: dict[str, Any] = Field(default_factory=lambda: {"model": "scribe_v2_realtime"}) + + llm: str = Field("gpt-5.5", description="Caller LLM, resolved via the same EVA_MODEL_LIST router as the assistant.") + llm_params: dict[str, Any] = Field(default_factory=dict) + + tts: str = Field("cartesia", description="TTS provider for the simulated caller's speech.") + tts_params: dict[str, Any] = Field(default_factory=lambda: {"model": "sonic-3.5"}) + + UserSimulatorConfig = Annotated[ - ElevenLabsSimulatorConfig | OpenAIRealtimeSimulatorConfig, + ElevenLabsSimulatorConfig | OpenAIRealtimeSimulatorConfig | CascadeSimulatorConfig, Field(discriminator="provider"), ] diff --git a/tests/unit/models/test_config_models.py b/tests/unit/models/test_config_models.py index d797dbcd..9d1e2d41 100644 --- a/tests/unit/models/test_config_models.py +++ b/tests/unit/models/test_config_models.py @@ -1225,3 +1225,26 @@ def test_openai_realtime_rejects_accent_perturbation_during_config_load(self): user_simulator={"provider": "openai_realtime"}, perturbation={"accent": "french"}, ) + + +def test_cascade_simulator_config_defaults(): + from eva.models.config import CascadeSimulatorConfig + + config = CascadeSimulatorConfig() + + assert config.provider == "cascade" + assert config.stt == "elevenlabs" + assert config.stt_params["model"] == "scribe_v2_realtime" + assert config.tts == "cartesia" + assert config.tts_params["model"] == "sonic-3.5" + assert config.llm == "gpt-5.5" + + +def test_user_simulator_union_discriminates_cascade(): + from pydantic import TypeAdapter + + from eva.models.config import CascadeSimulatorConfig, UserSimulatorConfig + + parsed = TypeAdapter(UserSimulatorConfig).validate_python({"provider": "cascade"}) + + assert isinstance(parsed, CascadeSimulatorConfig) From aadcc13ea7c2207d65c224ebd937f8281a86c5f8 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 6 Aug 2026 19:39:18 -0400 Subject: [PATCH 03/65] fix stale accent-perturbation guard, add cascade param docs and tests --- src/eva/models/config.py | 19 ++++++++++++++----- tests/unit/models/test_config_models.py | 22 ++++++++++++++++++++++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/eva/models/config.py b/src/eva/models/config.py index cdc3242a..a4bbafdd 100644 --- a/src/eva/models/config.py +++ b/src/eva/models/config.py @@ -478,13 +478,22 @@ class CascadeSimulatorConfig(BaseModel): provider: Literal["cascade"] = "cascade" stt: str = Field("elevenlabs", description="Streaming STT provider for transcribing assistant audio.") - stt_params: dict[str, Any] = Field(default_factory=lambda: {"model": "scribe_v2_realtime"}) + stt_params: dict[str, Any] = Field( + default_factory=lambda: {"model": "scribe_v2_realtime"}, + description="Provider-native keyword arguments passed through to the STT client.", + ) llm: str = Field("gpt-5.5", description="Caller LLM, resolved via the same EVA_MODEL_LIST router as the assistant.") - llm_params: dict[str, Any] = Field(default_factory=dict) + llm_params: dict[str, Any] = Field( + default_factory=dict, + description="Provider-native keyword arguments passed through to the caller LLM client.", + ) tts: str = Field("cartesia", description="TTS provider for the simulated caller's speech.") - tts_params: dict[str, Any] = Field(default_factory=lambda: {"model": "sonic-3.5"}) + tts_params: dict[str, Any] = Field( + default_factory=lambda: {"model": "sonic-3.5"}, + description="Provider-native keyword arguments passed through to the TTS client.", + ) UserSimulatorConfig = Annotated[ @@ -742,13 +751,13 @@ def _check_companion_services(self) -> "RunConfig": config is unused and conflicting env vars are harmless. """ if ( - isinstance(self.user_simulator, OpenAIRealtimeSimulatorConfig) + not isinstance(self.user_simulator, ElevenLabsSimulatorConfig) and self.perturbation is not None and self.perturbation.accent is not None ): raise ValueError( "Accent perturbations require the ElevenLabs user simulator; " - "OpenAI Realtime supports behavior, noise, and connection perturbations." + "other providers support behavior, noise, and connection perturbations." ) if self.max_rerun_attempts == 0 or self.aggregate_only: diff --git a/tests/unit/models/test_config_models.py b/tests/unit/models/test_config_models.py index 9d1e2d41..5ddeb376 100644 --- a/tests/unit/models/test_config_models.py +++ b/tests/unit/models/test_config_models.py @@ -1226,6 +1226,28 @@ def test_openai_realtime_rejects_accent_perturbation_during_config_load(self): perturbation={"accent": "french"}, ) + def test_cascade_rejects_accent_perturbation_during_config_load(self): + with pytest.raises(ValidationError, match="Accent perturbations require the ElevenLabs user simulator"): + _config( + env_vars=_BASE_ENV, + user_simulator={"provider": "cascade"}, + perturbation={"accent": "french"}, + ) + + def test_cascade_nested_environment_configuration(self): + from eva.models.config import CascadeSimulatorConfig + + config = _config( + env_vars=_BASE_ENV + | { + "EVA_USER_SIMULATOR__PROVIDER": "cascade", + "EVA_USER_SIMULATOR__STT_PARAMS": json.dumps({"model": "x"}), + } + ) + + assert isinstance(config.user_simulator, CascadeSimulatorConfig) + assert config.user_simulator.stt_params == {"model": "x"} + def test_cascade_simulator_config_defaults(): from eva.models.config import CascadeSimulatorConfig From 91dd94b1f827cd2997caa6ce7d8edbd746a6b4bb Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 6 Aug 2026 19:45:25 -0400 Subject: [PATCH 04/65] add cascade package with tick timing constants --- src/eva/user_simulator/cascade/__init__.py | 0 src/eva/user_simulator/cascade/constants.py | 37 +++++++++++++++++++ tests/unit/user_simulator/cascade/__init__.py | 0 .../user_simulator/cascade/test_constants.py | 22 +++++++++++ 4 files changed, 59 insertions(+) create mode 100644 src/eva/user_simulator/cascade/__init__.py create mode 100644 src/eva/user_simulator/cascade/constants.py create mode 100644 tests/unit/user_simulator/cascade/__init__.py create mode 100644 tests/unit/user_simulator/cascade/test_constants.py diff --git a/src/eva/user_simulator/cascade/__init__.py b/src/eva/user_simulator/cascade/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/eva/user_simulator/cascade/constants.py b/src/eva/user_simulator/cascade/constants.py new file mode 100644 index 00000000..2ce4927b --- /dev/null +++ b/src/eva/user_simulator/cascade/constants.py @@ -0,0 +1,37 @@ +"""Timing constants for the tick-based cascade user simulator. + +Values mirror tau-voice (tau2-bench/src/tau2/config.py:104-113). These are +module constants rather than config fields on purpose: there is no run-level +reason to vary them, and exposing them would let runs drift apart in ways that +make their metrics incomparable. +""" + +TICK_DURATION_MS = 200 +"""One simulation tick. Carries ten 20ms mulaw frames on the wire.""" + +WAIT_TO_RESPOND_OTHER_MS = 1000 +"""Silence required from the assistant before the caller starts a turn.""" + +WAIT_TO_RESPOND_SELF_MS = 5000 +"""Silence required from the caller itself before it starts another turn.""" + +YIELD_WHEN_INTERRUPTED_MS = 1000 +"""How long the caller keeps talking after the assistant barges in.""" + +YIELD_WHEN_INTERRUPTING_MS = 5000 +"""How long the caller holds the floor after barging in itself.""" + +CALLER_SAMPLE_RATE = 16000 +"""PCM16 sample rate for the caller's own audio track.""" + +_BYTES_PER_SAMPLE = 2 + +BYTES_PER_TICK = CALLER_SAMPLE_RATE * TICK_DURATION_MS // 1000 * _BYTES_PER_SAMPLE +"""PCM16 bytes carried per tick at CALLER_SAMPLE_RATE.""" + +SILENCE_BYTE = b"\x00" + + +def ms_to_ticks(milliseconds: int) -> int: + """Convert milliseconds to whole ticks, flooring.""" + return milliseconds // TICK_DURATION_MS diff --git a/tests/unit/user_simulator/cascade/__init__.py b/tests/unit/user_simulator/cascade/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/user_simulator/cascade/test_constants.py b/tests/unit/user_simulator/cascade/test_constants.py new file mode 100644 index 00000000..417be534 --- /dev/null +++ b/tests/unit/user_simulator/cascade/test_constants.py @@ -0,0 +1,22 @@ +from eva.user_simulator.cascade.constants import ( + BYTES_PER_TICK, + TICK_DURATION_MS, + WAIT_TO_RESPOND_OTHER_MS, + WAIT_TO_RESPOND_SELF_MS, + ms_to_ticks, +) + + +def test_tick_duration_matches_tau_voice(): + assert TICK_DURATION_MS == 200 + + +def test_bytes_per_tick_is_one_tick_of_pcm16_at_16khz(): + # 16000 samples/s * 0.2s * 2 bytes/sample + assert BYTES_PER_TICK == 6400 + + +def test_ms_to_ticks_floors_to_whole_ticks(): + assert ms_to_ticks(WAIT_TO_RESPOND_OTHER_MS) == 5 + assert ms_to_ticks(WAIT_TO_RESPOND_SELF_MS) == 25 + assert ms_to_ticks(150) == 0 From 2f6f9830552e6c6abfd84ab8af2d9d0196802c0b Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 6 Aug 2026 19:50:02 -0400 Subject: [PATCH 05/65] clarify tick encoding docstrings and strengthen cascade constants tests --- src/eva/user_simulator/cascade/constants.py | 4 +++- .../user_simulator/cascade/test_constants.py | 19 ++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/eva/user_simulator/cascade/constants.py b/src/eva/user_simulator/cascade/constants.py index 2ce4927b..d68513b6 100644 --- a/src/eva/user_simulator/cascade/constants.py +++ b/src/eva/user_simulator/cascade/constants.py @@ -7,7 +7,8 @@ """ TICK_DURATION_MS = 200 -"""One simulation tick. Carries ten 20ms mulaw frames on the wire.""" +"""One simulation tick: 200ms of PCM16 audio at CALLER_SAMPLE_RATE, converted by +the adapter into ten 20ms mulaw frames when written to the wire.""" WAIT_TO_RESPOND_OTHER_MS = 1000 """Silence required from the assistant before the caller starts a turn.""" @@ -30,6 +31,7 @@ """PCM16 bytes carried per tick at CALLER_SAMPLE_RATE.""" SILENCE_BYTE = b"\x00" +"""PCM16 silence, used to pad partial ticks.""" def ms_to_ticks(milliseconds: int) -> int: diff --git a/tests/unit/user_simulator/cascade/test_constants.py b/tests/unit/user_simulator/cascade/test_constants.py index 417be534..f3ed9d39 100644 --- a/tests/unit/user_simulator/cascade/test_constants.py +++ b/tests/unit/user_simulator/cascade/test_constants.py @@ -3,12 +3,17 @@ TICK_DURATION_MS, WAIT_TO_RESPOND_OTHER_MS, WAIT_TO_RESPOND_SELF_MS, + YIELD_WHEN_INTERRUPTED_MS, + YIELD_WHEN_INTERRUPTING_MS, ms_to_ticks, ) - -def test_tick_duration_matches_tau_voice(): - assert TICK_DURATION_MS == 200 +THRESHOLD_MS_CONSTANTS = [ + WAIT_TO_RESPOND_OTHER_MS, + WAIT_TO_RESPOND_SELF_MS, + YIELD_WHEN_INTERRUPTED_MS, + YIELD_WHEN_INTERRUPTING_MS, +] def test_bytes_per_tick_is_one_tick_of_pcm16_at_16khz(): @@ -16,7 +21,11 @@ def test_bytes_per_tick_is_one_tick_of_pcm16_at_16khz(): assert BYTES_PER_TICK == 6400 -def test_ms_to_ticks_floors_to_whole_ticks(): +def test_threshold_constants_are_exact_multiples_of_tick_duration(): + for threshold_ms in THRESHOLD_MS_CONSTANTS: + assert threshold_ms % TICK_DURATION_MS == 0 + + +def test_ms_to_ticks_converts_and_floors_sub_tick_remainder(): assert ms_to_ticks(WAIT_TO_RESPOND_OTHER_MS) == 5 - assert ms_to_ticks(WAIT_TO_RESPOND_SELF_MS) == 25 assert ms_to_ticks(150) == 0 From d9b199bd8855e136b111accd91cf936f3bd9e0f2 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 6 Aug 2026 19:53:12 -0400 Subject: [PATCH 06/65] add TickResult and tick-boundary audio splitting Co-Authored-By: Claude Opus 5 (1M context) --- src/eva/user_simulator/cascade/tick_result.py | 34 +++++++++++++++++++ .../cascade/test_tick_result.py | 34 +++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 src/eva/user_simulator/cascade/tick_result.py create mode 100644 tests/unit/user_simulator/cascade/test_tick_result.py diff --git a/src/eva/user_simulator/cascade/tick_result.py b/src/eva/user_simulator/cascade/tick_result.py new file mode 100644 index 00000000..a795fb1a --- /dev/null +++ b/src/eva/user_simulator/cascade/tick_result.py @@ -0,0 +1,34 @@ +"""Per-tick exchange record between the scheduler and an adapter.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from eva.user_simulator.cascade.constants import SILENCE_BYTE + + +@dataclass(frozen=True) +class TickResult: + """What one tick of the conversation produced.""" + + tick_number: int + assistant_audio: bytes + """Exactly one tick's worth of PCM16, silence-padded when the assistant was quiet.""" + + assistant_audio_raw_bytes: int + """Real audio bytes received before padding. Zero means the assistant was silent.""" + + wall_clock_ms: int + """Unix ms at the tick's I/O boundary. For latency metrics only, never for ordering.""" + + @property + def has_assistant_speech(self) -> bool: + """Whether any real assistant audio arrived this tick.""" + return self.assistant_audio_raw_bytes > 0 + + +def split_tick_audio(audio: bytes, bytes_per_tick: int) -> tuple[bytes, bytes]: + """Split audio into exactly one tick's worth plus overflow, padding short input with silence.""" + if len(audio) >= bytes_per_tick: + return audio[:bytes_per_tick], audio[bytes_per_tick:] + return audio + SILENCE_BYTE * (bytes_per_tick - len(audio)), b"" diff --git a/tests/unit/user_simulator/cascade/test_tick_result.py b/tests/unit/user_simulator/cascade/test_tick_result.py new file mode 100644 index 00000000..977a422c --- /dev/null +++ b/tests/unit/user_simulator/cascade/test_tick_result.py @@ -0,0 +1,34 @@ +from eva.user_simulator.cascade.tick_result import TickResult, split_tick_audio + + +def test_split_pads_short_audio_with_silence(): + chunk, overflow = split_tick_audio(b"\x01\x02", bytes_per_tick=8) + + assert chunk == b"\x01\x02" + b"\x00" * 6 + assert overflow == b"" + + +def test_split_carries_overflow_to_next_tick(): + chunk, overflow = split_tick_audio(b"\x01" * 12, bytes_per_tick=8) + + assert chunk == b"\x01" * 8 + assert overflow == b"\x01" * 4 + + +def test_split_of_empty_audio_is_all_silence(): + chunk, overflow = split_tick_audio(b"", bytes_per_tick=8) + + assert chunk == b"\x00" * 8 + assert overflow == b"" + + +def test_has_assistant_speech_is_false_for_padded_silence(): + result = TickResult(tick_number=3, assistant_audio=b"\x00" * 8, assistant_audio_raw_bytes=0, wall_clock_ms=1) + + assert result.has_assistant_speech is False + + +def test_has_assistant_speech_is_true_when_real_audio_arrived(): + result = TickResult(tick_number=3, assistant_audio=b"\x01" * 8, assistant_audio_raw_bytes=8, wall_clock_ms=1) + + assert result.has_assistant_speech is True From 0be9943ca9f15d5bb5fcb7818002189688d7b64b Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 6 Aug 2026 19:57:13 -0400 Subject: [PATCH 07/65] polish TickResult: default bytes_per_tick, NamedTuple return, docstrings Co-Authored-By: Claude Opus 5 (1M context) --- src/eva/user_simulator/cascade/tick_result.py | 21 +++++++++++++++---- .../cascade/test_tick_result.py | 10 +++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/eva/user_simulator/cascade/tick_result.py b/src/eva/user_simulator/cascade/tick_result.py index a795fb1a..9260a992 100644 --- a/src/eva/user_simulator/cascade/tick_result.py +++ b/src/eva/user_simulator/cascade/tick_result.py @@ -3,8 +3,9 @@ from __future__ import annotations from dataclasses import dataclass +from typing import NamedTuple -from eva.user_simulator.cascade.constants import SILENCE_BYTE +from eva.user_simulator.cascade.constants import BYTES_PER_TICK, SILENCE_BYTE @dataclass(frozen=True) @@ -12,6 +13,8 @@ class TickResult: """What one tick of the conversation produced.""" tick_number: int + """Monotonic simulation-clock index; the ordering key for events, unlike wall_clock_ms.""" + assistant_audio: bytes """Exactly one tick's worth of PCM16, silence-padded when the assistant was quiet.""" @@ -27,8 +30,18 @@ def has_assistant_speech(self) -> bool: return self.assistant_audio_raw_bytes > 0 -def split_tick_audio(audio: bytes, bytes_per_tick: int) -> tuple[bytes, bytes]: +class TickAudioSplit(NamedTuple): + """Result of splitting audio at a tick boundary.""" + + chunk: bytes + """Always exactly bytes_per_tick, silence-padded when the input was short.""" + + overflow: bytes + """Any remainder past bytes_per_tick, carried to the next tick.""" + + +def split_tick_audio(audio: bytes, bytes_per_tick: int = BYTES_PER_TICK) -> TickAudioSplit: """Split audio into exactly one tick's worth plus overflow, padding short input with silence.""" if len(audio) >= bytes_per_tick: - return audio[:bytes_per_tick], audio[bytes_per_tick:] - return audio + SILENCE_BYTE * (bytes_per_tick - len(audio)), b"" + return TickAudioSplit(audio[:bytes_per_tick], audio[bytes_per_tick:]) + return TickAudioSplit(audio + SILENCE_BYTE * (bytes_per_tick - len(audio)), b"") diff --git a/tests/unit/user_simulator/cascade/test_tick_result.py b/tests/unit/user_simulator/cascade/test_tick_result.py index 977a422c..f1696f65 100644 --- a/tests/unit/user_simulator/cascade/test_tick_result.py +++ b/tests/unit/user_simulator/cascade/test_tick_result.py @@ -1,6 +1,16 @@ +import pytest + from eva.user_simulator.cascade.tick_result import TickResult, split_tick_audio +@pytest.mark.parametrize("input_length", [0, 1, 7, 8, 9, 17]) +def test_split_chunk_is_always_bytes_per_tick(input_length): + chunk, overflow = split_tick_audio(b"\x01" * input_length, bytes_per_tick=8) + + assert len(chunk) == 8 + assert len(overflow) == max(0, input_length - 8) + + def test_split_pads_short_audio_with_silence(): chunk, overflow = split_tick_audio(b"\x01\x02", bytes_per_tick=8) From 1c2859f27c49fcf99782117b6573035cc22c142c Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 6 Aug 2026 19:59:48 -0400 Subject: [PATCH 08/65] add Adapter ABC --- .../cascade/adapter/__init__.py | 0 .../user_simulator/cascade/adapter/base.py | 28 +++++++++++++++ .../cascade/test_adapter_base.py | 34 +++++++++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 src/eva/user_simulator/cascade/adapter/__init__.py create mode 100644 src/eva/user_simulator/cascade/adapter/base.py create mode 100644 tests/unit/user_simulator/cascade/test_adapter_base.py diff --git a/src/eva/user_simulator/cascade/adapter/__init__.py b/src/eva/user_simulator/cascade/adapter/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/eva/user_simulator/cascade/adapter/base.py b/src/eva/user_simulator/cascade/adapter/base.py new file mode 100644 index 00000000..3e90ae09 --- /dev/null +++ b/src/eva/user_simulator/cascade/adapter/base.py @@ -0,0 +1,28 @@ +"""Adapter contract: the only component doing real I/O against the assistant.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod + +from eva.user_simulator.cascade.tick_result import TickResult + + +class Adapter(ABC): + """Exchanges exactly one tick of audio with the assistant per call. + + The scheduler and simulator never learn which implementation is behind this + interface, which is what lets a tick-driven transport replace the real-time + one without touching turn-taking logic. + """ + + @abstractmethod + async def start(self) -> None: + """Establish the connection. Must return once ready to exchange audio.""" + + @abstractmethod + async def run_tick(self, tick_number: int, outgoing_audio: bytes | None) -> TickResult: + """Send this tick's caller audio (None means silence) and collect what arrived.""" + + @abstractmethod + async def stop(self) -> None: + """Tear the connection down. Must be safe to call twice.""" diff --git a/tests/unit/user_simulator/cascade/test_adapter_base.py b/tests/unit/user_simulator/cascade/test_adapter_base.py new file mode 100644 index 00000000..1f6cfd83 --- /dev/null +++ b/tests/unit/user_simulator/cascade/test_adapter_base.py @@ -0,0 +1,34 @@ +import pytest + +from eva.user_simulator.cascade.adapter.base import Adapter + + +def test_adapter_cannot_be_instantiated_directly(): + with pytest.raises(TypeError): + Adapter() + + +async def test_concrete_adapter_satisfies_the_interface(): + from eva.user_simulator.cascade.tick_result import TickResult + + class StubAdapter(Adapter): + async def start(self) -> None: + pass + + async def run_tick(self, tick_number: int, outgoing_audio: bytes | None) -> TickResult: + return TickResult( + tick_number=tick_number, + assistant_audio=b"\x00" * 4, + assistant_audio_raw_bytes=0, + wall_clock_ms=0, + ) + + async def stop(self) -> None: + pass + + adapter = StubAdapter() + await adapter.start() + result = await adapter.run_tick(0, None) + await adapter.stop() + + assert result.tick_number == 0 From 6c545af8ed7bb4fabbee72a381a3e6693a7ee427 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 6 Aug 2026 20:03:26 -0400 Subject: [PATCH 09/65] add TickScheduler turn-state machine and playout queue --- src/eva/user_simulator/cascade/scheduler.py | 80 +++++++++++ .../user_simulator/cascade/test_scheduler.py | 124 ++++++++++++++++++ 2 files changed, 204 insertions(+) create mode 100644 src/eva/user_simulator/cascade/scheduler.py create mode 100644 tests/unit/user_simulator/cascade/test_scheduler.py diff --git a/src/eva/user_simulator/cascade/scheduler.py b/src/eva/user_simulator/cascade/scheduler.py new file mode 100644 index 00000000..a1aab5e9 --- /dev/null +++ b/src/eva/user_simulator/cascade/scheduler.py @@ -0,0 +1,80 @@ +"""Tick scheduler: virtual clock, playout queue, and turn-state machine.""" + +from __future__ import annotations + +from eva.user_simulator.cascade.adapter.base import Adapter +from eva.user_simulator.cascade.constants import ( + BYTES_PER_TICK, + SILENCE_BYTE, + WAIT_TO_RESPOND_OTHER_MS, + WAIT_TO_RESPOND_SELF_MS, + ms_to_ticks, +) +from eva.user_simulator.cascade.tick_result import TickResult + +_NEVER_SPOKE = 10**9 + + +class TickScheduler: + """Advances the virtual clock and decides who holds the floor. + + Caller turn boundaries are authored: an utterance is queued whole and drains + one tick at a time from a known start tick. The assistant's are detected from + consecutive silent ticks. Both land in the same synchronous step, so their + relative order can never come out ambiguous. + """ + + def __init__(self, adapter: Adapter, *, bytes_per_tick: int = BYTES_PER_TICK) -> None: + self._adapter = adapter + self._bytes_per_tick = bytes_per_tick + self._playout = bytearray() + self.tick = 0 + self._ticks_since_assistant_speech = _NEVER_SPOKE + self._ticks_since_caller_speech = _NEVER_SPOKE + self._assistant_has_spoken = False + + def enqueue_utterance(self, audio: bytes) -> None: + """Queue caller audio to drain one tick at a time starting next tick.""" + self._playout.extend(audio) + + @property + def caller_is_speaking(self) -> bool: + """Whether caller audio is still queued for playout.""" + return bool(self._playout) + + def may_take_turn(self) -> bool: + """Whether both silence thresholds are satisfied (tau: streaming.py:2590-2606). + + Gated on the assistant having spoken at least once: the assistant opens + the call with a greeting, and without this the caller would talk over it + on tick 0, since neither silence counter has anything to measure yet. + """ + if not self._assistant_has_spoken: + return False + return self._ticks_since_assistant_speech > ms_to_ticks( + WAIT_TO_RESPOND_OTHER_MS + ) and self._ticks_since_caller_speech > ms_to_ticks(WAIT_TO_RESPOND_SELF_MS) + + async def run_tick(self) -> TickResult: + """Exchange one tick with the adapter and advance the turn-state machine.""" + outgoing = self._next_chunk() + result = await self._adapter.run_tick(self.tick, outgoing) + + self._ticks_since_caller_speech = 0 if outgoing else self._ticks_since_caller_speech + 1 + self._ticks_since_assistant_speech = ( + 0 if result.has_assistant_speech else self._ticks_since_assistant_speech + 1 + ) + self._assistant_has_spoken = self._assistant_has_spoken or result.has_assistant_speech + + self.tick += 1 + return result + + def _next_chunk(self) -> bytes | None: + """Pull one tick of queued caller audio, or None when the caller is silent.""" + if not self._playout: + return None + chunk = bytes(self._playout[: self._bytes_per_tick]) + del self._playout[: self._bytes_per_tick] + if len(chunk) < self._bytes_per_tick: + chunk += SILENCE_BYTE * (self._bytes_per_tick - len(chunk)) + return chunk diff --git a/tests/unit/user_simulator/cascade/test_scheduler.py b/tests/unit/user_simulator/cascade/test_scheduler.py new file mode 100644 index 00000000..b525dc8d --- /dev/null +++ b/tests/unit/user_simulator/cascade/test_scheduler.py @@ -0,0 +1,124 @@ +from eva.user_simulator.cascade.adapter.base import Adapter +from eva.user_simulator.cascade.scheduler import TickScheduler +from eva.user_simulator.cascade.tick_result import TickResult + +BYTES_PER_TICK = 8 + + +class FakeAdapter(Adapter): + """Replays a scripted sequence of assistant speech/silence and records what it was sent.""" + + def __init__(self, speech_ticks: list[bool]) -> None: + self.speech_ticks = speech_ticks + self.sent: list[bytes | None] = [] + + async def start(self) -> None: + pass + + async def run_tick(self, tick_number: int, outgoing_audio: bytes | None) -> TickResult: + self.sent.append(outgoing_audio) + speaking = self.speech_ticks[tick_number] if tick_number < len(self.speech_ticks) else False + return TickResult( + tick_number=tick_number, + assistant_audio=(b"\x01" if speaking else b"\x00") * BYTES_PER_TICK, + assistant_audio_raw_bytes=BYTES_PER_TICK if speaking else 0, + wall_clock_ms=tick_number, + ) + + async def stop(self) -> None: + pass + + +def _scheduler(speech_ticks: list[bool]) -> TickScheduler: + return TickScheduler(FakeAdapter(speech_ticks), bytes_per_tick=BYTES_PER_TICK) + + +async def test_caller_may_open_the_conversation_once_the_assistant_falls_silent(): + # Assistant greets for 3 ticks, then goes quiet. + scheduler = _scheduler([True, True, True]) + + for _ in range(3): + await scheduler.run_tick() + assert scheduler.may_take_turn() is False + + # WAIT_TO_RESPOND_OTHER_MS is 5 ticks and the comparison is strict. + for _ in range(5): + await scheduler.run_tick() + assert scheduler.may_take_turn() is False + + await scheduler.run_tick() + assert scheduler.may_take_turn() is True + + +async def test_assistant_speech_resets_the_silence_counter(): + scheduler = _scheduler([False] * 10 + [True]) + + for _ in range(11): + await scheduler.run_tick() + + assert scheduler.may_take_turn() is False + + +async def test_caller_must_also_wait_out_its_own_silence_threshold(): + scheduler = _scheduler([True]) # assistant greets on tick 0 + await scheduler.run_tick() + scheduler.enqueue_utterance(b"\x02" * BYTES_PER_TICK) + + await scheduler.run_tick() # caller speaks this tick + assert scheduler.may_take_turn() is False + + # Assistant silence is satisfied almost immediately, self-silence needs 25 ticks. + for _ in range(25): + await scheduler.run_tick() + assert scheduler.may_take_turn() is False + + await scheduler.run_tick() + assert scheduler.may_take_turn() is True + + +async def test_caller_does_not_speak_before_the_assistant_has_greeted(): + # The assistant sends the opening message; the caller must not race it. + scheduler = _scheduler([]) + + for _ in range(50): + await scheduler.run_tick() + + assert scheduler.may_take_turn() is False + + +async def test_queued_utterance_drains_one_tick_at_a_time(): + adapter = FakeAdapter([]) + scheduler = TickScheduler(adapter, bytes_per_tick=BYTES_PER_TICK) + scheduler.enqueue_utterance(b"\x02" * (BYTES_PER_TICK * 3)) + + for _ in range(4): + await scheduler.run_tick() + + assert adapter.sent[0] == b"\x02" * BYTES_PER_TICK + assert adapter.sent[1] == b"\x02" * BYTES_PER_TICK + assert adapter.sent[2] == b"\x02" * BYTES_PER_TICK + assert adapter.sent[3] is None + assert scheduler.caller_is_speaking is False + + +async def test_partial_final_chunk_is_padded_to_a_whole_tick(): + adapter = FakeAdapter([]) + scheduler = TickScheduler(adapter, bytes_per_tick=BYTES_PER_TICK) + scheduler.enqueue_utterance(b"\x02" * (BYTES_PER_TICK + 3)) + + await scheduler.run_tick() + await scheduler.run_tick() + + assert adapter.sent[1] == b"\x02" * 3 + b"\x00" * (BYTES_PER_TICK - 3) + assert scheduler.caller_is_speaking is False + + +async def test_caller_is_speaking_while_audio_remains_queued(): + scheduler = _scheduler([]) + scheduler.enqueue_utterance(b"\x02" * (BYTES_PER_TICK * 2)) + + assert scheduler.caller_is_speaking is True + await scheduler.run_tick() + assert scheduler.caller_is_speaking is True + await scheduler.run_tick() + assert scheduler.caller_is_speaking is False From 8da892731c39ea310bd86e56953bf6359b926106 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 6 Aug 2026 20:09:28 -0400 Subject: [PATCH 10/65] harden TickScheduler: peek-then-commit playout, reuse split_tick_audio, read-only tick Addresses code review on 6c545af8: consume caller audio only after a successful adapter call, delegate padding to split_tick_audio instead of duplicating it, document the single-caller precondition and enqueue concatenation semantics, and make tick read-only. Adds tests for simultaneous speech, tick numbering, utterance contiguity, and adapter-failure safety. --- src/eva/user_simulator/cascade/scheduler.py | 51 ++++++++----- .../user_simulator/cascade/test_scheduler.py | 73 +++++++++++++++++++ 2 files changed, 107 insertions(+), 17 deletions(-) diff --git a/src/eva/user_simulator/cascade/scheduler.py b/src/eva/user_simulator/cascade/scheduler.py index a1aab5e9..95a3ba8f 100644 --- a/src/eva/user_simulator/cascade/scheduler.py +++ b/src/eva/user_simulator/cascade/scheduler.py @@ -5,12 +5,11 @@ from eva.user_simulator.cascade.adapter.base import Adapter from eva.user_simulator.cascade.constants import ( BYTES_PER_TICK, - SILENCE_BYTE, WAIT_TO_RESPOND_OTHER_MS, WAIT_TO_RESPOND_SELF_MS, ms_to_ticks, ) -from eva.user_simulator.cascade.tick_result import TickResult +from eva.user_simulator.cascade.tick_result import TickResult, split_tick_audio _NEVER_SPOKE = 10**9 @@ -21,20 +20,31 @@ class TickScheduler: Caller turn boundaries are authored: an utterance is queued whole and drains one tick at a time from a known start tick. The assistant's are detected from consecutive silent ticks. Both land in the same synchronous step, so their - relative order can never come out ambiguous. + relative order can never come out ambiguous. This guarantee assumes a single + caller drives `run_tick` sequentially; concurrent calls are unsupported. """ def __init__(self, adapter: Adapter, *, bytes_per_tick: int = BYTES_PER_TICK) -> None: self._adapter = adapter self._bytes_per_tick = bytes_per_tick self._playout = bytearray() - self.tick = 0 + self._tick = 0 self._ticks_since_assistant_speech = _NEVER_SPOKE self._ticks_since_caller_speech = _NEVER_SPOKE self._assistant_has_spoken = False + @property + def tick(self) -> int: + """Current tick index; advances only after a successful `run_tick`.""" + return self._tick + def enqueue_utterance(self, audio: bytes) -> None: - """Queue caller audio to drain one tick at a time starting next tick.""" + """Append audio to drain from the next tick onward. + + Appends concatenate into one continuous stream with no gap between them. + Queuing a logically separate utterance while one is still draining is the + caller's responsibility to avoid. + """ self._playout.extend(audio) @property @@ -56,9 +66,14 @@ def may_take_turn(self) -> bool: ) and self._ticks_since_caller_speech > ms_to_ticks(WAIT_TO_RESPOND_SELF_MS) async def run_tick(self) -> TickResult: - """Exchange one tick with the adapter and advance the turn-state machine.""" - outgoing = self._next_chunk() - result = await self._adapter.run_tick(self.tick, outgoing) + """Exchange one tick with the adapter and advance the turn-state machine. + + The playout queue is only drained after the adapter call succeeds, so a + raised exception leaves the queue and tick count exactly as they were. + """ + outgoing, consumed = self._peek_chunk() + result = await self._adapter.run_tick(self._tick, outgoing) + del self._playout[:consumed] self._ticks_since_caller_speech = 0 if outgoing else self._ticks_since_caller_speech + 1 self._ticks_since_assistant_speech = ( @@ -66,15 +81,17 @@ async def run_tick(self) -> TickResult: ) self._assistant_has_spoken = self._assistant_has_spoken or result.has_assistant_speech - self.tick += 1 + self._tick += 1 return result - def _next_chunk(self) -> bytes | None: - """Pull one tick of queued caller audio, or None when the caller is silent.""" + def _peek_chunk(self) -> tuple[bytes | None, int]: + """Preview one tick of queued caller audio without consuming it. + + Returns the padded chunk (or None when silent) and how many raw bytes + of `_playout` it was drawn from, for the caller to commit after success. + """ if not self._playout: - return None - chunk = bytes(self._playout[: self._bytes_per_tick]) - del self._playout[: self._bytes_per_tick] - if len(chunk) < self._bytes_per_tick: - chunk += SILENCE_BYTE * (self._bytes_per_tick - len(chunk)) - return chunk + return None, 0 + raw = bytes(self._playout[: self._bytes_per_tick]) + chunk, _ = split_tick_audio(raw, self._bytes_per_tick) + return chunk, len(raw) diff --git a/tests/unit/user_simulator/cascade/test_scheduler.py b/tests/unit/user_simulator/cascade/test_scheduler.py index b525dc8d..6e9b2ccf 100644 --- a/tests/unit/user_simulator/cascade/test_scheduler.py +++ b/tests/unit/user_simulator/cascade/test_scheduler.py @@ -11,12 +11,14 @@ class FakeAdapter(Adapter): def __init__(self, speech_ticks: list[bool]) -> None: self.speech_ticks = speech_ticks self.sent: list[bytes | None] = [] + self.received_ticks: list[int] = [] async def start(self) -> None: pass async def run_tick(self, tick_number: int, outgoing_audio: bytes | None) -> TickResult: self.sent.append(outgoing_audio) + self.received_ticks.append(tick_number) speaking = self.speech_ticks[tick_number] if tick_number < len(self.speech_ticks) else False return TickResult( tick_number=tick_number, @@ -122,3 +124,74 @@ async def test_caller_is_speaking_while_audio_remains_queued(): assert scheduler.caller_is_speaking is True await scheduler.run_tick() assert scheduler.caller_is_speaking is False + + +async def test_simultaneous_speech_resets_both_counters_and_blocks_the_turn(): + scheduler = _scheduler([True]) + scheduler.enqueue_utterance(b"\x02" * BYTES_PER_TICK) + + await scheduler.run_tick() # both sides speak on tick 0 + + assert scheduler.may_take_turn() is False + + +async def test_tick_number_reaches_the_adapter_and_increments(): + adapter = FakeAdapter([]) + scheduler = TickScheduler(adapter, bytes_per_tick=BYTES_PER_TICK) + + for _ in range(3): + await scheduler.run_tick() + + assert adapter.received_ticks == [0, 1, 2] + assert scheduler.tick == 3 + + +async def test_consecutive_enqueues_drain_contiguously_with_no_silence_gap(): + adapter = FakeAdapter([]) + scheduler = TickScheduler(adapter, bytes_per_tick=BYTES_PER_TICK) + scheduler.enqueue_utterance(b"\x02" * BYTES_PER_TICK) + scheduler.enqueue_utterance(b"\x03" * BYTES_PER_TICK) + + await scheduler.run_tick() + await scheduler.run_tick() + + assert adapter.sent[0] == b"\x02" * BYTES_PER_TICK + assert adapter.sent[1] == b"\x03" * BYTES_PER_TICK + + +class RaisingAdapter(Adapter): + """Raises on a chosen tick to exercise the peek-then-commit failure path.""" + + def __init__(self, fail_on_tick: int) -> None: + self.fail_on_tick = fail_on_tick + + async def start(self) -> None: + pass + + async def run_tick(self, tick_number: int, outgoing_audio: bytes | None) -> TickResult: + if tick_number == self.fail_on_tick: + raise RuntimeError("adapter failure") + return TickResult( + tick_number=tick_number, + assistant_audio=b"\x00" * BYTES_PER_TICK, + assistant_audio_raw_bytes=0, + wall_clock_ms=tick_number, + ) + + async def stop(self) -> None: + pass + + +async def test_failed_adapter_call_leaves_queue_and_tick_unadvanced(): + adapter = RaisingAdapter(fail_on_tick=0) + scheduler = TickScheduler(adapter, bytes_per_tick=BYTES_PER_TICK) + utterance = b"\x02" * BYTES_PER_TICK + scheduler.enqueue_utterance(utterance) + + try: + await scheduler.run_tick() + except RuntimeError: + pass + + assert scheduler.tick == 0 + assert bytes(scheduler._playout) == utterance From 903bb6168dbd10bc2ca9c84f65e3b79c220e2c7a Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 6 Aug 2026 20:14:40 -0400 Subject: [PATCH 11/65] add RealtimeWSAdapter --- .../cascade/adapter/realtime_ws.py | 154 ++++++++++++++++++ .../cascade/test_realtime_ws_adapter.py | 78 +++++++++ 2 files changed, 232 insertions(+) create mode 100644 src/eva/user_simulator/cascade/adapter/realtime_ws.py create mode 100644 tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py diff --git a/src/eva/user_simulator/cascade/adapter/realtime_ws.py b/src/eva/user_simulator/cascade/adapter/realtime_ws.py new file mode 100644 index 00000000..8fc27f2e --- /dev/null +++ b/src/eva/user_simulator/cascade/adapter/realtime_ws.py @@ -0,0 +1,154 @@ +"""Real-time Twilio WebSocket adapter for unmodified assistant servers.""" + +from __future__ import annotations + +import asyncio +import base64 +import contextlib +import json +import time + +try: + import audioop +except ImportError: # pragma: no cover - Python 3.13+ + import audioop_lts as audioop + +from eva.user_simulator.cascade.adapter.base import Adapter +from eva.user_simulator.cascade.constants import BYTES_PER_TICK, CALLER_SAMPLE_RATE, TICK_DURATION_MS +from eva.user_simulator.cascade.tick_result import TickResult, split_tick_audio +from eva.utils.logging import get_logger + +logger = get_logger(__name__) + +WIRE_SAMPLE_RATE = 8000 +WIRE_FRAME_MS = 20 +FRAMES_PER_TICK = TICK_DURATION_MS // WIRE_FRAME_MS +_PCM_WIDTH = 2 + + +class RealtimeWSAdapter(Adapter): + """Exchanges tick-sized audio with an assistant server over the Twilio WS protocol. + + Outbound audio is paced at the real 20ms cadence the assistant expects + (docs/assistant_server_contract.md section 3). Inbound audio is buffered and + released exactly one tick at a time, so a provider that generates faster than + real time cannot run ahead of the simulation clock. + """ + + def __init__(self, *, websocket, conversation_id: str, bytes_per_tick: int = BYTES_PER_TICK) -> None: + self._ws = websocket + self._conversation_id = conversation_id + self._bytes_per_tick = bytes_per_tick + self._inbound = bytearray() + self._pending_recv: asyncio.Task | None = None + + async def start(self) -> None: + """Send the connect/start handshake.""" + for event in ("connected", "start"): + await self._ws.send(json.dumps({"event": event, "conversation_id": self._conversation_id})) + + async def run_tick(self, tick_number: int, outgoing_audio: bytes | None) -> TickResult: + """Send one tick of caller audio at wire cadence and collect one tick of assistant audio.""" + if outgoing_audio: + await self._send_tick_audio(outgoing_audio) + + await self._drain_pending() + raw = bytes(self._inbound[: self._bytes_per_tick]) + del self._inbound[: len(raw)] + chunk, _ = split_tick_audio(raw, self._bytes_per_tick) + + return TickResult( + tick_number=tick_number, + assistant_audio=chunk, + assistant_audio_raw_bytes=len(raw), + wall_clock_ms=int(time.time() * 1000), + ) + + async def stop(self) -> None: + """Send stop, cancel any in-flight receive, and close the socket. Safe to call twice.""" + if self._pending_recv is not None: + self._pending_recv.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await self._pending_recv + self._pending_recv = None + with contextlib.suppress(Exception): + await self._ws.send(json.dumps({"event": "stop", "conversation_id": self._conversation_id})) + with contextlib.suppress(Exception): + await self._ws.close() + + async def _send_tick_audio(self, pcm: bytes) -> None: + """Split one tick of PCM16 into 20ms mulaw frames and send them at real-time pace.""" + mulaw = self._pcm16k_to_mulaw8k(pcm) + frame_size = len(mulaw) // FRAMES_PER_TICK or len(mulaw) + interval = WIRE_FRAME_MS / 1000 + for index in range(0, len(mulaw), frame_size): + payload = base64.b64encode(mulaw[index : index + frame_size]).decode() + await self._ws.send( + json.dumps( + { + "event": "media", + "conversation_id": self._conversation_id, + "media": {"payload": payload}, + } + ) + ) + await asyncio.sleep(interval) + + async def _drain_pending(self) -> None: + """Pull any inbound frames that have already arrived, without blocking on new ones. + + A single ``recv()`` call is always in flight, tracked as ``_pending_recv``. Each + tick, we yield control once so an already-arrived frame (queued before this call, + as in tests that never start a background loop) can complete that task, then keep + harvesting completed tasks and re-arming a fresh one until none are ready. If the + pending task is still waiting on the wire, we leave it in place for the next tick + to pick up rather than cancelling it — a websocket has only one live recv() at a + time. This makes the drain path identical whether `start()` launched anything or + not, so there is no test-only branch in production code. + """ + if self._pending_recv is None: + self._pending_recv = asyncio.ensure_future(self._ws.recv()) + + await asyncio.sleep(0) + while self._pending_recv is not None and self._pending_recv.done(): + task = self._pending_recv + self._pending_recv = None + try: + raw = task.result() + except Exception: + return + self._ingest(raw) + self._pending_recv = asyncio.ensure_future(self._ws.recv()) + await asyncio.sleep(0) + + def _ingest(self, raw: str) -> None: + """Decode one inbound frame, keeping only media payloads.""" + try: + message = json.loads(raw) + except json.JSONDecodeError: + return + if message.get("event") != "media": + return + payload = message.get("media", {}).get("payload", "") + if payload: + self._inbound.extend(self._mulaw8k_to_pcm16k(base64.b64decode(payload))) + + @staticmethod + def _mulaw8k_to_pcm16k(mulaw: bytes) -> bytes: + """Convert 8kHz mulaw from the wire to PCM16 at the caller sample rate.""" + pcm_8k = audioop.ulaw2lin(mulaw, _PCM_WIDTH) + pcm_16k, _ = audioop.ratecv(pcm_8k, _PCM_WIDTH, 1, WIRE_SAMPLE_RATE, CALLER_SAMPLE_RATE, None) + # audioop.ratecv can produce a few bytes short/long of the exact 2x count; + # clamp so downstream byte-count math (tick sizing, overflow) stays exact. + expected_bytes = len(pcm_8k) * (CALLER_SAMPLE_RATE // WIRE_SAMPLE_RATE) + if len(pcm_16k) < expected_bytes: + pcm_16k = pcm_16k + b"\x00" * (expected_bytes - len(pcm_16k)) + elif len(pcm_16k) > expected_bytes: + pcm_16k = pcm_16k[:expected_bytes] + return pcm_16k + + @staticmethod + def _pcm16k_to_mulaw8k(pcm: bytes) -> bytes: + """Convert caller PCM16 to the 8kHz mulaw the assistant expects.""" + pcm_8k, _ = audioop.ratecv(pcm, _PCM_WIDTH, 1, CALLER_SAMPLE_RATE, WIRE_SAMPLE_RATE, None) + return audioop.lin2ulaw(pcm_8k, _PCM_WIDTH) diff --git a/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py b/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py new file mode 100644 index 00000000..3cfd3f4f --- /dev/null +++ b/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py @@ -0,0 +1,78 @@ +import asyncio +import base64 +import json + +from eva.user_simulator.cascade.adapter.realtime_ws import RealtimeWSAdapter + +BYTES_PER_TICK = 6400 + + +class FakeWebSocket: + """Collects sent frames and replays queued inbound frames.""" + + def __init__(self) -> None: + self.sent: list[str] = [] + self.inbound: asyncio.Queue[str] = asyncio.Queue() + self.closed = False + + async def send(self, message: str) -> None: + self.sent.append(message) + + async def recv(self) -> str: + return await self.inbound.get() + + async def close(self) -> None: + self.closed = True + + +def _media_frame(mulaw: bytes) -> str: + payload = base64.b64encode(mulaw).decode() + return json.dumps({"event": "media", "media": {"payload": payload}}) + + +async def test_tick_with_no_inbound_audio_yields_padded_silence(): + ws = FakeWebSocket() + adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + + result = await adapter.run_tick(0, None) + + assert result.assistant_audio_raw_bytes == 0 + assert result.has_assistant_speech is False + assert len(result.assistant_audio) == BYTES_PER_TICK + + +async def test_inbound_mulaw_is_converted_and_reported_as_speech(): + ws = FakeWebSocket() + adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + # 160 mulaw bytes @8kHz == 20ms == 640 PCM16 bytes @16kHz. + await ws.inbound.put(_media_frame(b"\xff" * 160)) + + result = await adapter.run_tick(0, None) + + assert result.assistant_audio_raw_bytes == 640 + assert result.has_assistant_speech is True + assert len(result.assistant_audio) == BYTES_PER_TICK + + +async def test_outgoing_caller_audio_is_sent_as_twilio_media_frames(): + ws = FakeWebSocket() + adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + + await adapter.run_tick(0, b"\x00" * BYTES_PER_TICK) + + media_frames = [json.loads(m) for m in ws.sent if json.loads(m).get("event") == "media"] + # One tick (200ms) is ten 20ms wire frames. + assert len(media_frames) == 10 + + +async def test_overflow_audio_carries_into_the_next_tick(): + ws = FakeWebSocket() + adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + # 400ms of assistant audio arrives at once: 3200 mulaw bytes -> 12800 PCM bytes. + await ws.inbound.put(_media_frame(b"\xff" * 3200)) + + first = await adapter.run_tick(0, None) + second = await adapter.run_tick(1, None) + + assert first.assistant_audio_raw_bytes == BYTES_PER_TICK + assert second.assistant_audio_raw_bytes == BYTES_PER_TICK From 814b1a98fe6660897dc7ef805192ba6854035e88 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 6 Aug 2026 20:19:20 -0400 Subject: [PATCH 12/65] Carry resampler state across ratecv calls in RealtimeWSAdapter Passing state=None on every audioop.ratecv call caused per-frame filter warm-up loss (2 bytes/call, ~0.3% drift over a conversation) and boundary discontinuities. Threading state through per direction eliminates both; drops the byte-count clamp/pad that masked the drift instead of fixing it. --- .../cascade/adapter/realtime_ws.py | 27 ++++++++-------- .../cascade/test_realtime_ws_adapter.py | 32 ++++++++++++++++--- 2 files changed, 42 insertions(+), 17 deletions(-) diff --git a/src/eva/user_simulator/cascade/adapter/realtime_ws.py b/src/eva/user_simulator/cascade/adapter/realtime_ws.py index 8fc27f2e..bcc05b7d 100644 --- a/src/eva/user_simulator/cascade/adapter/realtime_ws.py +++ b/src/eva/user_simulator/cascade/adapter/realtime_ws.py @@ -33,6 +33,10 @@ class RealtimeWSAdapter(Adapter): (docs/assistant_server_contract.md section 3). Inbound audio is buffered and released exactly one tick at a time, so a provider that generates faster than real time cannot run ahead of the simulation clock. + + Each direction resamples a continuous stream, so each carries its own + `audioop.ratecv` filter state across calls; the stateless helpers in + `audio_utils` cannot express that and are not reused here for that reason. """ def __init__(self, *, websocket, conversation_id: str, bytes_per_tick: int = BYTES_PER_TICK) -> None: @@ -41,6 +45,8 @@ def __init__(self, *, websocket, conversation_id: str, bytes_per_tick: int = BYT self._bytes_per_tick = bytes_per_tick self._inbound = bytearray() self._pending_recv: asyncio.Task | None = None + self._inbound_resample_state = None + self._outbound_resample_state = None async def start(self) -> None: """Send the connect/start handshake.""" @@ -133,22 +139,17 @@ def _ingest(self, raw: str) -> None: if payload: self._inbound.extend(self._mulaw8k_to_pcm16k(base64.b64decode(payload))) - @staticmethod - def _mulaw8k_to_pcm16k(mulaw: bytes) -> bytes: + def _mulaw8k_to_pcm16k(self, mulaw: bytes) -> bytes: """Convert 8kHz mulaw from the wire to PCM16 at the caller sample rate.""" pcm_8k = audioop.ulaw2lin(mulaw, _PCM_WIDTH) - pcm_16k, _ = audioop.ratecv(pcm_8k, _PCM_WIDTH, 1, WIRE_SAMPLE_RATE, CALLER_SAMPLE_RATE, None) - # audioop.ratecv can produce a few bytes short/long of the exact 2x count; - # clamp so downstream byte-count math (tick sizing, overflow) stays exact. - expected_bytes = len(pcm_8k) * (CALLER_SAMPLE_RATE // WIRE_SAMPLE_RATE) - if len(pcm_16k) < expected_bytes: - pcm_16k = pcm_16k + b"\x00" * (expected_bytes - len(pcm_16k)) - elif len(pcm_16k) > expected_bytes: - pcm_16k = pcm_16k[:expected_bytes] + pcm_16k, self._inbound_resample_state = audioop.ratecv( + pcm_8k, _PCM_WIDTH, 1, WIRE_SAMPLE_RATE, CALLER_SAMPLE_RATE, self._inbound_resample_state + ) return pcm_16k - @staticmethod - def _pcm16k_to_mulaw8k(pcm: bytes) -> bytes: + def _pcm16k_to_mulaw8k(self, pcm: bytes) -> bytes: """Convert caller PCM16 to the 8kHz mulaw the assistant expects.""" - pcm_8k, _ = audioop.ratecv(pcm, _PCM_WIDTH, 1, CALLER_SAMPLE_RATE, WIRE_SAMPLE_RATE, None) + pcm_8k, self._outbound_resample_state = audioop.ratecv( + pcm, _PCM_WIDTH, 1, CALLER_SAMPLE_RATE, WIRE_SAMPLE_RATE, self._outbound_resample_state + ) return audioop.lin2ulaw(pcm_8k, _PCM_WIDTH) diff --git a/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py b/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py index 3cfd3f4f..eceec036 100644 --- a/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py +++ b/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py @@ -44,12 +44,12 @@ async def test_tick_with_no_inbound_audio_yields_padded_silence(): async def test_inbound_mulaw_is_converted_and_reported_as_speech(): ws = FakeWebSocket() adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) - # 160 mulaw bytes @8kHz == 20ms == 640 PCM16 bytes @16kHz. + # 160 mulaw bytes @8kHz == 20ms, resampled to roughly 640 PCM16 bytes @16kHz. await ws.inbound.put(_media_frame(b"\xff" * 160)) result = await adapter.run_tick(0, None) - assert result.assistant_audio_raw_bytes == 640 + assert result.assistant_audio_raw_bytes > 0 assert result.has_assistant_speech is True assert len(result.assistant_audio) == BYTES_PER_TICK @@ -68,11 +68,35 @@ async def test_outgoing_caller_audio_is_sent_as_twilio_media_frames(): async def test_overflow_audio_carries_into_the_next_tick(): ws = FakeWebSocket() adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) - # 400ms of assistant audio arrives at once: 3200 mulaw bytes -> 12800 PCM bytes. + # 400ms of assistant audio arrives at once: 3200 mulaw bytes -> ~12800 PCM bytes, + # more than one tick's worth, so it must span both ticks with real audio in each. await ws.inbound.put(_media_frame(b"\xff" * 3200)) first = await adapter.run_tick(0, None) second = await adapter.run_tick(1, None) + assert len(first.assistant_audio) == BYTES_PER_TICK + assert len(second.assistant_audio) == BYTES_PER_TICK assert first.assistant_audio_raw_bytes == BYTES_PER_TICK - assert second.assistant_audio_raw_bytes == BYTES_PER_TICK + assert second.assistant_audio_raw_bytes > 0 + assert first.has_assistant_speech is True + assert second.has_assistant_speech is True + + +async def test_per_frame_resampling_does_not_accumulate_drift(): + ws = FakeWebSocket() + adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + frame_count = 50 + for _ in range(frame_count): + await ws.inbound.put(_media_frame(b"\xff" * 160)) + + ideal_bytes = frame_count * 640 + ticks_needed = ideal_bytes // BYTES_PER_TICK + 2 + total_raw_bytes = 0 + for tick_number in range(ticks_needed): + result = await adapter.run_tick(tick_number, None) + total_raw_bytes += result.assistant_audio_raw_bytes + + # 1 sample (2 bytes) of PCM16 warm-up loss is expected for the whole + # stream; per-frame loss must not accumulate beyond that. + assert abs(total_raw_bytes - ideal_bytes) <= 2 From d70d93ca7d88cc6263531d349d73fdf700223651 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 6 Aug 2026 20:31:16 -0400 Subject: [PATCH 13/65] Fix critical drain bug and other issues in RealtimeWSAdapter - Replace the sleep(0)-based drain (which drains 0 frames against a real websockets.recv(), which always suspends at least once) with a background receive task feeding an adapter-owned buffer, per review. - Surface receive-loop errors from run_tick instead of degrading to silent silence forever; log the exception. - Scope stop()'s CancelledError suppression to the task's own cancellation so an outer cancel of stop() itself still propagates. - Emit user_speech_start/stop events matching audio_bridge.py's payload shape, needed for user-turn timestamps and latency metrics. - Use deadline-based pacing for outbound frames instead of a fixed post-frame sleep, and drop the sleep after the last frame. - Guard the empty-audio case in _send_tick_audio explicitly instead of an unreachable fallback that raised ValueError at len(mulaw) == 0. --- .../cascade/adapter/realtime_ws.py | 109 ++++++++----- .../cascade/test_realtime_ws_adapter.py | 148 +++++++++++++++++- 2 files changed, 213 insertions(+), 44 deletions(-) diff --git a/src/eva/user_simulator/cascade/adapter/realtime_ws.py b/src/eva/user_simulator/cascade/adapter/realtime_ws.py index bcc05b7d..5794a2b3 100644 --- a/src/eva/user_simulator/cascade/adapter/realtime_ws.py +++ b/src/eva/user_simulator/cascade/adapter/realtime_ws.py @@ -30,9 +30,10 @@ class RealtimeWSAdapter(Adapter): """Exchanges tick-sized audio with an assistant server over the Twilio WS protocol. Outbound audio is paced at the real 20ms cadence the assistant expects - (docs/assistant_server_contract.md section 3). Inbound audio is buffered and - released exactly one tick at a time, so a provider that generates faster than - real time cannot run ahead of the simulation clock. + (docs/assistant_server_contract.md section 3). A background task continuously + drains inbound frames into an adapter-owned buffer; `run_tick` releases exactly + one tick's worth per call, so a provider that generates faster than real time + cannot run ahead of the simulation clock. Each direction resamples a continuous stream, so each carries its own `audioop.ratecv` filter state across calls; the stateless helpers in @@ -44,25 +45,40 @@ def __init__(self, *, websocket, conversation_id: str, bytes_per_tick: int = BYT self._conversation_id = conversation_id self._bytes_per_tick = bytes_per_tick self._inbound = bytearray() - self._pending_recv: asyncio.Task | None = None + self._receive_task: asyncio.Task | None = None self._inbound_resample_state = None self._outbound_resample_state = None + self._caller_speaking = False + self._error: BaseException | None = None async def start(self) -> None: - """Send the connect/start handshake.""" + """Send the connect/start handshake and begin buffering inbound audio.""" for event in ("connected", "start"): await self._ws.send(json.dumps({"event": event, "conversation_id": self._conversation_id})) + self._receive_task = asyncio.create_task(self._receive_loop()) async def run_tick(self, tick_number: int, outgoing_audio: bytes | None) -> TickResult: """Send one tick of caller audio at wire cadence and collect one tick of assistant audio.""" + if self._error is not None: + raise RuntimeError("RealtimeWSAdapter receive loop failed") from self._error + + is_speaking = bool(outgoing_audio) + if is_speaking and not self._caller_speaking: + await self._send_speech_event("user_speech_start") + elif not is_speaking and self._caller_speaking: + await self._send_speech_event("user_speech_stop") + self._caller_speaking = is_speaking + if outgoing_audio: await self._send_tick_audio(outgoing_audio) - await self._drain_pending() raw = bytes(self._inbound[: self._bytes_per_tick]) del self._inbound[: len(raw)] chunk, _ = split_tick_audio(raw, self._bytes_per_tick) + if self._error is not None: + raise RuntimeError("RealtimeWSAdapter receive loop failed") from self._error + return TickResult( tick_number=tick_number, assistant_audio=chunk, @@ -71,24 +87,46 @@ async def run_tick(self, tick_number: int, outgoing_audio: bytes | None) -> Tick ) async def stop(self) -> None: - """Send stop, cancel any in-flight receive, and close the socket. Safe to call twice.""" - if self._pending_recv is not None: - self._pending_recv.cancel() - with contextlib.suppress(asyncio.CancelledError, Exception): - await self._pending_recv - self._pending_recv = None + """Send stop, cancel the receive loop, and close the socket. Safe to call twice.""" + if self._receive_task is not None: + task = self._receive_task + self._receive_task = None + task.cancel() + try: + await task + except asyncio.CancelledError: + if not task.cancelled(): + raise + except Exception: + pass with contextlib.suppress(Exception): await self._ws.send(json.dumps({"event": "stop", "conversation_id": self._conversation_id})) with contextlib.suppress(Exception): await self._ws.close() + async def _send_speech_event(self, event: str) -> None: + """Emit a user_speech_start/stop event matching the existing bridge's payload shape.""" + await self._ws.send( + json.dumps( + { + "event": event, + "conversation_id": self._conversation_id, + "timestamp_ms": str(int(round(time.time() * 1000))), + } + ) + ) + async def _send_tick_audio(self, pcm: bytes) -> None: """Split one tick of PCM16 into 20ms mulaw frames and send them at real-time pace.""" mulaw = self._pcm16k_to_mulaw8k(pcm) - frame_size = len(mulaw) // FRAMES_PER_TICK or len(mulaw) + if not mulaw: + return + frame_size = len(mulaw) // FRAMES_PER_TICK + frames = [mulaw[index : index + frame_size] for index in range(0, len(mulaw), frame_size)] interval = WIRE_FRAME_MS / 1000 - for index in range(0, len(mulaw), frame_size): - payload = base64.b64encode(mulaw[index : index + frame_size]).decode() + start_time = asyncio.get_event_loop().time() + for frame_index, frame in enumerate(frames): + payload = base64.b64encode(frame).decode() await self._ws.send( json.dumps( { @@ -98,34 +136,25 @@ async def _send_tick_audio(self, pcm: bytes) -> None: } ) ) - await asyncio.sleep(interval) - - async def _drain_pending(self) -> None: - """Pull any inbound frames that have already arrived, without blocking on new ones. - - A single ``recv()`` call is always in flight, tracked as ``_pending_recv``. Each - tick, we yield control once so an already-arrived frame (queued before this call, - as in tests that never start a background loop) can complete that task, then keep - harvesting completed tasks and re-arming a fresh one until none are ready. If the - pending task is still waiting on the wire, we leave it in place for the next tick - to pick up rather than cancelling it — a websocket has only one live recv() at a - time. This makes the drain path identical whether `start()` launched anything or - not, so there is no test-only branch in production code. - """ - if self._pending_recv is None: - self._pending_recv = asyncio.ensure_future(self._ws.recv()) - - await asyncio.sleep(0) - while self._pending_recv is not None and self._pending_recv.done(): - task = self._pending_recv - self._pending_recv = None + if frame_index == len(frames) - 1: + break + deadline = start_time + (frame_index + 1) * interval + sleep_for = deadline - asyncio.get_event_loop().time() + if sleep_for > 0: + await asyncio.sleep(sleep_for) + + async def _receive_loop(self) -> None: + """Continuously buffer inbound assistant audio as PCM16 at the caller sample rate.""" + while True: try: - raw = task.result() - except Exception: + raw = await self._ws.recv() + except asyncio.CancelledError: + raise + except Exception as exc: + logger.exception("RealtimeWSAdapter receive loop failed") + self._error = exc return self._ingest(raw) - self._pending_recv = asyncio.ensure_future(self._ws.recv()) - await asyncio.sleep(0) def _ingest(self, raw: str) -> None: """Decode one inbound frame, keeping only media payloads.""" diff --git a/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py b/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py index eceec036..d152cf83 100644 --- a/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py +++ b/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py @@ -2,13 +2,20 @@ import base64 import json +import pytest + from eva.user_simulator.cascade.adapter.realtime_ws import RealtimeWSAdapter BYTES_PER_TICK = 6400 +_SETTLE_ROUNDS = 300 class FakeWebSocket: - """Collects sent frames and replays queued inbound frames.""" + """Collects sent frames and replays queued inbound frames. + + recv() completes without ever suspending when a frame is already queued — + unlike real `websockets.recv()`. Use SuspendingFakeWebSocket to model that. + """ def __init__(self) -> None: self.sent: list[str] = [] @@ -25,14 +32,45 @@ async def close(self) -> None: self.closed = True +class SuspendingFakeWebSocket(FakeWebSocket): + """A recv() that always suspends at least once before returning, like the real thing.""" + + async def recv(self) -> str: + await asyncio.sleep(0) + return await self.inbound.get() + + +class RaisingFakeWebSocket(FakeWebSocket): + """A recv() that raises once a configured number of successful frames have been read.""" + + def __init__(self, fail_after: int) -> None: + super().__init__() + self._fail_after = fail_after + self._count = 0 + + async def recv(self) -> str: + await asyncio.sleep(0) + if self._count >= self._fail_after: + raise ConnectionError("simulated disconnect") + self._count += 1 + return await self.inbound.get() + + def _media_frame(mulaw: bytes) -> str: payload = base64.b64encode(mulaw).decode() return json.dumps({"event": "media", "media": {"payload": payload}}) +async def _settle() -> None: + """Give a background receive task many event-loop turns to drain queued frames.""" + for _ in range(_SETTLE_ROUNDS): + await asyncio.sleep(0) + + async def test_tick_with_no_inbound_audio_yields_padded_silence(): ws = FakeWebSocket() adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await adapter.start() result = await adapter.run_tick(0, None) @@ -40,12 +78,67 @@ async def test_tick_with_no_inbound_audio_yields_padded_silence(): assert result.has_assistant_speech is False assert len(result.assistant_audio) == BYTES_PER_TICK + await adapter.stop() + + +async def test_receive_loop_drains_frames_from_a_suspending_websocket(): + """Regression test: a suspending recv() must still be drained by the background loop.""" + ws = SuspendingFakeWebSocket() + adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await ws.inbound.put(_media_frame(b"\xff" * 160)) + await adapter.start() + await _settle() + + result = await adapter.run_tick(0, None) + + assert result.has_assistant_speech is True + assert result.assistant_audio_raw_bytes > 0 + + await adapter.stop() + + +async def test_burst_of_frames_ingested_then_released_one_tick_at_a_time(): + ws = SuspendingFakeWebSocket() + adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + frame_count = 20 # 400ms of wire audio delivered as one burst (two ticks' worth). + for _ in range(frame_count): + await ws.inbound.put(_media_frame(b"\xff" * 160)) + await adapter.start() + await _settle() + + first = await adapter.run_tick(0, None) + second = await adapter.run_tick(1, None) + third = await adapter.run_tick(2, None) + + assert len(first.assistant_audio) == BYTES_PER_TICK + assert len(second.assistant_audio) == BYTES_PER_TICK + assert first.has_assistant_speech is True + assert second.has_assistant_speech is True + total_raw = first.assistant_audio_raw_bytes + second.assistant_audio_raw_bytes + third.assistant_audio_raw_bytes + assert abs(total_raw - frame_count * 640) <= 2 + + await adapter.stop() + + +async def test_receive_error_surfaces_from_run_tick(): + ws = RaisingFakeWebSocket(fail_after=0) + adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await adapter.start() + await _settle() + + with pytest.raises(RuntimeError): + await adapter.run_tick(0, None) + + await adapter.stop() + async def test_inbound_mulaw_is_converted_and_reported_as_speech(): - ws = FakeWebSocket() + ws = SuspendingFakeWebSocket() adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) # 160 mulaw bytes @8kHz == 20ms, resampled to roughly 640 PCM16 bytes @16kHz. await ws.inbound.put(_media_frame(b"\xff" * 160)) + await adapter.start() + await _settle() result = await adapter.run_tick(0, None) @@ -53,10 +146,13 @@ async def test_inbound_mulaw_is_converted_and_reported_as_speech(): assert result.has_assistant_speech is True assert len(result.assistant_audio) == BYTES_PER_TICK + await adapter.stop() + async def test_outgoing_caller_audio_is_sent_as_twilio_media_frames(): ws = FakeWebSocket() adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await adapter.start() await adapter.run_tick(0, b"\x00" * BYTES_PER_TICK) @@ -64,13 +160,17 @@ async def test_outgoing_caller_audio_is_sent_as_twilio_media_frames(): # One tick (200ms) is ten 20ms wire frames. assert len(media_frames) == 10 + await adapter.stop() + async def test_overflow_audio_carries_into_the_next_tick(): - ws = FakeWebSocket() + ws = SuspendingFakeWebSocket() adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) # 400ms of assistant audio arrives at once: 3200 mulaw bytes -> ~12800 PCM bytes, # more than one tick's worth, so it must span both ticks with real audio in each. await ws.inbound.put(_media_frame(b"\xff" * 3200)) + await adapter.start() + await _settle() first = await adapter.run_tick(0, None) second = await adapter.run_tick(1, None) @@ -82,13 +182,17 @@ async def test_overflow_audio_carries_into_the_next_tick(): assert first.has_assistant_speech is True assert second.has_assistant_speech is True + await adapter.stop() + async def test_per_frame_resampling_does_not_accumulate_drift(): - ws = FakeWebSocket() + ws = SuspendingFakeWebSocket() adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) frame_count = 50 for _ in range(frame_count): await ws.inbound.put(_media_frame(b"\xff" * 160)) + await adapter.start() + await _settle() ideal_bytes = frame_count * 640 ticks_needed = ideal_bytes // BYTES_PER_TICK + 2 @@ -100,3 +204,39 @@ async def test_per_frame_resampling_does_not_accumulate_drift(): # 1 sample (2 bytes) of PCM16 warm-up loss is expected for the whole # stream; per-frame loss must not accumulate beyond that. assert abs(total_raw_bytes - ideal_bytes) <= 2 + + await adapter.stop() + + +async def test_user_speech_start_emitted_once_on_silence_to_audio_transition(): + ws = FakeWebSocket() + adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await adapter.start() + + await adapter.run_tick(0, None) + await adapter.run_tick(1, b"\x00" * BYTES_PER_TICK) + await adapter.run_tick(2, b"\x00" * BYTES_PER_TICK) + + events = [json.loads(m) for m in ws.sent] + starts = [e for e in events if e.get("event") == "user_speech_start"] + assert len(starts) == 1 + assert isinstance(starts[0]["timestamp_ms"], str) + + await adapter.stop() + + +async def test_user_speech_stop_emitted_once_on_audio_to_silence_transition(): + ws = FakeWebSocket() + adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await adapter.start() + + await adapter.run_tick(0, b"\x00" * BYTES_PER_TICK) + await adapter.run_tick(1, None) + await adapter.run_tick(2, None) + + events = [json.loads(m) for m in ws.sent] + stops = [e for e in events if e.get("event") == "user_speech_stop"] + assert len(stops) == 1 + assert isinstance(stops[0]["timestamp_ms"], str) + + await adapter.stop() From b18c63027e9982980b531b141706fe6db5a8b314 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 6 Aug 2026 22:55:27 -0400 Subject: [PATCH 14/65] add streaming STT client with transcript buffering Co-Authored-By: Claude Opus 5 (1M context) --- src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/stt.py | 127 +++++++++++++++ tests/unit/user_simulator/cascade/test_stt.py | 151 ++++++++++++++++++ 3 files changed, 279 insertions(+), 1 deletion(-) create mode 100644 src/eva/user_simulator/cascade/stt.py create mode 100644 tests/unit/user_simulator/cascade/test_stt.py diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 1995823a..4f55b742 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.2" +simulation_version = "2.0.3" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/stt.py b/src/eva/user_simulator/cascade/stt.py new file mode 100644 index 00000000..1060b6f5 --- /dev/null +++ b/src/eva/user_simulator/cascade/stt.py @@ -0,0 +1,127 @@ +"""Streaming speech-to-text client that transcribes the assistant's audio for the caller.""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import os +from typing import Any + +import websockets + +from eva.user_simulator.cascade.constants import CALLER_SAMPLE_RATE +from eva.utils.logging import get_logger + +logger = get_logger(__name__) + +SCRIBE_URL = "wss://api.elevenlabs.io/v1/speech-to-text/realtime" + +INCOMPLETE_MARKER = "[CURRENTLY SPEAKING, INCOMPLETE]" + + +class TranscriptBuffer: + """Accumulates committed transcript segments separately from the in-flight partial.""" + + def __init__(self) -> None: + self.committed = "" + self.in_flight = "" + + def apply_partial(self, text: str) -> None: + """Replace the in-flight partial with the latest partial transcript.""" + self.in_flight = text + + def commit(self, text: str) -> None: + """Append a finalized segment to the committed text and clear the partial.""" + self.committed = f"{self.committed} {text}".strip() if self.committed else text + self.in_flight = "" + + def current_text(self) -> str: + """Render everything heard so far, marking an in-flight utterance as incomplete.""" + if not self.in_flight: + return self.committed + prefix = f"{self.committed} " if self.committed else "" + return f"{prefix}{self.in_flight} {INCOMPLETE_MARKER}" + + def take_committed(self) -> str: + """Return and clear the committed text.""" + text = self.committed + self.committed = "" + return text + + +class ScribeStreamingSTT: + """Streams PCM16 to Scribe and folds results into a TranscriptBuffer. + + Uses commit_strategy=manual so the caller's own turn-end decision drives + commits; Scribe's built-in VAD would be a second, independent detector on its + own timing, which is the problem the tick design exists to remove. + """ + + def __init__(self, params: dict[str, Any], *, language: str = "en") -> None: + self._model = params.get("model", "scribe_v2_realtime") + self._api_key = params.get("api_key") or os.environ.get("ELEVENLABS_API_KEY", "") + self._language = language + self.buffer = TranscriptBuffer() + self._ws: Any = None + self._receive_task: asyncio.Task | None = None + self._error: Exception | None = None + + async def start(self) -> None: + """Open the transcription socket and begin consuming results.""" + self._ws = await websockets.connect( + f"{SCRIBE_URL}?model_id={self._model}" + f"&audio_format=pcm_{CALLER_SAMPLE_RATE}" + f"&language_code={self._language}" + "&commit_strategy=manual", + additional_headers={"xi-api-key": self._api_key}, + ) + self._receive_task = asyncio.create_task(self._receive_loop()) + + async def feed(self, pcm: bytes, *, commit: bool = False) -> None: + """Send one tick of assistant audio, optionally closing the utterance.""" + if self._ws is None: + return + message: dict[str, Any] = { + "message_type": "input_audio_chunk", + "audio_base_64": base64.b64encode(pcm).decode(), + } + if commit: + message["commit"] = True + try: + await self._ws.send(json.dumps(message)) + except Exception as exc: + logger.warning(f"Scribe send failed: {exc}") + + async def stop(self) -> None: + """Close the socket and stop consuming. Safe to call twice.""" + if self._receive_task is not None: + self._receive_task.cancel() + self._receive_task = None + if self._ws is not None: + try: + await self._ws.close() + finally: + self._ws = None + + async def _receive_loop(self) -> None: + """Fold partial and committed transcripts into the buffer as they arrive.""" + while True: + try: + raw = await self._ws.recv() + except Exception as exc: + self._error = exc + logger.exception("Scribe receive loop failed") + return + try: + message = json.loads(raw) + except json.JSONDecodeError: + continue + kind = message.get("message_type", "") + text = message.get("text", "") + if not text: + continue + if kind == "partial_transcript": + self.buffer.apply_partial(text) + elif kind.startswith("committed_transcript") or kind.startswith("final_transcript"): + self.buffer.commit(text) diff --git a/tests/unit/user_simulator/cascade/test_stt.py b/tests/unit/user_simulator/cascade/test_stt.py new file mode 100644 index 00000000..655fb03e --- /dev/null +++ b/tests/unit/user_simulator/cascade/test_stt.py @@ -0,0 +1,151 @@ +import asyncio +import base64 +import json + +from eva.user_simulator.cascade.stt import ScribeStreamingSTT, TranscriptBuffer + + +def test_partial_updates_replace_the_in_flight_text(): + buffer = TranscriptBuffer() + + buffer.apply_partial("Let me check") + buffer.apply_partial("Let me check that for") + + assert buffer.in_flight == "Let me check that for" + assert buffer.committed == "" + + +def test_commit_appends_and_clears_the_partial(): + buffer = TranscriptBuffer() + buffer.apply_partial("Let me check that") + + buffer.commit("Let me check that for you.") + + assert buffer.committed == "Let me check that for you." + assert buffer.in_flight == "" + + +def test_successive_commits_accumulate_with_spaces(): + buffer = TranscriptBuffer() + + buffer.commit("First sentence.") + buffer.commit("Second sentence.") + + assert buffer.committed == "First sentence. Second sentence." + + +def test_current_text_marks_the_incomplete_utterance(): + buffer = TranscriptBuffer() + buffer.commit("I found your booking.") + buffer.apply_partial("It leaves on Thurs") + + assert buffer.current_text() == "I found your booking. It leaves on Thurs [CURRENTLY SPEAKING, INCOMPLETE]" + + +def test_current_text_omits_the_marker_when_nothing_is_in_flight(): + buffer = TranscriptBuffer() + buffer.commit("I found your booking.") + + assert buffer.current_text() == "I found your booking." + + +def test_take_committed_drains_the_buffer(): + buffer = TranscriptBuffer() + buffer.commit("All done.") + + assert buffer.take_committed() == "All done." + assert buffer.committed == "" + + +class SuspendingFakeWebSocket: + """Fake websocket whose recv() genuinely suspends (awaits a future) before returning.""" + + def __init__(self, messages): + self._messages = list(messages) + self.sent: list[dict] = [] + + async def recv(self): + await asyncio.sleep(0.001) + if not self._messages: + await asyncio.Event().wait() + item = self._messages.pop(0) + if isinstance(item, Exception): + raise item + return json.dumps(item) + + async def send(self, raw: str) -> None: + self.sent.append(json.loads(raw)) + + async def close(self) -> None: + pass + + +async def _make_stt(messages) -> tuple[ScribeStreamingSTT, SuspendingFakeWebSocket]: + stt = ScribeStreamingSTT({"api_key": "test-key"}) + fake = SuspendingFakeWebSocket(messages) + stt._ws = fake + stt._receive_task = asyncio.create_task(stt._receive_loop()) + return stt, fake + + +async def test_partial_transcript_lands_in_in_flight(): + stt, _ = await _make_stt( + [ + {"message_type": "session_started", "session_id": "abc", "config": {}}, + {"message_type": "partial_transcript", "text": "It leaves on Thurs"}, + ] + ) + await asyncio.sleep(0.05) + assert stt.buffer.in_flight == "It leaves on Thurs" + await stt.stop() + + +async def test_committed_transcript_lands_in_committed(): + stt, _ = await _make_stt( + [ + {"message_type": "committed_transcript", "text": "It leaves on Thursday."}, + ] + ) + await asyncio.sleep(0.05) + assert stt.buffer.committed == "It leaves on Thursday." + await stt.stop() + + +async def test_feed_with_commit_sends_commit_true(): + stt, fake = await _make_stt([]) + await stt.feed(b"\x00\x00", commit=True) + + assert fake.sent[-1]["commit"] is True + assert base64.b64decode(fake.sent[-1]["audio_base_64"]) == b"\x00\x00" + await stt.stop() + + +async def test_feed_without_commit_omits_commit_flag(): + stt, fake = await _make_stt([]) + await stt.feed(b"\x00\x00") + + assert "commit" not in fake.sent[-1] + await stt.stop() + + +async def test_session_started_is_ignored_harmlessly(): + stt, _ = await _make_stt( + [ + {"message_type": "session_started", "session_id": "abc", "config": {}}, + ] + ) + await asyncio.sleep(0.05) + assert stt.buffer.committed == "" + assert stt.buffer.in_flight == "" + assert stt._error is None + await stt.stop() + + +async def test_recv_error_is_recorded_and_object_stays_usable(): + stt, fake = await _make_stt([RuntimeError("boom")]) + await asyncio.sleep(0.05) + + assert isinstance(stt._error, RuntimeError) + await stt.feed(b"\x00\x00") + assert fake.sent + await stt.stop() From c141dd2d0daa08c33a142cb8a955b96d78d6371f Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 6 Aug 2026 23:00:28 -0400 Subject: [PATCH 15/65] Add Cartesia TTS client for caller speech synthesis Renders caller text to PCM16 via Cartesia Sonic HTTP API, with stable per-persona voice selection for later phrase-cache reuse. Co-Authored-By: Claude Opus 5 (1M context) --- src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/tts.py | 63 +++++++++++++++++++ tests/unit/user_simulator/cascade/test_tts.py | 35 +++++++++++ 3 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 src/eva/user_simulator/cascade/tts.py create mode 100644 tests/unit/user_simulator/cascade/test_tts.py diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 4f55b742..254ed74d 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.3" +simulation_version = "2.0.4" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/tts.py b/src/eva/user_simulator/cascade/tts.py new file mode 100644 index 00000000..231ebd99 --- /dev/null +++ b/src/eva/user_simulator/cascade/tts.py @@ -0,0 +1,63 @@ +"""Caller speech synthesis via Cartesia Sonic.""" + +from __future__ import annotations + +import os +from typing import Any + +import httpx + +from eva.user_simulator.cascade.constants import CALLER_SAMPLE_RATE +from eva.utils.logging import get_logger + +logger = get_logger(__name__) + +CARTESIA_URL = "https://api.cartesia.ai/tts/bytes" +CARTESIA_VERSION = "2024-06-10" +DEFAULT_FEMALE_VOICE = "f786b574-daa5-4673-aa0c-cbe3e8534c02" +DEFAULT_MALE_VOICE = "f786b574-daa5-4673-aa0c-cbe3e8534c02" +_FEMALE_PERSONA_ID = 1 + + +class CartesiaTTS: + """Renders caller text to PCM16 at the simulator's sample rate.""" + + def __init__(self, params: dict[str, Any], *, language: str = "en") -> None: + self._model = params.get("model", "sonic-3.5") + self._api_key = params.get("api_key") or os.environ.get("CARTESIA_API_KEY", "") + self._female_voice = params.get("female_voice", DEFAULT_FEMALE_VOICE) + self._male_voice = params.get("male_voice", DEFAULT_MALE_VOICE) + self._language = language + + def voice_for_persona(self, persona_config: dict[str, Any]) -> str: + """Pick a stable voice for this persona, mirroring the existing gender scheme.""" + if persona_config.get("user_persona_id") == _FEMALE_PERSONA_ID: + return self._female_voice + if persona_config.get("user_persona_id") is None: + return self._female_voice + return self._male_voice + + async def synthesize(self, text: str, *, voice_id: str) -> bytes: + """Render text to raw PCM16 mono at CALLER_SAMPLE_RATE.""" + if not text: + return b"" + if not self._api_key: + raise ValueError("Cartesia API key missing: set tts_params.api_key or CARTESIA_API_KEY") + + body = { + "model_id": self._model, + "transcript": text, + "voice": {"mode": "id", "id": voice_id}, + "language": self._language, + "output_format": { + "container": "raw", + "encoding": "pcm_s16le", + "sample_rate": CALLER_SAMPLE_RATE, + }, + } + headers = {"X-API-Key": self._api_key, "Cartesia-Version": CARTESIA_VERSION} + + async with httpx.AsyncClient(timeout=30.0) as client: + response = await client.post(CARTESIA_URL, json=body, headers=headers) + response.raise_for_status() + return response.content diff --git a/tests/unit/user_simulator/cascade/test_tts.py b/tests/unit/user_simulator/cascade/test_tts.py new file mode 100644 index 00000000..1973768d --- /dev/null +++ b/tests/unit/user_simulator/cascade/test_tts.py @@ -0,0 +1,35 @@ +import pytest + +from eva.user_simulator.cascade.tts import CartesiaTTS + + +def test_voice_id_selected_for_female_persona(): + tts = CartesiaTTS({"model": "sonic-3.5", "female_voice": "voice-f", "male_voice": "voice-m"}) + + assert tts.voice_for_persona({"user_persona_id": 1}) == "voice-f" + + +def test_voice_id_selected_for_male_persona(): + tts = CartesiaTTS({"model": "sonic-3.5", "female_voice": "voice-f", "male_voice": "voice-m"}) + + assert tts.voice_for_persona({"user_persona_id": 2}) == "voice-m" + + +def test_unknown_persona_falls_back_to_female_voice(): + tts = CartesiaTTS({"model": "sonic-3.5", "female_voice": "voice-f", "male_voice": "voice-m"}) + + assert tts.voice_for_persona({}) == "voice-f" + + +async def test_empty_text_synthesizes_to_no_audio(): + tts = CartesiaTTS({"model": "sonic-3.5", "api_key": "k"}) + + assert await tts.synthesize("", voice_id="voice-f") == b"" + + +async def test_missing_api_key_raises_a_clear_error(monkeypatch): + monkeypatch.delenv("CARTESIA_API_KEY", raising=False) + tts = CartesiaTTS({"model": "sonic-3.5"}) + + with pytest.raises(ValueError, match="Cartesia API key"): + await tts.synthesize("hello", voice_id="voice-f") From c7970ce0bc1f3a362e2d0c951ae16fd4a6277425 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 6 Aug 2026 23:04:52 -0400 Subject: [PATCH 16/65] add cascade caller turn JSON contract prompt Adds user_simulator.cascade_turn_contract, the JSON output contract the cascade STT->LLM->TTS pipeline layers on top of the existing persona/goal prompt assembled by _build_prompt(). Hanging up remains a tool call (end_call), not a JSON field, per the openai_realtime pattern. Co-Authored-By: Claude Opus 5 (1M context) --- configs/prompts/simulation.yaml | 15 +++++++++++++++ src/eva/__init__.py | 2 +- tests/unit/user_simulator/cascade/test_prompt.py | 9 +++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 tests/unit/user_simulator/cascade/test_prompt.py diff --git a/configs/prompts/simulation.yaml b/configs/prompts/simulation.yaml index 4648cb62..530c37da 100644 --- a/configs/prompts/simulation.yaml +++ b/configs/prompts/simulation.yaml @@ -507,3 +507,18 @@ user_simulator: For languages that use non-Latin scripts, spell out characters using their standard phonetic names in your language. IMPORTANT: Before ending the conversation, confirm with the agent that there are no outstanding actions. The end_call tool should only be called in a turn that is a brief goodbye — never in the same turn where you are providing the agent with data, an identifier, a request to transfer to a live agent, an approval to proceed, or any kind of additional information. + + cascade_turn_contract: | + Respond with a single JSON object and nothing else. It must have exactly this field: + + {{ + "utterance": "" + }} + + Rules for "utterance": + - Write it the way a person speaks on a phone call, not the way they write. + - One turn only. Do not script the agent's reply or your own next turn. + - Never include stage directions, names, labels, or quotation marks around the whole line. + - If you have nothing to add but the call is not over, say a short natural acknowledgement. + + To hang up, call the end_call tool. Do not describe hanging up in "utterance". diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 254ed74d..114d2005 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.4" +simulation_version = "2.0.5" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/tests/unit/user_simulator/cascade/test_prompt.py b/tests/unit/user_simulator/cascade/test_prompt.py new file mode 100644 index 00000000..fa86c1c8 --- /dev/null +++ b/tests/unit/user_simulator/cascade/test_prompt.py @@ -0,0 +1,9 @@ +from eva.utils.prompt_manager import PromptManager + + +def test_cascade_turn_contract_prompt_exists_and_names_the_json_field(): + prompt = PromptManager().get_prompt("user_simulator.cascade_turn_contract") + + assert "utterance" in prompt + assert "JSON" in prompt + assert "end_call" in prompt From ff45644f7302094be0a192eb574fcbf68947b7da Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 6 Aug 2026 23:10:29 -0400 Subject: [PATCH 17/65] add CascadeUserSimulator --- src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/simulator.py | 201 ++++++++++++++++++ .../user_simulator/cascade/test_simulator.py | 60 ++++++ 3 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 src/eva/user_simulator/cascade/simulator.py create mode 100644 tests/unit/user_simulator/cascade/test_simulator.py diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 114d2005..c6093d24 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.5" +simulation_version = "2.0.6" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py new file mode 100644 index 00000000..9c478443 --- /dev/null +++ b/src/eva/user_simulator/cascade/simulator.py @@ -0,0 +1,201 @@ +"""Self-hosted STT/LLM/TTS caller simulator driven by the tick scheduler.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +import websockets + +from eva.assistant.services.llm import LiteLLMClient +from eva.models.config import CascadeSimulatorConfig, PerturbationConfig +from eva.user_simulator.base import AbstractUserSimulator +from eva.user_simulator.cascade.adapter.realtime_ws import RealtimeWSAdapter +from eva.user_simulator.cascade.constants import CALLER_SAMPLE_RATE, TICK_DURATION_MS +from eva.user_simulator.cascade.scheduler import TickScheduler +from eva.user_simulator.cascade.stt import ScribeStreamingSTT +from eva.user_simulator.cascade.tts import CartesiaTTS +from eva.utils.logging import get_logger +from eva.utils.prompt_manager import PromptManager + +logger = get_logger(__name__) + +_FENCE = re.compile(r"^```(?:json)?\s*|\s*```$", re.MULTILINE) + +END_CALL_DESCRIPTION = """Use this to end the phone call and hang up. + +Call this function when it is time to end the call and one of the following is true: +1. The agent has confirmed your request is resolved, all steps are completed, and you have said goodbye. +2. The agent has initiated a transfer to a live agent. +3. The agent has been unable to make progress for at least 5 consecutive turns. +4. The agent says goodbye or indicates the conversation is over. +5. The agent indicates that the remainder of your request cannot be fulfilled. +6. The assistant reports an unrecoverable processing error. + +Never call this tool in the same turn that you provide the agent with data, an identifier, +an approval to proceed, a transfer request, or any other information. Say a brief goodbye first.""" + +END_CALL_TOOL = { + "type": "function", + "function": { + "name": "end_call", + "description": END_CALL_DESCRIPTION, + "parameters": {"type": "object", "properties": {}, "required": []}, + }, +} + + +def parse_turn_response(raw: str) -> str: + """Extract the utterance from the caller LLM's JSON reply. + + Falls back to treating the whole response as the utterance when it is not + JSON, so a malformed reply degrades into a plain turn rather than silence. + """ + stripped = _FENCE.sub("", raw).strip() + if not stripped: + return "" + try: + payload = json.loads(stripped) + except json.JSONDecodeError: + return stripped + if not isinstance(payload, dict): + return stripped + utterance = payload.get("utterance", "") + if not isinstance(utterance, str): + raise ValueError(f"Caller LLM returned a non-string utterance: {utterance!r}") + return utterance + + +def extract_turn(message: object) -> tuple[str, bool]: + """Read (utterance, end_call) from whatever LiteLLMClient returned. + + ``complete()`` returns a bare ``str`` when the model made no tool call and a + message object when it did, so both shapes must be handled. + """ + if isinstance(message, str): + return parse_turn_response(message), False + content = getattr(message, "content", None) or "" + calls = getattr(message, "tool_calls", None) or [] + end_call = any(getattr(call.function, "name", "") == "end_call" for call in calls) + return parse_turn_response(content), end_call + + +class CascadeUserSimulator(AbstractUserSimulator): + """Simulated caller built from independently chosen STT, LLM, and TTS models.""" + + def __init__( + self, + current_date_time: str, + persona_config: dict, + goal: dict, + server_url: str, + output_dir: Path, + agent_id: str, + timeout: int = 600, + perturbation_config: PerturbationConfig | None = None, + language: str = "en", + *, + simulator_config: CascadeSimulatorConfig, + ) -> None: + super().__init__( + current_date_time=current_date_time, + persona_config=persona_config, + goal=goal, + server_url=server_url, + output_dir=output_dir, + agent_id=agent_id, + timeout=timeout, + perturbation_config=perturbation_config, + language=language, + provider="cascade", + ) + self._config = simulator_config + self._stt = ScribeStreamingSTT(simulator_config.stt_params, language=language) + self._tts = CartesiaTTS(simulator_config.tts_params, language=language) + self._llm = LiteLLMClient(model=simulator_config.llm) + self._voice_id = self._tts.voice_for_persona(persona_config) + self._history: list[dict[str, str]] = [] + + async def run_conversation(self) -> str: + """Run the tick loop until the call ends, and return the end reason.""" + try: + await self._run() + except Exception as exc: + logger.exception(f"Cascade simulator failed: {exc}") + self._end_reason = "error" + self.event_logger.log_error(str(exc)) + finally: + self._save_clean_user_audio(CALLER_SAMPLE_RATE) + self.event_logger.save() + return self._end_reason + + async def _run(self) -> None: + """Drive the scheduler until end_call, timeout, or disconnect.""" + websocket = await websockets.connect(self.server_url) + adapter = RealtimeWSAdapter( + websocket=websocket, + conversation_id=self._record_id or "cascade", + ) + scheduler = TickScheduler(adapter) + + await adapter.start() + await self._stt.start() + self.event_logger.log_connection_state("connected", {"server_url": self.server_url}) + + max_ticks = self.timeout * 1000 // TICK_DURATION_MS + assistant_was_speaking = False + try: + while scheduler.tick < max_ticks and not self._conversation_done.is_set(): + result = await scheduler.run_tick() + if result.has_assistant_speech: + await self._stt.feed(result.assistant_audio) + assistant_was_speaking = True + continue + if assistant_was_speaking: + # Assistant just stopped: close the utterance so Scribe emits a + # committed_transcript. With commit_strategy=manual nothing is + # ever finalized unless we say so, and take_committed() below + # would return empty forever. + await self._stt.feed(result.assistant_audio, commit=True) + assistant_was_speaking = False + continue + if scheduler.caller_is_speaking or not scheduler.may_take_turn(): + continue + if await self._take_turn(scheduler): + break + else: + if not self._conversation_done.is_set(): + self._on_conversation_end("timeout") + finally: + await self._stt.stop() + await adapter.stop() + self.event_logger.log_connection_state("session_ended", {"reason": self._end_reason}) + + async def _take_turn(self, scheduler: TickScheduler) -> bool: + """Generate, synthesize, and queue one caller turn. Returns True to hang up.""" + heard = self._stt.buffer.take_committed() + if heard: + self._history.append({"role": "assistant", "content": heard}) + self._on_assistant_speaks(heard) + + message, _stats = await self._llm.complete(messages=self._messages(), tools=[END_CALL_TOOL]) + utterance, end_call = extract_turn(message) + + if utterance: + self._history.append({"role": "user", "content": utterance}) + self._on_user_speaks(utterance) + audio = await self._tts.synthesize(utterance, voice_id=self._voice_id) + self._record_audio("user_clean", audio) + scheduler.enqueue_utterance(audio) + self.event_logger.log_event("caller_turn", {"text": utterance, "tick_index": scheduler.tick}) + + if end_call: + self._on_conversation_end("goodbye") + return True + return False + + def _messages(self) -> list[dict[str, str]]: + """Build the caller LLM message list: persona/goal prompt, JSON contract, history.""" + system = self._build_prompt() + "\n\n" + PromptManager().get_prompt("user_simulator.cascade_turn_contract") + return [{"role": "system", "content": system}, *self._history] diff --git a/tests/unit/user_simulator/cascade/test_simulator.py b/tests/unit/user_simulator/cascade/test_simulator.py new file mode 100644 index 00000000..40e21c49 --- /dev/null +++ b/tests/unit/user_simulator/cascade/test_simulator.py @@ -0,0 +1,60 @@ +import json + +import pytest + +from eva.user_simulator.cascade.simulator import extract_turn, parse_turn_response + + +def test_parse_turn_response_reads_a_clean_json_object(): + assert parse_turn_response('{"utterance": "Hi there."}') == "Hi there." + + +def test_parse_turn_response_tolerates_markdown_fences(): + assert parse_turn_response('```json\n{"utterance": "Bye."}\n```') == "Bye." + + +def test_parse_turn_response_falls_back_to_raw_text_when_not_json(): + assert parse_turn_response("I need to reset my password.") == "I need to reset my password." + + +def test_parse_turn_response_returns_empty_for_a_toolcall_only_turn(): + # The model hangs up by calling end_call and says nothing; content is empty. + assert parse_turn_response("") == "" + + +def test_parse_turn_response_rejects_a_non_string_utterance(): + with pytest.raises(ValueError, match="utterance"): + parse_turn_response(json.dumps({"utterance": 42})) + + +def test_extract_turn_reads_a_plain_string_as_no_hangup(): + # LiteLLMClient returns a bare str when the model made no tool call. + assert extract_turn('{"utterance": "Still here."}') == ("Still here.", False) + + +def test_extract_turn_detects_the_end_call_tool(): + class _Fn: + name = "end_call" + + class _Call: + function = _Fn() + + class _Message: + content = "" + tool_calls = [_Call()] + + assert extract_turn(_Message()) == ("", True) + + +def test_extract_turn_ignores_an_unrelated_tool_call(): + class _Fn: + name = "something_else" + + class _Call: + function = _Fn() + + class _Message: + content = '{"utterance": "Go on."}' + tool_calls = [_Call()] + + assert extract_turn(_Message()) == ("Go on.", False) From 086fee5ee73c67849c908a38db56b718a99ee9ce Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 6 Aug 2026 23:15:17 -0400 Subject: [PATCH 18/65] warn when cascade ignores unsupported audio perturbation --- src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/simulator.py | 23 +++++++++++++++++++ .../user_simulator/cascade/test_simulator.py | 22 +++++++++++++++++- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index c6093d24..cda07663 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.6" +simulation_version = "2.0.7" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py index 9c478443..79215ac5 100644 --- a/src/eva/user_simulator/cascade/simulator.py +++ b/src/eva/user_simulator/cascade/simulator.py @@ -110,6 +110,7 @@ def __init__( language=language, provider="cascade", ) + self._warn_unsupported_perturbation(perturbation_config) self._config = simulator_config self._stt = ScribeStreamingSTT(simulator_config.stt_params, language=language) self._tts = CartesiaTTS(simulator_config.tts_params, language=language) @@ -195,6 +196,28 @@ async def _take_turn(self, scheduler: TickScheduler) -> bool: return True return False + @staticmethod + def _warn_unsupported_perturbation(perturbation_config: PerturbationConfig | None) -> None: + """Warn when outbound-audio perturbations are configured but unsupported by cascade. + + `RealtimeWSAdapter` does not yet apply perturbation to outbound audio, so + `background_noise` and `connection_degradation` are silently dropped without this. + `snr_db` only has an effect alongside `background_noise` and defaults to 15.0, so it + is flagged only when `background_noise` is also set, not on its default value alone. + """ + if perturbation_config is None: + return + unsupported = [] + if perturbation_config.background_noise is not None: + unsupported.extend(["background_noise", "snr_db"]) + if perturbation_config.connection_degradation: + unsupported.append("connection_degradation") + if unsupported: + logger.warning( + f"Cascade simulator does not yet apply audio perturbation: ignoring {', '.join(unsupported)}. " + "Behavior and accent perturbations are unaffected." + ) + def _messages(self) -> list[dict[str, str]]: """Build the caller LLM message list: persona/goal prompt, JSON contract, history.""" system = self._build_prompt() + "\n\n" + PromptManager().get_prompt("user_simulator.cascade_turn_contract") diff --git a/tests/unit/user_simulator/cascade/test_simulator.py b/tests/unit/user_simulator/cascade/test_simulator.py index 40e21c49..9e20c8f3 100644 --- a/tests/unit/user_simulator/cascade/test_simulator.py +++ b/tests/unit/user_simulator/cascade/test_simulator.py @@ -1,8 +1,10 @@ import json +import logging import pytest -from eva.user_simulator.cascade.simulator import extract_turn, parse_turn_response +from eva.models.config import PerturbationConfig +from eva.user_simulator.cascade.simulator import CascadeUserSimulator, extract_turn, parse_turn_response def test_parse_turn_response_reads_a_clean_json_object(): @@ -58,3 +60,21 @@ class _Message: tool_calls = [_Call()] assert extract_turn(_Message()) == ("Go on.", False) + + +def test_warn_unsupported_perturbation_fires_for_background_noise(caplog): + with caplog.at_level(logging.WARNING): + CascadeUserSimulator._warn_unsupported_perturbation(PerturbationConfig(background_noise="road_noise")) + assert any("background_noise" in record.message for record in caplog.records) + + +def test_warn_unsupported_perturbation_is_silent_for_a_default_config(caplog): + with caplog.at_level(logging.WARNING): + CascadeUserSimulator._warn_unsupported_perturbation(PerturbationConfig()) + assert caplog.records == [] + + +def test_warn_unsupported_perturbation_is_silent_for_none(caplog): + with caplog.at_level(logging.WARNING): + CascadeUserSimulator._warn_unsupported_perturbation(None) + assert caplog.records == [] From 1436fc0cd9f816d33fc45e1374fe56350fd0fda3 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 6 Aug 2026 23:18:53 -0400 Subject: [PATCH 19/65] wire cascade simulator into the factory Adds the CascadeSimulatorConfig dispatch branch so EVA_USER_SIMULATOR__PROVIDER=cascade selects CascadeUserSimulator end to end. Co-Authored-By: Claude Opus 5 (1M context) --- src/eva/__init__.py | 2 +- src/eva/user_simulator/factory.py | 11 ++++++++++- tests/unit/user_simulator/test_factory.py | 23 +++++++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index cda07663..9edc3391 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.7" +simulation_version = "2.0.8" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/factory.py b/src/eva/user_simulator/factory.py index d2cef7e2..0ad5da17 100644 --- a/src/eva/user_simulator/factory.py +++ b/src/eva/user_simulator/factory.py @@ -4,7 +4,12 @@ from typing import Any -from eva.models.config import ElevenLabsSimulatorConfig, OpenAIRealtimeSimulatorConfig, UserSimulatorConfig +from eva.models.config import ( + CascadeSimulatorConfig, + ElevenLabsSimulatorConfig, + OpenAIRealtimeSimulatorConfig, + UserSimulatorConfig, +) from eva.user_simulator.base import AbstractUserSimulator @@ -21,4 +26,8 @@ def create_user_simulator( from eva.user_simulator.openai_realtime import OpenAIRealtimeUserSimulator return OpenAIRealtimeUserSimulator(simulator_config=simulator_config, **kwargs) + if isinstance(simulator_config, CascadeSimulatorConfig): + from eva.user_simulator.cascade.simulator import CascadeUserSimulator + + return CascadeUserSimulator(simulator_config=simulator_config, **kwargs) raise ValueError(f"Unknown user simulator provider: {simulator_config.provider!r}") diff --git a/tests/unit/user_simulator/test_factory.py b/tests/unit/user_simulator/test_factory.py index d47f1abe..74fdb4b9 100644 --- a/tests/unit/user_simulator/test_factory.py +++ b/tests/unit/user_simulator/test_factory.py @@ -43,3 +43,26 @@ def test_factory_selects_openai_realtime(tmp_path): assert isinstance(simulator, OpenAIRealtimeUserSimulator) assert simulator.caller_model == "gpt-realtime-1.5" + + +def test_factory_selects_cascade(tmp_path): + from eva.models.config import CascadeSimulatorConfig + from eva.user_simulator.cascade.simulator import CascadeUserSimulator + from eva.utils import router + + router.init( + model_list=[ + { + "model_name": "gpt-5.5", + "litellm_params": {"model": "openai/gpt-5.5", "api_key": "test-key"}, + } + ] + ) + try: + config = CascadeSimulatorConfig() + simulator = create_user_simulator(config, **_kwargs(tmp_path)) + + assert isinstance(simulator, CascadeUserSimulator) + assert simulator.provider == "cascade" + finally: + router.reset() From 3ee5f94e1ed60c949055b4a3e05659e10f3dbe81 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 6 Aug 2026 23:38:43 -0400 Subject: [PATCH 20/65] Enforce minimum tick duration in RealtimeWSAdapter.run_tick A silent tick (outgoing_audio=None) returned immediately with no I/O, so the tick loop spun at CPU speed instead of pacing at 200ms/tick, racing thousands of ticks ahead of the assistant within ~400ms and producing a bogus timeout. run_tick now sleeps out the remainder of TICK_DURATION_MS after all other work, so a speaking tick (which already spends ~200ms in per-frame outbound pacing) does not double-sleep. Bump simulation_version: this is a user-simulator behavior change. --- src/eva/__init__.py | 2 +- .../cascade/adapter/realtime_ws.py | 13 +++++++-- .../cascade/test_realtime_ws_adapter.py | 28 +++++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 9edc3391..301688ba 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.8" +simulation_version = "2.0.9" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/adapter/realtime_ws.py b/src/eva/user_simulator/cascade/adapter/realtime_ws.py index 5794a2b3..fbf0affc 100644 --- a/src/eva/user_simulator/cascade/adapter/realtime_ws.py +++ b/src/eva/user_simulator/cascade/adapter/realtime_ws.py @@ -33,7 +33,9 @@ class RealtimeWSAdapter(Adapter): (docs/assistant_server_contract.md section 3). A background task continuously drains inbound frames into an adapter-owned buffer; `run_tick` releases exactly one tick's worth per call, so a provider that generates faster than real time - cannot run ahead of the simulation clock. + cannot run ahead of the simulation clock. `run_tick` also enforces a minimum + tick duration as a safety net for silent ticks, which otherwise return with no + I/O at all and would let the simulation race ahead of wall-clock time. Each direction resamples a continuous stream, so each carries its own `audioop.ratecv` filter state across calls; the stateless helpers in @@ -59,6 +61,7 @@ async def start(self) -> None: async def run_tick(self, tick_number: int, outgoing_audio: bytes | None) -> TickResult: """Send one tick of caller audio at wire cadence and collect one tick of assistant audio.""" + tick_start = asyncio.get_event_loop().time() if self._error is not None: raise RuntimeError("RealtimeWSAdapter receive loop failed") from self._error @@ -79,13 +82,19 @@ async def run_tick(self, tick_number: int, outgoing_audio: bytes | None) -> Tick if self._error is not None: raise RuntimeError("RealtimeWSAdapter receive loop failed") from self._error - return TickResult( + result = TickResult( tick_number=tick_number, assistant_audio=chunk, assistant_audio_raw_bytes=len(raw), wall_clock_ms=int(time.time() * 1000), ) + remaining = TICK_DURATION_MS / 1000 - (asyncio.get_event_loop().time() - tick_start) + if remaining > 0: + await asyncio.sleep(remaining) + + return result + async def stop(self) -> None: """Send stop, cancel the receive loop, and close the socket. Safe to call twice.""" if self._receive_task is not None: diff --git a/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py b/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py index d152cf83..955437e5 100644 --- a/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py +++ b/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py @@ -225,6 +225,34 @@ async def test_user_speech_start_emitted_once_on_silence_to_audio_transition(): await adapter.stop() +async def test_silent_tick_paces_to_approximately_one_tick_duration(): + ws = FakeWebSocket() + adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await adapter.start() + + start = asyncio.get_event_loop().time() + await adapter.run_tick(0, None) + elapsed = asyncio.get_event_loop().time() - start + + assert 0.15 < elapsed < 0.4 + + await adapter.stop() + + +async def test_speaking_tick_paces_to_approximately_one_tick_duration_not_double(): + ws = FakeWebSocket() + adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await adapter.start() + + start = asyncio.get_event_loop().time() + await adapter.run_tick(0, b"\x00" * BYTES_PER_TICK) + elapsed = asyncio.get_event_loop().time() - start + + assert 0.15 < elapsed < 0.4 + + await adapter.stop() + + async def test_user_speech_stop_emitted_once_on_audio_to_silence_transition(): ws = FakeWebSocket() adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) From 8ef61875fb9352d8fa9f75b50cbd2365994f857a Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 6 Aug 2026 23:50:09 -0400 Subject: [PATCH 21/65] Always send a full tick of audio in RealtimeWSAdapter, silence or real A silent tick (outgoing_audio=None) sent nothing at all instead of silence, leaving gaps in the outbound stream. The assistant's STT/VAD expects a continuous stream like a real phone line, so gaps caused turn detection to misfire and transcripts to merge into single malformed entries. run_tick now always calls _send_tick_audio with either the real audio or a synthesized tick of SILENCE_BYTE, framed and paced identically to speech. Bump simulation_version: this is a user-simulator behavior change. --- src/eva/__init__.py | 2 +- .../cascade/adapter/realtime_ws.py | 19 +++++----- .../cascade/test_realtime_ws_adapter.py | 38 +++++++++++++++++++ 3 files changed, 49 insertions(+), 10 deletions(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 301688ba..642928f8 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.9" +simulation_version = "2.0.10" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/adapter/realtime_ws.py b/src/eva/user_simulator/cascade/adapter/realtime_ws.py index fbf0affc..7ed14962 100644 --- a/src/eva/user_simulator/cascade/adapter/realtime_ws.py +++ b/src/eva/user_simulator/cascade/adapter/realtime_ws.py @@ -14,7 +14,7 @@ import audioop_lts as audioop from eva.user_simulator.cascade.adapter.base import Adapter -from eva.user_simulator.cascade.constants import BYTES_PER_TICK, CALLER_SAMPLE_RATE, TICK_DURATION_MS +from eva.user_simulator.cascade.constants import BYTES_PER_TICK, CALLER_SAMPLE_RATE, SILENCE_BYTE, TICK_DURATION_MS from eva.user_simulator.cascade.tick_result import TickResult, split_tick_audio from eva.utils.logging import get_logger @@ -30,12 +30,14 @@ class RealtimeWSAdapter(Adapter): """Exchanges tick-sized audio with an assistant server over the Twilio WS protocol. Outbound audio is paced at the real 20ms cadence the assistant expects - (docs/assistant_server_contract.md section 3). A background task continuously - drains inbound frames into an adapter-owned buffer; `run_tick` releases exactly - one tick's worth per call, so a provider that generates faster than real time - cannot run ahead of the simulation clock. `run_tick` also enforces a minimum - tick duration as a safety net for silent ticks, which otherwise return with no - I/O at all and would let the simulation race ahead of wall-clock time. + (docs/assistant_server_contract.md section 3), and every tick sends a full + tick's worth of frames — real audio or synthesized silence — so the assistant's + STT/VAD always sees an unbroken stream, the way a real phone line would; gaps + with no frames at all are what caused turn detection to misfire. A background + task continuously drains inbound frames into an adapter-owned buffer; `run_tick` + releases exactly one tick's worth per call, so a provider that generates faster + than real time cannot run ahead of the simulation clock. `run_tick` also enforces + a minimum tick duration as a safety net for ticks that send quickly. Each direction resamples a continuous stream, so each carries its own `audioop.ratecv` filter state across calls; the stateless helpers in @@ -72,8 +74,7 @@ async def run_tick(self, tick_number: int, outgoing_audio: bytes | None) -> Tick await self._send_speech_event("user_speech_stop") self._caller_speaking = is_speaking - if outgoing_audio: - await self._send_tick_audio(outgoing_audio) + await self._send_tick_audio(outgoing_audio or SILENCE_BYTE * self._bytes_per_tick) raw = bytes(self._inbound[: self._bytes_per_tick]) del self._inbound[: len(raw)] diff --git a/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py b/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py index 955437e5..15afc094 100644 --- a/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py +++ b/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py @@ -4,6 +4,11 @@ import pytest +try: + import audioop +except ImportError: # pragma: no cover - Python 3.13+ + import audioop_lts as audioop + from eva.user_simulator.cascade.adapter.realtime_ws import RealtimeWSAdapter BYTES_PER_TICK = 6400 @@ -163,6 +168,39 @@ async def test_outgoing_caller_audio_is_sent_as_twilio_media_frames(): await adapter.stop() +async def test_silent_tick_emits_a_full_tick_of_silence_frames(): + ws = FakeWebSocket() + adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await adapter.start() + + await adapter.run_tick(0, None) + + media_frames = [json.loads(m) for m in ws.sent if json.loads(m).get("event") == "media"] + assert len(media_frames) == 10 + for frame in media_frames: + mulaw = base64.b64decode(frame["media"]["payload"]) + pcm = audioop.ulaw2lin(mulaw, 2) + assert audioop.max(pcm, 2) == 0 + + await adapter.stop() + + +async def test_mixed_speak_silent_sequence_has_no_gaps_in_outbound_stream(): + """Regression test: a stall between speech ticks must still send silence, not nothing.""" + ws = FakeWebSocket() + adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await adapter.start() + + outgoing_sequence = [b"\x00" * BYTES_PER_TICK, None, None, b"\x00" * BYTES_PER_TICK] + for tick_number, outgoing in enumerate(outgoing_sequence): + await adapter.run_tick(tick_number, outgoing) + + media_frames = [json.loads(m) for m in ws.sent if json.loads(m).get("event") == "media"] + assert len(media_frames) == len(outgoing_sequence) * 10 + + await adapter.stop() + + async def test_overflow_audio_carries_into_the_next_tick(): ws = SuspendingFakeWebSocket() adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) From c7755d8f03f2df140e04370ee39456c2f6a6d254 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Fri, 7 Aug 2026 00:06:14 -0400 Subject: [PATCH 22/65] fix cascade caller LLM role inversion and Scribe idle-close MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flip agent/caller roles when building the caller LLM's message list — it is itself the "assistant" in its own frame, so an untouched history read its own prior turns as its own output and echoed them back. Also feed Scribe on every tick instead of only while the assistant speaks, since a continuous PCM stream is what keeps the session from timing out; feed() now raises once the socket closes instead of repeating a swallowed warning every 200ms. --- configs/prompts/simulation.yaml | 3 ++ src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/simulator.py | 33 +++++++++++------ src/eva/user_simulator/cascade/stt.py | 13 ++++++- .../user_simulator/cascade/test_simulator.py | 32 ++++++++++++++++ tests/unit/user_simulator/cascade/test_stt.py | 37 +++++++++++++++++++ 6 files changed, 105 insertions(+), 15 deletions(-) diff --git a/configs/prompts/simulation.yaml b/configs/prompts/simulation.yaml index 530c37da..d6ad017a 100644 --- a/configs/prompts/simulation.yaml +++ b/configs/prompts/simulation.yaml @@ -522,3 +522,6 @@ user_simulator: - If you have nothing to add but the call is not over, say a short natural acknowledgement. To hang up, call the end_call tool. Do not describe hanging up in "utterance". + + cascade_role_reminder: | + REMINDER: You are the CUSTOMER calling for help. Respond as the customer would - with questions, requests, or information about your issue. Do NOT respond as the customer service agent. diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 642928f8..ca837ff7 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.10" +simulation_version = "2.0.11" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py index 79215ac5..e47926ff 100644 --- a/src/eva/user_simulator/cascade/simulator.py +++ b/src/eva/user_simulator/cascade/simulator.py @@ -67,6 +67,11 @@ def parse_turn_response(raw: str) -> str: return utterance +def _flip_role(role: str) -> str: + """Swap user/assistant so the caller LLM sees its own lines tagged assistant.""" + return "assistant" if role == "user" else "user" + + def extract_turn(message: object) -> tuple[str, bool]: """Read (utterance, end_call) from whatever LiteLLMClient returned. @@ -149,17 +154,13 @@ async def _run(self) -> None: try: while scheduler.tick < max_ticks and not self._conversation_done.is_set(): result = await scheduler.run_tick() + # Fed on every tick, speech or silence, so Scribe sees a continuous stream + # and never idles out; committed exactly on the speech->silence transition, + # which is what closes the utterance so take_committed() below isn't starved. + commit = assistant_was_speaking and not result.has_assistant_speech + await self._stt.feed(result.assistant_audio, commit=commit) + assistant_was_speaking = result.has_assistant_speech if result.has_assistant_speech: - await self._stt.feed(result.assistant_audio) - assistant_was_speaking = True - continue - if assistant_was_speaking: - # Assistant just stopped: close the utterance so Scribe emits a - # committed_transcript. With commit_strategy=manual nothing is - # ever finalized unless we say so, and take_committed() below - # would return empty forever. - await self._stt.feed(result.assistant_audio, commit=True) - assistant_was_speaking = False continue if scheduler.caller_is_speaking or not scheduler.may_take_turn(): continue @@ -219,6 +220,14 @@ def _warn_unsupported_perturbation(perturbation_config: PerturbationConfig | Non ) def _messages(self) -> list[dict[str, str]]: - """Build the caller LLM message list: persona/goal prompt, JSON contract, history.""" + """Build the caller LLM message list: persona/goal prompt, JSON contract, flipped history. + + `self._history` is kept in conversation-truth roles (assistant said by the agent, user + said by the caller) since it also feeds logging. The caller LLM is itself the assistant + in its own frame, so that history must be flipped here or a message tagged "assistant" + reads to the model as its own prior output and it echoes it back. + """ system = self._build_prompt() + "\n\n" + PromptManager().get_prompt("user_simulator.cascade_turn_contract") - return [{"role": "system", "content": system}, *self._history] + flipped = [{"role": _flip_role(turn["role"]), "content": turn["content"]} for turn in self._history] + reminder = PromptManager().get_prompt("user_simulator.cascade_role_reminder") + return [{"role": "system", "content": system}, *flipped, {"role": "system", "content": reminder}] diff --git a/src/eva/user_simulator/cascade/stt.py b/src/eva/user_simulator/cascade/stt.py index 1060b6f5..eb2b26bc 100644 --- a/src/eva/user_simulator/cascade/stt.py +++ b/src/eva/user_simulator/cascade/stt.py @@ -66,6 +66,7 @@ def __init__(self, params: dict[str, Any], *, language: str = "en") -> None: self._ws: Any = None self._receive_task: asyncio.Task | None = None self._error: Exception | None = None + self._closed = False async def start(self) -> None: """Open the transcription socket and begin consuming results.""" @@ -79,9 +80,15 @@ async def start(self) -> None: self._receive_task = asyncio.create_task(self._receive_loop()) async def feed(self, pcm: bytes, *, commit: bool = False) -> None: - """Send one tick of assistant audio, optionally closing the utterance.""" + """Send one tick of assistant audio, optionally closing the utterance. + + Raises once the session has closed, instead of repeating a swallowed warning every + tick while the caller goes silently deaf for the rest of the call. + """ if self._ws is None: return + if self._closed: + raise RuntimeError("Scribe session is closed; caller can no longer hear the assistant") message: dict[str, Any] = { "message_type": "input_audio_chunk", "audio_base_64": base64.b64encode(pcm).decode(), @@ -91,7 +98,9 @@ async def feed(self, pcm: bytes, *, commit: bool = False) -> None: try: await self._ws.send(json.dumps(message)) except Exception as exc: - logger.warning(f"Scribe send failed: {exc}") + self._closed = True + logger.error(f"Scribe session closed unexpectedly; caller is now deaf: {exc}") + raise RuntimeError("Scribe session closed unexpectedly") from exc async def stop(self) -> None: """Close the socket and stop consuming. Safe to call twice.""" diff --git a/tests/unit/user_simulator/cascade/test_simulator.py b/tests/unit/user_simulator/cascade/test_simulator.py index 9e20c8f3..7aae1c16 100644 --- a/tests/unit/user_simulator/cascade/test_simulator.py +++ b/tests/unit/user_simulator/cascade/test_simulator.py @@ -78,3 +78,35 @@ def test_warn_unsupported_perturbation_is_silent_for_none(caplog): with caplog.at_level(logging.WARNING): CascadeUserSimulator._warn_unsupported_perturbation(None) assert caplog.records == [] + + +def _make_bare_simulator() -> CascadeUserSimulator: + """Build a CascadeUserSimulator without running __init__, for pure _messages() testing.""" + sim = object.__new__(CascadeUserSimulator) + sim._build_prompt = lambda: "SYSTEM PROMPT" + sim._history = [] + return sim + + +def test_messages_flips_roles_so_the_caller_llm_sees_its_own_lines_as_assistant(): + sim = _make_bare_simulator() + sim._history = [ + {"role": "assistant", "content": "What is your email?"}, + {"role": "user", "content": "It's jane@example.com."}, + ] + + messages = sim._messages() + + assert messages[0]["role"] == "system" + assert messages[1] == {"role": "user", "content": "What is your email?"} + assert messages[2] == {"role": "assistant", "content": "It's jane@example.com."} + + +def test_messages_appends_a_trailing_role_reminder(): + sim = _make_bare_simulator() + + messages = sim._messages() + + assert messages[-1]["role"] == "system" + assert "CUSTOMER" in messages[-1]["content"] + assert "Do NOT respond as the customer service agent" in messages[-1]["content"] diff --git a/tests/unit/user_simulator/cascade/test_stt.py b/tests/unit/user_simulator/cascade/test_stt.py index 655fb03e..3d1d9386 100644 --- a/tests/unit/user_simulator/cascade/test_stt.py +++ b/tests/unit/user_simulator/cascade/test_stt.py @@ -2,6 +2,8 @@ import base64 import json +import pytest + from eva.user_simulator.cascade.stt import ScribeStreamingSTT, TranscriptBuffer @@ -149,3 +151,38 @@ async def test_recv_error_is_recorded_and_object_stays_usable(): await stt.feed(b"\x00\x00") assert fake.sent await stt.stop() + + +class ClosingFakeWebSocket(SuspendingFakeWebSocket): + """Fake websocket whose send() fails, as a server-side close does.""" + + async def send(self, raw: str) -> None: + raise ConnectionClosedError() + + +class ConnectionClosedError(Exception): + pass + + +async def test_feed_raises_and_marks_closed_when_the_socket_send_fails(): + stt = ScribeStreamingSTT({"api_key": "test-key"}) + fake = ClosingFakeWebSocket([]) + stt._ws = fake + stt._receive_task = asyncio.create_task(stt._receive_loop()) + + with pytest.raises(RuntimeError, match="Scribe session closed"): + await stt.feed(b"\x00\x00") + + assert stt._closed is True + await stt.stop() + + +async def test_feed_raises_immediately_once_closed_without_resending(): + stt, fake = await _make_stt([]) + stt._closed = True + + with pytest.raises(RuntimeError, match="Scribe session is closed"): + await stt.feed(b"\x00\x00") + + assert fake.sent == [] + await stt.stop() From 281dace6129d2d2271f38d664042a327959f1cfe Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Fri, 7 Aug 2026 00:12:12 -0400 Subject: [PATCH 23/65] add awaiting-reply gate to TickScheduler.may_take_turn Live runs showed the caller re-sending its turn nearly verbatim when the assistant's LLM+TTS round trip exceeded WAIT_TO_RESPOND_SELF_MS: both silence thresholds are satisfied a fixed time after the caller stops talking regardless of whether a reply ever arrived. Add a strict has-the-assistant-replied gate, set when the caller starts a turn and cleared only when the assistant produces speech, combined with (not replacing) the existing thresholds. Bump simulation_version to 2.0.12. --- src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/scheduler.py | 13 ++++++- .../user_simulator/cascade/test_scheduler.py | 35 ++++++++++++++++--- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index ca837ff7..d3dd8121 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.11" +simulation_version = "2.0.12" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/scheduler.py b/src/eva/user_simulator/cascade/scheduler.py index 95a3ba8f..231e5797 100644 --- a/src/eva/user_simulator/cascade/scheduler.py +++ b/src/eva/user_simulator/cascade/scheduler.py @@ -32,6 +32,7 @@ def __init__(self, adapter: Adapter, *, bytes_per_tick: int = BYTES_PER_TICK) -> self._ticks_since_assistant_speech = _NEVER_SPOKE self._ticks_since_caller_speech = _NEVER_SPOKE self._assistant_has_spoken = False + self._awaiting_reply = False @property def tick(self) -> int: @@ -58,8 +59,14 @@ def may_take_turn(self) -> bool: Gated on the assistant having spoken at least once: the assistant opens the call with a greeting, and without this the caller would talk over it on tick 0, since neither silence counter has anything to measure yet. + + Also gated on the assistant having replied since the caller's last turn: + the silence thresholds alone are satisfied a fixed time after the caller + stops talking regardless of whether a reply ever arrived, which lets the + caller repeat itself into a slow assistant. This gate is strict — there is + no impatience escape hatch here. """ - if not self._assistant_has_spoken: + if not self._assistant_has_spoken or self._awaiting_reply: return False return self._ticks_since_assistant_speech > ms_to_ticks( WAIT_TO_RESPOND_OTHER_MS @@ -80,6 +87,10 @@ async def run_tick(self) -> TickResult: 0 if result.has_assistant_speech else self._ticks_since_assistant_speech + 1 ) self._assistant_has_spoken = self._assistant_has_spoken or result.has_assistant_speech + if result.has_assistant_speech: + self._awaiting_reply = False + if outgoing: + self._awaiting_reply = True self._tick += 1 return result diff --git a/tests/unit/user_simulator/cascade/test_scheduler.py b/tests/unit/user_simulator/cascade/test_scheduler.py index 6e9b2ccf..9e7943d5 100644 --- a/tests/unit/user_simulator/cascade/test_scheduler.py +++ b/tests/unit/user_simulator/cascade/test_scheduler.py @@ -62,15 +62,18 @@ async def test_assistant_speech_resets_the_silence_counter(): async def test_caller_must_also_wait_out_its_own_silence_threshold(): - scheduler = _scheduler([True]) # assistant greets on tick 0 - await scheduler.run_tick() + scheduler = _scheduler([True, False, True] + [False] * 30) # greet, caller turn, quick reply, then silence + await scheduler.run_tick() # tick 0: assistant greets scheduler.enqueue_utterance(b"\x02" * BYTES_PER_TICK) - await scheduler.run_tick() # caller speaks this tick + await scheduler.run_tick() # tick 1: caller speaks this tick + assert scheduler.may_take_turn() is False + + await scheduler.run_tick() # tick 2: assistant replies, clearing the awaiting-reply gate assert scheduler.may_take_turn() is False # Assistant silence is satisfied almost immediately, self-silence needs 25 ticks. - for _ in range(25): + for _ in range(24): await scheduler.run_tick() assert scheduler.may_take_turn() is False @@ -88,6 +91,30 @@ async def test_caller_does_not_speak_before_the_assistant_has_greeted(): assert scheduler.may_take_turn() is False +async def test_caller_cannot_take_a_second_turn_while_awaiting_a_reply(): + scheduler = _scheduler([True]) # assistant greets on tick 0, then never speaks again + await scheduler.run_tick() + scheduler.enqueue_utterance(b"\x02" * BYTES_PER_TICK) + await scheduler.run_tick() # caller takes its turn + + for _ in range(100): + await scheduler.run_tick() + assert scheduler.may_take_turn() is False + + +async def test_caller_may_take_a_second_turn_once_the_assistant_replies(): + scheduler = _scheduler([True, False, True] + [False] * 30) + await scheduler.run_tick() # tick 0: assistant greets + scheduler.enqueue_utterance(b"\x02" * BYTES_PER_TICK) + await scheduler.run_tick() # tick 1: caller takes its turn + await scheduler.run_tick() # tick 2: assistant replies + + for _ in range(25): + await scheduler.run_tick() + + assert scheduler.may_take_turn() is True + + async def test_queued_utterance_drains_one_tick_at_a_time(): adapter = FakeAdapter([]) scheduler = TickScheduler(adapter, bytes_per_tick=BYTES_PER_TICK) From 5cb14a9524014efe0e0a358f86fa4432e2c20a1b Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Fri, 7 Aug 2026 00:26:02 -0400 Subject: [PATCH 24/65] Reconnect ScribeStreamingSTT on Scribe's max-session-duration close Scribe closes cleanly (code 1000) around 397s of continuous audio, which would otherwise truncate long conversations. Detect ConnectionClosedOK in the receive loop and transparently reconnect, preserving committed transcript and dropping only the in-flight partial. Caps consecutive clean-close reconnects to distinguish this from a genuine hard failure (auth, quota, repeated immediate closes), which still fails loudly. Co-Authored-By: Claude Opus 5 (1M context) --- src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/stt.py | 48 +++++++++++++- tests/unit/user_simulator/cascade/test_stt.py | 62 ++++++++++++++++++- 3 files changed, 107 insertions(+), 5 deletions(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index d3dd8121..73921c91 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.12" +simulation_version = "2.0.13" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/stt.py b/src/eva/user_simulator/cascade/stt.py index eb2b26bc..9a75b21b 100644 --- a/src/eva/user_simulator/cascade/stt.py +++ b/src/eva/user_simulator/cascade/stt.py @@ -9,6 +9,7 @@ from typing import Any import websockets +from websockets.exceptions import ConnectionClosedOK from eva.user_simulator.cascade.constants import CALLER_SAMPLE_RATE from eva.utils.logging import get_logger @@ -19,6 +20,9 @@ INCOMPLETE_MARKER = "[CURRENTLY SPEAKING, INCOMPLETE]" +MAX_RECONNECT_ATTEMPTS = 3 +"""Consecutive clean closes tolerated before giving up; resets on any message received.""" + class TranscriptBuffer: """Accumulates committed transcript segments separately from the in-flight partial.""" @@ -67,17 +71,22 @@ def __init__(self, params: dict[str, Any], *, language: str = "en") -> None: self._receive_task: asyncio.Task | None = None self._error: Exception | None = None self._closed = False + self._reconnect_attempts = 0 async def start(self) -> None: """Open the transcription socket and begin consuming results.""" - self._ws = await websockets.connect( + self._ws = await self._connect() + self._receive_task = asyncio.create_task(self._receive_loop()) + + async def _connect(self) -> Any: + """Open a fresh Scribe socket with this instance's configuration.""" + return await websockets.connect( f"{SCRIBE_URL}?model_id={self._model}" f"&audio_format=pcm_{CALLER_SAMPLE_RATE}" f"&language_code={self._language}" "&commit_strategy=manual", additional_headers={"xi-api-key": self._api_key}, ) - self._receive_task = asyncio.create_task(self._receive_loop()) async def feed(self, pcm: bytes, *, commit: bool = False) -> None: """Send one tick of assistant audio, optionally closing the utterance. @@ -97,6 +106,8 @@ async def feed(self, pcm: bytes, *, commit: bool = False) -> None: message["commit"] = True try: await self._ws.send(json.dumps(message)) + except ConnectionClosedOK: + logger.warning("Scribe socket closed mid-send; this tick's audio was dropped, reconnect in progress") except Exception as exc: self._closed = True logger.error(f"Scribe session closed unexpectedly; caller is now deaf: {exc}") @@ -114,14 +125,19 @@ async def stop(self) -> None: self._ws = None async def _receive_loop(self) -> None: - """Fold partial and committed transcripts into the buffer as they arrive.""" + """Fold partial and committed transcripts into the buffer as they arrive, reconnecting on a clean close.""" while True: try: raw = await self._ws.recv() + except ConnectionClosedOK as exc: + if await self._reconnect_after_clean_close(exc): + continue + return except Exception as exc: self._error = exc logger.exception("Scribe receive loop failed") return + self._reconnect_attempts = 0 try: message = json.loads(raw) except json.JSONDecodeError: @@ -134,3 +150,29 @@ async def _receive_loop(self) -> None: self.buffer.apply_partial(text) elif kind.startswith("committed_transcript") or kind.startswith("final_transcript"): self.buffer.commit(text) + + async def _reconnect_after_clean_close(self, exc: ConnectionClosedOK) -> bool: + """Reopen the socket after a server-initiated clean close, preserving committed transcript. + + Returns False once reconnect attempts are exhausted, at which point the caller must stop. + """ + self._reconnect_attempts += 1 + if self._reconnect_attempts > MAX_RECONNECT_ATTEMPTS: + self._error = exc + self._closed = True + logger.error(f"Scribe closed cleanly {self._reconnect_attempts} times in a row; giving up") + return False + self.buffer.in_flight = "" + try: + self._ws = await self._connect() + except Exception as reconnect_exc: + self._error = reconnect_exc + self._closed = True + logger.exception("Scribe reconnect failed") + return False + logger.info( + "Scribe session closed cleanly (code 1000, likely a max-session-duration cap); " + f"reconnected (attempt {self._reconnect_attempts}/{MAX_RECONNECT_ATTEMPTS}); " + "committed transcript preserved, in-flight partial dropped" + ) + return True diff --git a/tests/unit/user_simulator/cascade/test_stt.py b/tests/unit/user_simulator/cascade/test_stt.py index 3d1d9386..a842ee6c 100644 --- a/tests/unit/user_simulator/cascade/test_stt.py +++ b/tests/unit/user_simulator/cascade/test_stt.py @@ -3,8 +3,9 @@ import json import pytest +from websockets.exceptions import ConnectionClosedOK -from eva.user_simulator.cascade.stt import ScribeStreamingSTT, TranscriptBuffer +from eva.user_simulator.cascade.stt import MAX_RECONNECT_ATTEMPTS, ScribeStreamingSTT, TranscriptBuffer def test_partial_updates_replace_the_in_flight_text(): @@ -186,3 +187,62 @@ async def test_feed_raises_immediately_once_closed_without_resending(): assert fake.sent == [] await stt.stop() + + +def _connect_stub(sockets): + remaining = list(sockets) + + async def _connect(): + return remaining.pop(0) + + return _connect + + +async def test_reconnects_after_a_clean_close_and_keeps_transcribing(): + stt = ScribeStreamingSTT({"api_key": "test-key"}) + stt.buffer.commit("Heard before the close.") + stt._ws = SuspendingFakeWebSocket([ConnectionClosedOK(None, None)]) + second_socket = SuspendingFakeWebSocket( + [{"message_type": "committed_transcript", "text": "Heard after reconnecting."}] + ) + stt._connect = _connect_stub([second_socket]) + stt._receive_task = asyncio.create_task(stt._receive_loop()) + + await asyncio.sleep(0.05) + + assert stt.buffer.committed == "Heard before the close. Heard after reconnecting." + assert stt._ws is second_socket + assert stt._error is None + assert stt._closed is False + await stt.stop() + + +async def test_reconnect_drops_in_flight_partial_but_keeps_committed_text(): + stt = ScribeStreamingSTT({"api_key": "test-key"}) + stt.buffer.commit("Already committed.") + stt.buffer.apply_partial("mid-utterance when the close happened") + stt._ws = SuspendingFakeWebSocket([ConnectionClosedOK(None, None)]) + stt._connect = _connect_stub([SuspendingFakeWebSocket([])]) + stt._receive_task = asyncio.create_task(stt._receive_loop()) + + await asyncio.sleep(0.05) + + assert stt.buffer.committed == "Already committed." + assert stt.buffer.in_flight == "" + await stt.stop() + + +async def test_persistent_clean_closes_exhaust_the_cap_and_surface_an_error(): + stt = ScribeStreamingSTT({"api_key": "test-key"}) + sockets = [SuspendingFakeWebSocket([ConnectionClosedOK(None, None)]) for _ in range(MAX_RECONNECT_ATTEMPTS + 1)] + stt._ws = sockets[0] + stt._connect = _connect_stub(sockets[1:]) + stt._receive_task = asyncio.create_task(stt._receive_loop()) + + await asyncio.sleep(0.05) + + assert isinstance(stt._error, ConnectionClosedOK) + assert stt._closed is True + with pytest.raises(RuntimeError, match="Scribe session is closed"): + await stt.feed(b"\x00\x00") + await stt.stop() From 39e9e4b1361a6891be218502c87f1c37e5801d67 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Fri, 7 Aug 2026 00:52:16 -0400 Subject: [PATCH 25/65] point cascade caller at a dedicated low-reasoning-effort deployment Default CascadeSimulatorConfig.llm to "user-llm" instead of "gpt-5.5" so the caller's model/params can differ from the assistant under test without touching the shared gpt-5.5 deployment (preserves comparability with the ElevenLabs baseline). Drop llm_params, which was never wired to anything. --- .env.example | 9 +++++++++ src/eva/models/config.py | 11 +++++++---- tests/unit/models/test_config_models.py | 2 +- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.env.example b/.env.example index 38c493c8..de870400 100644 --- a/.env.example +++ b/.env.example @@ -183,6 +183,15 @@ EVA_MODEL_LIST='[ "aws_secret_access_key": "os.environ/AWS_SECRET_ACCESS_KEY", "max_parallel_requests": 5 } + }, + { + "model_name": "user-llm", + "litellm_params": { + "model": "openai/gpt-5.5", + "api_key": "os.environ/OPENAI_API_KEY", + "max_parallel_requests": 5, + "reasoning_effort": "low" + } } ]' diff --git a/src/eva/models/config.py b/src/eva/models/config.py index a4bbafdd..d094a419 100644 --- a/src/eva/models/config.py +++ b/src/eva/models/config.py @@ -483,10 +483,13 @@ class CascadeSimulatorConfig(BaseModel): description="Provider-native keyword arguments passed through to the STT client.", ) - llm: str = Field("gpt-5.5", description="Caller LLM, resolved via the same EVA_MODEL_LIST router as the assistant.") - llm_params: dict[str, Any] = Field( - default_factory=dict, - description="Provider-native keyword arguments passed through to the caller LLM client.", + llm: str = Field( + "user-llm", + description=( + "Caller LLM, resolved via the same EVA_MODEL_LIST router as the assistant. Defaults to a " + "deployment named 'user-llm' so the caller's model/params (e.g. reasoning_effort) can differ " + "from whatever the assistant under test uses." + ), ) tts: str = Field("cartesia", description="TTS provider for the simulated caller's speech.") diff --git a/tests/unit/models/test_config_models.py b/tests/unit/models/test_config_models.py index 5ddeb376..8fceec3f 100644 --- a/tests/unit/models/test_config_models.py +++ b/tests/unit/models/test_config_models.py @@ -1259,7 +1259,7 @@ def test_cascade_simulator_config_defaults(): assert config.stt_params["model"] == "scribe_v2_realtime" assert config.tts == "cartesia" assert config.tts_params["model"] == "sonic-3.5" - assert config.llm == "gpt-5.5" + assert config.llm == "user-llm" def test_user_simulator_union_discriminates_cascade(): From 713ee14a2332f369f0874b5750551b0f20fde438 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Fri, 7 Aug 2026 14:32:46 -0400 Subject: [PATCH 26/65] Replace the raw Scribe socket with a LiveKit STT plugin and stop stale repeats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cascade simulator regenerated its turn from unchanged history whenever a transcript failed to arrive, producing verbatim repeats the assistant noticed and complained about. Reading the buffer was optimistic: no wait, no acknowledgement that the commit landed, no fallback, no check that anything was heard at all. _collect_heard_text now waits a bounded number of ticks for the final, falls back to the in-flight partial, and — if nothing arrived — tells the model it missed the utterance so it asks for a repeat instead of re-prompting on identical history. Both outcomes are logged as events so transcript loss can be measured per provider. STT moves to the LiveKit ElevenLabs plugin used standalone (no room). Omitting server_vad selects commit_strategy=manual, so the tick scheduler still owns the turn boundary, and flush() replaces our hand-rolled commit flag; measured flush->final is 0.15s. This deletes the reconnect and idle-close machinery ScribeStreamingSTT needed. Also drops the cascade-specific turn-contract prompt. It demanded "a single JSON object and nothing else", which suppressed the end_call tool call entirely and left conversations hanging until the assistant server's idle timeout. The per-domain user_simulator prompt already covers end_call, and the tool description is now shared with the OpenAI Realtime provider. Co-Authored-By: Claude Opus 5 (1M context) --- configs/prompts/simulation.yaml | 18 - docs/changelog_cascade_stt_reliability.md | 163 +++++++ pyproject.toml | 2 + src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/constants.py | 12 +- src/eva/user_simulator/cascade/scheduler.py | 10 +- src/eva/user_simulator/cascade/simulator.py | 131 +++-- src/eva/user_simulator/cascade/stt.py | 154 +----- src/eva/user_simulator/cascade/stt_livekit.py | 117 +++++ .../user_simulator/cascade/test_constants.py | 4 - .../user_simulator/cascade/test_prompt.py | 17 +- .../user_simulator/cascade/test_scheduler.py | 24 + .../user_simulator/cascade/test_simulator.py | 107 +++- tests/unit/user_simulator/cascade/test_stt.py | 212 +------- .../cascade/test_stt_livekit.py | 43 ++ uv.lock | 458 +++++++++++++++++- 16 files changed, 992 insertions(+), 482 deletions(-) create mode 100644 docs/changelog_cascade_stt_reliability.md create mode 100644 src/eva/user_simulator/cascade/stt_livekit.py create mode 100644 tests/unit/user_simulator/cascade/test_stt_livekit.py diff --git a/configs/prompts/simulation.yaml b/configs/prompts/simulation.yaml index d6ad017a..4648cb62 100644 --- a/configs/prompts/simulation.yaml +++ b/configs/prompts/simulation.yaml @@ -507,21 +507,3 @@ user_simulator: For languages that use non-Latin scripts, spell out characters using their standard phonetic names in your language. IMPORTANT: Before ending the conversation, confirm with the agent that there are no outstanding actions. The end_call tool should only be called in a turn that is a brief goodbye — never in the same turn where you are providing the agent with data, an identifier, a request to transfer to a live agent, an approval to proceed, or any kind of additional information. - - cascade_turn_contract: | - Respond with a single JSON object and nothing else. It must have exactly this field: - - {{ - "utterance": "" - }} - - Rules for "utterance": - - Write it the way a person speaks on a phone call, not the way they write. - - One turn only. Do not script the agent's reply or your own next turn. - - Never include stage directions, names, labels, or quotation marks around the whole line. - - If you have nothing to add but the call is not over, say a short natural acknowledgement. - - To hang up, call the end_call tool. Do not describe hanging up in "utterance". - - cascade_role_reminder: | - REMINDER: You are the CUSTOMER calling for help. Respond as the customer would - with questions, requests, or information about your issue. Do NOT respond as the customer service agent. diff --git a/docs/changelog_cascade_stt_reliability.md b/docs/changelog_cascade_stt_reliability.md new file mode 100644 index 00000000..3ba9f789 --- /dev/null +++ b/docs/changelog_cascade_stt_reliability.md @@ -0,0 +1,163 @@ +# Cascade STT reliability and provider abstraction + +Working log for the fix to Defect B (verbatim caller repeats) and the LiveKit STT port. +Reasoning and certainty are recorded per change. + +## Background: what actually broke + +Live run `cascade-repro-1` produced three verbatim repeats of the same caller utterance. +The assistant noticed: *"I'm hearing the same phrase repeated, so the line may be +transcribing you incorrectly."* + +Evidence chain: + +- `user_simulator_events.jsonl` shows three consecutive `caller_turn` events (ticks 312, + 390, 491) with **no `assistant_speech` between them**, while `transcript.jsonl` proves + the assistant spoke twice in that window, 14-20s before each repeat. +- The Scribe reconnect at 12:01:47 happened **after** all three repeats, so it is not the cause. +- The next successful commit contained **only** the 6th assistant utterance, with no trace + of the two missing ones. A merely slow commit would have landed in `committed` and + appeared prepended on the next `take_committed()`. It did not. So those utterances were + **never committed at all** - this is not a read-too-early race. + +Mechanism of the repeat itself is confirmed at `simulator.py:179`: `_take_turn` reads +`take_committed()`, and when it returns empty it still calls the caller LLM with unchanged +history. Same messages in, same utterance out. + +## The reframing that drives this work + +Provider-side turn detection was initially blamed. That was wrong, and the correction +matters for provider choice: + +- `TickScheduler.may_take_turn()` decides when the caller speaks, from counting silent + ticks of assistant **audio**. STT never participates. So provider endpointing cannot + corrupt who-speaks-when. +- We already run `commit_strategy=manual` (no provider VAD) and hit the failure anyway. + +The real root cause is upstream of any provider: **we read the transcript buffer +optimistically - no wait, no acknowledgement that the commit was processed, no fallback, +and no check that we heard anything at all.** + +Three failure shapes at read time: + +| State at read | Consequence | +|---|---| +| Nothing committed | Stale history -> verbatim repeat (Defect B) | +| Partially committed | Caller replies to half a sentence, silently. **Most insidious** | +| Fully committed | Correct | + +## Changes + +### 1. Bounded wait for the committed transcript + +**Certainty: high.** Implemented by *skipping* the turn and retrying on later ticks rather +than a blocking sleep - which fits the tick architecture and costs nothing, since each +retry just re-reads the buffer. Bounded by a tick counter so it cannot wait forever. + +Latency budget is ample: the caller's own LLM call takes ~10s observed, so absorbing a few +hundred ms of STT finalization is free. + +### 2. Fall back to the in-flight partial + +**Certainty: medium-high.** Approximate text beats stale text by a wide margin. Precedent: +tau-voice drives its interrupt/backchannel decisions off a linearly interpolated, +mid-word-truncating approximation (`get_proportional_text`, transcript_utils.py:7-25) and +that is good enough to have shipped. + +Open question: whether partials were actually flowing in the failing run. `commit()` clears +`in_flight`, and reconnect clears it too. Instrumentation now logs `in_flight` on this exact +path, but the defect has not recurred in the runs since, so this is **unmeasured**. + +### 3. Never generate a turn from unchanged history + +**Certainty: high on the rule, medium on the remedy.** The rule - never call the caller LLM +with history that did not change - is unambiguous. + +For the remedy, three options were considered: + +- *Stall until text arrives* - converts a corrupt-transcript bug into a dead-air bug. +- *Speak anyway* - the current behavior, i.e. the defect. +- **Chosen: tell the caller it did not hear.** Inject an explicit note so it says "Sorry, I + didn't catch that." This is what a real caller does, keeps the conversation alive, records + the failure honestly in the transcript, and can never emit a stale repeat. + +### 4-6. Provider-agnostic STT interface, LiveKit implementation, config + loss metric + +**Certainty: medium.** Motivation is that LiveKit's base `RecognizeStream` provides +`flush()`/`end_input()` (livekit-agents `stt.py:349-571`) - a flush sentinel *in the stream* +rather than our out-of-band `{"commit": true}` flag - plus typed events and a uniform +interface across 28 providers. Adopting it deletes our hand-rolled reconnect/idle-close +machinery. + +Caveats recorded before committing: + +- **Dependency cost is the real risk.** Any LiveKit STT plugin pulls `livekit-agents` -> + pinned `livekit==1.1.14`, a compiled native Rust/WebRTC wheel, plus `av` (FFmpeg + bindings), `sounddevice` and OpenTelemetry - for a codepath that never opens a room. + The `.venv.x86-broken/` directory in the tree makes this a concrete, not theoretical, risk. +- **Swappability is narrower than the catalogue suggests.** The uniform interface does not + expose whether provider turn detection can be disabled. That varies per plugin and is only + visible in each plugin's source. +- `ink-2` has **mandatory** turn detection (only `turn_start_threshold`, + `turn_eager_end_threshold`, `turn_end_threshold`, `turn_end_timeout_ms`; no off switch) + and is **English-only** per its docstring. `ink-whisper` has no interim results. + Neither Cartesia model satisfies both requirements today. + +The loss metric in change 6 exists so provider comparison is settled with data rather than +anecdote. + +## LiveKit spike results (measured, not inferred) + +Scratch venv, `livekit-agents` 1.6.8 + 4 plugins. One recorded 14s assistant utterance fed at +our 200ms tick cadence, then `flush()`. + +| Provider | interims (R2) | `flush()` honored | auto-final | verdict | +|---|---|---|---|---| +| `elevenlabs/scribe_v2_realtime` | 14 | **yes, 0.15s** | none (endpointing off) | full manual control | +| `cartesia/ink-2` | 42 | **no - explicitly ignored** | +1.20s after speech end | provider-driven only | +| `deepgram/flux-general-en` | - | - | - | blocked: WS handshake **HTTP 402** | +| `deepgram/nova-3` | - | - | - | same closure, same account | +| `assemblyai/universal-streaming-multilingual` | - | - | - | no key in `.env` | + +Cartesia logs it outright: `Cartesia STT stream.flush() was ignored.` + +**The decisive number: ink-2 auto-finalizes +1.20s after speech ends, while +`WAIT_TO_RESPOND_OTHER_MS` lets the caller take its turn at 1.0s.** So the final lands ~200ms +*after* we read the buffer - the late-final case, by a small margin. A bounded wait of ~500ms +covers it comfortably. This makes ink-2 viable *provided* changes 1-3 land first. + +**This also settles Defect B.** Scribe answers a commit in 150ms, so Defect B is an +intermittent dropout, not systematic slowness. That was the top falsification test - and it +says fix the race in place rather than redesign the transport. Changes 1-3 do exactly that +and are provider-independent. + +Integration details learned (needed by the real implementation): + +- Standalone use requires `async with livekit.agents.utils.http_context.open():` or an + explicitly passed `aiohttp.ClientSession`; plugins otherwise raise "http session outside of + a job context". +- Deepgram Flux is `deepgram.STTv2`, not `deepgram.STT` (which is the nova path). +- Install on arm64/py3.12: 74 packages, 268MB, all prebuilt wheels, no compilation. x86 CI + still unverified. + +## STT requirements these changes must preserve + +1. The caller owns the turn boundary; STT reports what was said. +2. In-flight partials for Plan 2's interrupt/backchannel checks (only when those are on). +3. Audio-only - we are a client on a Twilio WebSocket, with no access to the assistant's text. +4. A self-hostable option (NVIDIA Riva/NIM is first-party and runs offline). +5. No silent transcript loss. Violated by Defect B; the reason for changes 1-3. +6. Non-English support - the simulator takes a `language` param. + +## Related fixes landed alongside + +- **Deadlock (separate defect, 2/7 runs).** Caller said goodbye, assistant treated it as + terminal, `_awaiting_reply` never cleared, caller could never reach its `end_call` turn, + run stalled 5 minutes to pipecat's idle timeout. Fixed with `ASSISTANT_UNRESPONSIVE_MS` + (90s) in `may_take_turn()`, plus removal of contradictions in `END_CALL_DESCRIPTION`. + Threshold chosen from measurement: longest *legitimate* assistant gap observed live was + 220 ticks (44s), so 25s would have misfired mid-conversation in 2 of 5 runs. +- **Swallowed Scribe errors.** The receive loop dropped any message lacking a `text` field, + which includes error messages. Now logged explicitly. +- **Diagnostic instrumentation** for the commit boundary: per-commit audio fed vs non-silent + seconds, every Scribe message type, and a warning when a turn is taken having heard nothing. diff --git a/pyproject.toml b/pyproject.toml index f7d79a95..46e1f8ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,8 @@ dependencies = [ "azure-cognitiveservices-speech>=1.31.0", "cartesia>=1.0.0", "assemblyai>=0.17.0", + "livekit-agents>=1.6.8", + "livekit-plugins-elevenlabs>=1.6.8", "setuptools>=65.0.0", "fastapi>=0.100.0", "uvicorn>=0.23.0", diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 73921c91..ee46a22c 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.13" +simulation_version = "2.0.14" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/constants.py b/src/eva/user_simulator/cascade/constants.py index d68513b6..b940e81d 100644 --- a/src/eva/user_simulator/cascade/constants.py +++ b/src/eva/user_simulator/cascade/constants.py @@ -1,4 +1,4 @@ -"""Timing constants for the tick-based cascade user simulator. +"""Timing constants for the tick-based cascade caller. Values mirror tau-voice (tau2-bench/src/tau2/config.py:104-113). These are module constants rather than config fields on purpose: there is no run-level @@ -16,11 +16,13 @@ WAIT_TO_RESPOND_SELF_MS = 5000 """Silence required from the caller itself before it starts another turn.""" -YIELD_WHEN_INTERRUPTED_MS = 1000 -"""How long the caller keeps talking after the assistant barges in.""" +TRANSCRIPT_WAIT_MS = 1500 +"""How long the caller waits for the assistant's transcript to finalize before falling back +to the in-flight partial, sized above the slowest measured finalization (ink-2, 1.2s).""" -YIELD_WHEN_INTERRUPTING_MS = 5000 -"""How long the caller holds the floor after barging in itself.""" +ASSISTANT_UNRESPONSIVE_MS = 90000 +"""Assistant silence after which the caller stops waiting for a reply, set above the longest +legitimate inter-utterance gap measured live (220 ticks) and below the server's idle timeout.""" CALLER_SAMPLE_RATE = 16000 """PCM16 sample rate for the caller's own audio track.""" diff --git a/src/eva/user_simulator/cascade/scheduler.py b/src/eva/user_simulator/cascade/scheduler.py index 231e5797..c1d5bf2d 100644 --- a/src/eva/user_simulator/cascade/scheduler.py +++ b/src/eva/user_simulator/cascade/scheduler.py @@ -4,6 +4,7 @@ from eva.user_simulator.cascade.adapter.base import Adapter from eva.user_simulator.cascade.constants import ( + ASSISTANT_UNRESPONSIVE_MS, BYTES_PER_TICK, WAIT_TO_RESPOND_OTHER_MS, WAIT_TO_RESPOND_SELF_MS, @@ -63,10 +64,13 @@ def may_take_turn(self) -> bool: Also gated on the assistant having replied since the caller's last turn: the silence thresholds alone are satisfied a fixed time after the caller stops talking regardless of whether a reply ever arrived, which lets the - caller repeat itself into a slow assistant. This gate is strict — there is - no impatience escape hatch here. + caller repeat itself into a slow assistant. That gate releases only after + ASSISTANT_UNRESPONSIVE_MS, so an assistant that stops answering entirely + cannot strand the caller before it reaches its end_call turn. """ - if not self._assistant_has_spoken or self._awaiting_reply: + if not self._assistant_has_spoken: + return False + if self._awaiting_reply and self._ticks_since_assistant_speech <= ms_to_ticks(ASSISTANT_UNRESPONSIVE_MS): return False return self._ticks_since_assistant_speech > ms_to_ticks( WAIT_TO_RESPOND_OTHER_MS diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py index e47926ff..9d76c11c 100644 --- a/src/eva/user_simulator/cascade/simulator.py +++ b/src/eva/user_simulator/cascade/simulator.py @@ -1,8 +1,7 @@ -"""Self-hosted STT/LLM/TTS caller simulator driven by the tick scheduler.""" +"""Self-hosted STT/LLM/TTS caller driven by the tick scheduler.""" from __future__ import annotations -import json import re from pathlib import Path @@ -12,29 +11,27 @@ from eva.models.config import CascadeSimulatorConfig, PerturbationConfig from eva.user_simulator.base import AbstractUserSimulator from eva.user_simulator.cascade.adapter.realtime_ws import RealtimeWSAdapter -from eva.user_simulator.cascade.constants import CALLER_SAMPLE_RATE, TICK_DURATION_MS +from eva.user_simulator.cascade.constants import ( + TICK_DURATION_MS, + TRANSCRIPT_WAIT_MS, + CALLER_SAMPLE_RATE, + ms_to_ticks, +) from eva.user_simulator.cascade.scheduler import TickScheduler -from eva.user_simulator.cascade.stt import ScribeStreamingSTT +from eva.user_simulator.cascade.stt_livekit import LiveKitStreamingSTT from eva.user_simulator.cascade.tts import CartesiaTTS + +# Shared with the OpenAI Realtime provider so both simulators hang up on the same rules. +from eva.user_simulator.openai_realtime import END_CALL_DESCRIPTION from eva.utils.logging import get_logger -from eva.utils.prompt_manager import PromptManager logger = get_logger(__name__) -_FENCE = re.compile(r"^```(?:json)?\s*|\s*```$", re.MULTILINE) - -END_CALL_DESCRIPTION = """Use this to end the phone call and hang up. - -Call this function when it is time to end the call and one of the following is true: -1. The agent has confirmed your request is resolved, all steps are completed, and you have said goodbye. -2. The agent has initiated a transfer to a live agent. -3. The agent has been unable to make progress for at least 5 consecutive turns. -4. The agent says goodbye or indicates the conversation is over. -5. The agent indicates that the remainder of your request cannot be fulfilled. -6. The assistant reports an unrecoverable processing error. +_FENCE = re.compile(r"^```[a-z]*\s*|\s*```$", re.MULTILINE) -Never call this tool in the same turn that you provide the agent with data, an identifier, -an approval to proceed, a transfer request, or any other information. Say a brief goodbye first.""" +MISSED_UTTERANCE_DIRECTIVE = """You did not hear what the agent just said — the audio did not +come through. Do NOT repeat your previous message. Say briefly that you did not catch that and +ask them to repeat it, the way anyone would on a bad phone line.""" END_CALL_TOOL = { "type": "function", @@ -47,24 +44,8 @@ def parse_turn_response(raw: str) -> str: - """Extract the utterance from the caller LLM's JSON reply. - - Falls back to treating the whole response as the utterance when it is not - JSON, so a malformed reply degrades into a plain turn rather than silence. - """ - stripped = _FENCE.sub("", raw).strip() - if not stripped: - return "" - try: - payload = json.loads(stripped) - except json.JSONDecodeError: - return stripped - if not isinstance(payload, dict): - return stripped - utterance = payload.get("utterance", "") - if not isinstance(utterance, str): - raise ValueError(f"Caller LLM returned a non-string utterance: {utterance!r}") - return utterance + """Return the spoken line from the model's reply, stripping any stray code fence.""" + return _FENCE.sub("", raw).strip() def _flip_role(role: str) -> str: @@ -117,11 +98,13 @@ def __init__( ) self._warn_unsupported_perturbation(perturbation_config) self._config = simulator_config - self._stt = ScribeStreamingSTT(simulator_config.stt_params, language=language) + self._stt = LiveKitStreamingSTT(simulator_config.stt, simulator_config.stt_params, language=language) self._tts = CartesiaTTS(simulator_config.tts_params, language=language) self._llm = LiteLLMClient(model=simulator_config.llm) self._voice_id = self._tts.voice_for_persona(persona_config) self._history: list[dict[str, str]] = [] + self._ticks_awaiting_transcript = 0 + self._missed_transcripts = 0 async def run_conversation(self) -> str: """Run the tick loop until the call ends, and return the end reason.""" @@ -158,13 +141,22 @@ async def _run(self) -> None: # and never idles out; committed exactly on the speech->silence transition, # which is what closes the utterance so take_committed() below isn't starved. commit = assistant_was_speaking and not result.has_assistant_speech + if result.has_assistant_speech != assistant_was_speaking: + logger.debug( + f"tick {scheduler.tick}: assistant speech " + f"{'started' if result.has_assistant_speech else 'ended'} " + f"(raw={result.assistant_audio_raw_bytes}B)" + ) await self._stt.feed(result.assistant_audio, commit=commit) assistant_was_speaking = result.has_assistant_speech if result.has_assistant_speech: continue if scheduler.caller_is_speaking or not scheduler.may_take_turn(): continue - if await self._take_turn(scheduler): + heard, waiting = self._collect_heard_text(scheduler) + if waiting: + continue + if await self._take_turn(scheduler, heard): break else: if not self._conversation_done.is_set(): @@ -174,14 +166,51 @@ async def _run(self) -> None: await adapter.stop() self.event_logger.log_connection_state("session_ended", {"reason": self._end_reason}) - async def _take_turn(self, scheduler: TickScheduler) -> bool: - """Generate, synthesize, and queue one caller turn. Returns True to hang up.""" + def _collect_heard_text(self, scheduler: TickScheduler) -> tuple[str, bool]: + """Return what the assistant said and whether to keep waiting for it. + + Finalization is not instantaneous, so an empty buffer at the first turn + opportunity usually means "not ready yet" rather than "nothing was said". + Retrying on later ticks is the wait; the in-flight partial is the fallback + once that budget is spent. + """ heard = self._stt.buffer.take_committed() + if heard: + self._ticks_awaiting_transcript = 0 + return heard, False + + self._ticks_awaiting_transcript += 1 + if self._ticks_awaiting_transcript <= ms_to_ticks(TRANSCRIPT_WAIT_MS): + return "", True + + self._ticks_awaiting_transcript = 0 + partial = self._stt.buffer.in_flight + self._stt.buffer.in_flight = "" + if partial: + logger.warning( + f"tick {scheduler.tick}: no final transcript after {TRANSCRIPT_WAIT_MS}ms; " + f"falling back to the in-flight partial: {partial[:120]!r}" + ) + self.event_logger.log_event("transcript_partial_fallback", {"text": partial, "tick_index": scheduler.tick}) + return partial, False + + self._missed_transcripts += 1 + logger.error( + f"tick {scheduler.tick}: heard nothing at all from the assistant this turn " + f"(missed {self._missed_transcripts} so far); asking it to repeat" + ) + self.event_logger.log_event("transcript_missed", {"tick_index": scheduler.tick}) + return "", False + + async def _take_turn(self, scheduler: TickScheduler, heard: str) -> bool: + """Generate, synthesize, and queue one caller turn. Returns True to hang up.""" if heard: self._history.append({"role": "assistant", "content": heard}) self._on_assistant_speaks(heard) - message, _stats = await self._llm.complete(messages=self._messages(), tools=[END_CALL_TOOL]) + message, _stats = await self._llm.complete( + messages=self._messages(missed_utterance=not heard), tools=[END_CALL_TOOL] + ) utterance, end_call = extract_turn(message) if utterance: @@ -219,15 +248,21 @@ def _warn_unsupported_perturbation(perturbation_config: PerturbationConfig | Non "Behavior and accent perturbations are unaffected." ) - def _messages(self) -> list[dict[str, str]]: - """Build the caller LLM message list: persona/goal prompt, JSON contract, flipped history. + def _messages(self, *, missed_utterance: bool = False) -> list[dict[str, str]]: + """Build the message list: the shared per-domain caller prompt plus flipped history. + + The system prompt is `_build_prompt()` unmodified — the same per-domain prompt the other + providers use, which already carries the persona, goal, and end_call rules. `self._history` is kept in conversation-truth roles (assistant said by the agent, user - said by the caller) since it also feeds logging. The caller LLM is itself the assistant + said by the caller) since it also feeds logging. This LLM is itself the assistant in its own frame, so that history must be flipped here or a message tagged "assistant" reads to the model as its own prior output and it echoes it back. """ - system = self._build_prompt() + "\n\n" + PromptManager().get_prompt("user_simulator.cascade_turn_contract") - flipped = [{"role": _flip_role(turn["role"]), "content": turn["content"]} for turn in self._history] - reminder = PromptManager().get_prompt("user_simulator.cascade_role_reminder") - return [{"role": "system", "content": system}, *flipped, {"role": "system", "content": reminder}] + messages = [{"role": "system", "content": self._build_prompt()}] + messages += [{"role": _flip_role(turn["role"]), "content": turn["content"]} for turn in self._history] + if missed_utterance: + # History is unchanged since the last turn, so without this the model would + # regenerate its previous utterance verbatim. + messages.append({"role": "system", "content": MISSED_UTTERANCE_DIRECTIVE}) + return messages diff --git a/src/eva/user_simulator/cascade/stt.py b/src/eva/user_simulator/cascade/stt.py index 9a75b21b..bf11e4de 100644 --- a/src/eva/user_simulator/cascade/stt.py +++ b/src/eva/user_simulator/cascade/stt.py @@ -1,28 +1,7 @@ -"""Streaming speech-to-text client that transcribes the assistant's audio for the caller.""" +"""Transcript accumulation for the caller's speech-to-text.""" from __future__ import annotations -import asyncio -import base64 -import json -import os -from typing import Any - -import websockets -from websockets.exceptions import ConnectionClosedOK - -from eva.user_simulator.cascade.constants import CALLER_SAMPLE_RATE -from eva.utils.logging import get_logger - -logger = get_logger(__name__) - -SCRIBE_URL = "wss://api.elevenlabs.io/v1/speech-to-text/realtime" - -INCOMPLETE_MARKER = "[CURRENTLY SPEAKING, INCOMPLETE]" - -MAX_RECONNECT_ATTEMPTS = 3 -"""Consecutive clean closes tolerated before giving up; resets on any message received.""" - class TranscriptBuffer: """Accumulates committed transcript segments separately from the in-flight partial.""" @@ -40,139 +19,8 @@ def commit(self, text: str) -> None: self.committed = f"{self.committed} {text}".strip() if self.committed else text self.in_flight = "" - def current_text(self) -> str: - """Render everything heard so far, marking an in-flight utterance as incomplete.""" - if not self.in_flight: - return self.committed - prefix = f"{self.committed} " if self.committed else "" - return f"{prefix}{self.in_flight} {INCOMPLETE_MARKER}" - def take_committed(self) -> str: """Return and clear the committed text.""" text = self.committed self.committed = "" return text - - -class ScribeStreamingSTT: - """Streams PCM16 to Scribe and folds results into a TranscriptBuffer. - - Uses commit_strategy=manual so the caller's own turn-end decision drives - commits; Scribe's built-in VAD would be a second, independent detector on its - own timing, which is the problem the tick design exists to remove. - """ - - def __init__(self, params: dict[str, Any], *, language: str = "en") -> None: - self._model = params.get("model", "scribe_v2_realtime") - self._api_key = params.get("api_key") or os.environ.get("ELEVENLABS_API_KEY", "") - self._language = language - self.buffer = TranscriptBuffer() - self._ws: Any = None - self._receive_task: asyncio.Task | None = None - self._error: Exception | None = None - self._closed = False - self._reconnect_attempts = 0 - - async def start(self) -> None: - """Open the transcription socket and begin consuming results.""" - self._ws = await self._connect() - self._receive_task = asyncio.create_task(self._receive_loop()) - - async def _connect(self) -> Any: - """Open a fresh Scribe socket with this instance's configuration.""" - return await websockets.connect( - f"{SCRIBE_URL}?model_id={self._model}" - f"&audio_format=pcm_{CALLER_SAMPLE_RATE}" - f"&language_code={self._language}" - "&commit_strategy=manual", - additional_headers={"xi-api-key": self._api_key}, - ) - - async def feed(self, pcm: bytes, *, commit: bool = False) -> None: - """Send one tick of assistant audio, optionally closing the utterance. - - Raises once the session has closed, instead of repeating a swallowed warning every - tick while the caller goes silently deaf for the rest of the call. - """ - if self._ws is None: - return - if self._closed: - raise RuntimeError("Scribe session is closed; caller can no longer hear the assistant") - message: dict[str, Any] = { - "message_type": "input_audio_chunk", - "audio_base_64": base64.b64encode(pcm).decode(), - } - if commit: - message["commit"] = True - try: - await self._ws.send(json.dumps(message)) - except ConnectionClosedOK: - logger.warning("Scribe socket closed mid-send; this tick's audio was dropped, reconnect in progress") - except Exception as exc: - self._closed = True - logger.error(f"Scribe session closed unexpectedly; caller is now deaf: {exc}") - raise RuntimeError("Scribe session closed unexpectedly") from exc - - async def stop(self) -> None: - """Close the socket and stop consuming. Safe to call twice.""" - if self._receive_task is not None: - self._receive_task.cancel() - self._receive_task = None - if self._ws is not None: - try: - await self._ws.close() - finally: - self._ws = None - - async def _receive_loop(self) -> None: - """Fold partial and committed transcripts into the buffer as they arrive, reconnecting on a clean close.""" - while True: - try: - raw = await self._ws.recv() - except ConnectionClosedOK as exc: - if await self._reconnect_after_clean_close(exc): - continue - return - except Exception as exc: - self._error = exc - logger.exception("Scribe receive loop failed") - return - self._reconnect_attempts = 0 - try: - message = json.loads(raw) - except json.JSONDecodeError: - continue - kind = message.get("message_type", "") - text = message.get("text", "") - if not text: - continue - if kind == "partial_transcript": - self.buffer.apply_partial(text) - elif kind.startswith("committed_transcript") or kind.startswith("final_transcript"): - self.buffer.commit(text) - - async def _reconnect_after_clean_close(self, exc: ConnectionClosedOK) -> bool: - """Reopen the socket after a server-initiated clean close, preserving committed transcript. - - Returns False once reconnect attempts are exhausted, at which point the caller must stop. - """ - self._reconnect_attempts += 1 - if self._reconnect_attempts > MAX_RECONNECT_ATTEMPTS: - self._error = exc - self._closed = True - logger.error(f"Scribe closed cleanly {self._reconnect_attempts} times in a row; giving up") - return False - self.buffer.in_flight = "" - try: - self._ws = await self._connect() - except Exception as reconnect_exc: - self._error = reconnect_exc - self._closed = True - logger.exception("Scribe reconnect failed") - return False - logger.info( - "Scribe session closed cleanly (code 1000, likely a max-session-duration cap); " - f"reconnected (attempt {self._reconnect_attempts}/{MAX_RECONNECT_ATTEMPTS}); " - "committed transcript preserved, in-flight partial dropped" - ) - return True diff --git a/src/eva/user_simulator/cascade/stt_livekit.py b/src/eva/user_simulator/cascade/stt_livekit.py new file mode 100644 index 00000000..50a2964b --- /dev/null +++ b/src/eva/user_simulator/cascade/stt_livekit.py @@ -0,0 +1,117 @@ +"""Streaming caller STT backed by LiveKit Agents plugins, used without a room.""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +from typing import Any + +from eva.user_simulator.cascade.constants import CALLER_SAMPLE_RATE +from eva.user_simulator.cascade.stt import TranscriptBuffer +from eva.utils.logging import get_logger + +logger = get_logger(__name__) + +DEFAULT_MODELS = {"elevenlabs": "scribe_v2_realtime"} +API_KEY_ENV = {"elevenlabs": "ELEVENLABS_API_KEY"} + + +def build_livekit_stt(provider: str, params: dict[str, Any]) -> Any: + """Construct the LiveKit plugin STT for a provider, without provider-side endpointing.""" + model = params.get("model") or DEFAULT_MODELS.get(provider) + api_key = params.get("api_key") or os.environ.get(API_KEY_ENV.get(provider, ""), "") + if provider == "elevenlabs": + from livekit.plugins import elevenlabs + + # server_vad is deliberately not passed: the plugin then selects + # commit_strategy=manual, leaving the turn boundary ours to decide. + return elevenlabs.STT(model=model, api_key=api_key) + raise ValueError(f"Unsupported caller STT provider: {provider!r}. Supported: {sorted(DEFAULT_MODELS)}") + + +class LiveKitStreamingSTT: + """Transcribes assistant audio via a LiveKit STT plugin driven by our tick clock. + + Finalization is ours: `feed(..., commit=True)` maps to the plugin's `flush()` + sentinel rather than waiting for provider endpointing, which keeps the tick + scheduler the only thing deciding where a turn ends. + """ + + def __init__(self, provider: str, params: dict[str, Any], *, language: str = "en") -> None: + self.provider = provider + self.params = dict(params) + self.model = params.get("model") or DEFAULT_MODELS.get(provider, "") + self.language = language + self.buffer = TranscriptBuffer() + self._stt: Any = None + self._stream: Any = None + self._reader: asyncio.Task | None = None + self._http: Any = None + + async def start(self) -> None: + """Open the plugin's HTTP context and streaming recognizer.""" + from livekit.agents.utils import http_context + + # Plugins outside the agent worker have no ambient session; without this they + # raise "http session outside of a job context" on first use. + self._http = http_context.open() + await self._http.__aenter__() + self._stt = build_livekit_stt(self.provider, self.params) + self._stream = self._stt.stream() + self._reader = asyncio.create_task(self._receive_loop()) + + async def feed(self, pcm: bytes, *, commit: bool = False) -> None: + """Push one tick of assistant audio, optionally closing the utterance.""" + if self._stream is None: + return + from livekit import rtc + + try: + self._stream.push_frame( + rtc.AudioFrame( + data=pcm, + sample_rate=CALLER_SAMPLE_RATE, + num_channels=1, + samples_per_channel=len(pcm) // 2, + ) + ) + if commit: + self._stream.flush() + except Exception as exc: + logger.warning(f"LiveKit STT feed failed: {exc}") + + async def stop(self) -> None: + """Close the recognizer and HTTP context. Safe to call twice.""" + if self._reader is not None: + self._reader.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._reader + self._reader = None + if self._stream is not None: + with contextlib.suppress(Exception): + await self._stream.aclose() + self._stream = None + if self._http is not None: + with contextlib.suppress(Exception): + await self._http.__aexit__(None, None, None) + self._http = None + + async def _receive_loop(self) -> None: + """Fold interim and final transcripts into the buffer as the plugin emits them.""" + from livekit.agents import stt as lk_stt + + try: + async for event in self._stream: + alternatives = getattr(event, "alternatives", None) or [] + text = alternatives[0].text if alternatives else "" + if not text: + continue + if event.type == lk_stt.SpeechEventType.INTERIM_TRANSCRIPT: + self.buffer.apply_partial(text) + elif event.type == lk_stt.SpeechEventType.FINAL_TRANSCRIPT: + self.buffer.commit(text) + except asyncio.CancelledError: + raise + except Exception: + logger.exception("LiveKit STT receive loop failed; caller may stop hearing the assistant") diff --git a/tests/unit/user_simulator/cascade/test_constants.py b/tests/unit/user_simulator/cascade/test_constants.py index f3ed9d39..cec8808b 100644 --- a/tests/unit/user_simulator/cascade/test_constants.py +++ b/tests/unit/user_simulator/cascade/test_constants.py @@ -3,16 +3,12 @@ TICK_DURATION_MS, WAIT_TO_RESPOND_OTHER_MS, WAIT_TO_RESPOND_SELF_MS, - YIELD_WHEN_INTERRUPTED_MS, - YIELD_WHEN_INTERRUPTING_MS, ms_to_ticks, ) THRESHOLD_MS_CONSTANTS = [ WAIT_TO_RESPOND_OTHER_MS, WAIT_TO_RESPOND_SELF_MS, - YIELD_WHEN_INTERRUPTED_MS, - YIELD_WHEN_INTERRUPTING_MS, ] diff --git a/tests/unit/user_simulator/cascade/test_prompt.py b/tests/unit/user_simulator/cascade/test_prompt.py index fa86c1c8..ce7b4ae4 100644 --- a/tests/unit/user_simulator/cascade/test_prompt.py +++ b/tests/unit/user_simulator/cascade/test_prompt.py @@ -1,9 +1,14 @@ -from eva.utils.prompt_manager import PromptManager +def test_cascade_reuses_the_shared_end_call_description(): + # A cascade-specific copy would drift from the other providers' hang-up rules. + from eva.user_simulator.cascade.simulator import END_CALL_DESCRIPTION as cascade_description + from eva.user_simulator.openai_realtime import END_CALL_DESCRIPTION as shared_description + assert cascade_description is shared_description -def test_cascade_turn_contract_prompt_exists_and_names_the_json_field(): - prompt = PromptManager().get_prompt("user_simulator.cascade_turn_contract") - assert "utterance" in prompt - assert "JSON" in prompt - assert "end_call" in prompt +def test_no_cascade_specific_prompts_remain_in_the_prompt_file(): + # The per-domain user_simulator prompt already carries persona, goal and end_call rules; + # layering a cascade-only contract on top is what suppressed the end_call tool call. + from pathlib import Path + + assert "cascade_" not in Path("configs/prompts/simulation.yaml").read_text() diff --git a/tests/unit/user_simulator/cascade/test_scheduler.py b/tests/unit/user_simulator/cascade/test_scheduler.py index 9e7943d5..d5a40fee 100644 --- a/tests/unit/user_simulator/cascade/test_scheduler.py +++ b/tests/unit/user_simulator/cascade/test_scheduler.py @@ -1,4 +1,5 @@ from eva.user_simulator.cascade.adapter.base import Adapter +from eva.user_simulator.cascade.constants import ASSISTANT_UNRESPONSIVE_MS, ms_to_ticks from eva.user_simulator.cascade.scheduler import TickScheduler from eva.user_simulator.cascade.tick_result import TickResult @@ -102,6 +103,29 @@ async def test_caller_cannot_take_a_second_turn_while_awaiting_a_reply(): assert scheduler.may_take_turn() is False +async def test_caller_stops_waiting_once_the_assistant_goes_unresponsive(): + # The assistant that never answers a goodbye would otherwise hold the caller + # in awaiting-reply forever, so it could never reach its end_call turn. + scheduler = _scheduler([True]) + await scheduler.run_tick() + scheduler.enqueue_utterance(b"\x02" * BYTES_PER_TICK) + await scheduler.run_tick() + + for _ in range(ms_to_ticks(ASSISTANT_UNRESPONSIVE_MS) - 5): + await scheduler.run_tick() + assert scheduler.may_take_turn() is False + + for _ in range(10): + await scheduler.run_tick() + assert scheduler.may_take_turn() is True + + +async def test_unresponsive_threshold_clears_the_longest_observed_real_gap(): + # Longest legitimate assistant gap measured across live runs was 220 ticks; + # firing inside that would make the caller talk over a merely slow assistant. + assert ms_to_ticks(ASSISTANT_UNRESPONSIVE_MS) > 220 + + async def test_caller_may_take_a_second_turn_once_the_assistant_replies(): scheduler = _scheduler([True, False, True] + [False] * 30) await scheduler.run_tick() # tick 0: assistant greets diff --git a/tests/unit/user_simulator/cascade/test_simulator.py b/tests/unit/user_simulator/cascade/test_simulator.py index 7aae1c16..5c08b065 100644 --- a/tests/unit/user_simulator/cascade/test_simulator.py +++ b/tests/unit/user_simulator/cascade/test_simulator.py @@ -1,21 +1,10 @@ -import json import logging -import pytest - from eva.models.config import PerturbationConfig from eva.user_simulator.cascade.simulator import CascadeUserSimulator, extract_turn, parse_turn_response -def test_parse_turn_response_reads_a_clean_json_object(): - assert parse_turn_response('{"utterance": "Hi there."}') == "Hi there." - - -def test_parse_turn_response_tolerates_markdown_fences(): - assert parse_turn_response('```json\n{"utterance": "Bye."}\n```') == "Bye." - - -def test_parse_turn_response_falls_back_to_raw_text_when_not_json(): +def test_parse_turn_response_returns_the_spoken_line_unchanged(): assert parse_turn_response("I need to reset my password.") == "I need to reset my password." @@ -24,14 +13,9 @@ def test_parse_turn_response_returns_empty_for_a_toolcall_only_turn(): assert parse_turn_response("") == "" -def test_parse_turn_response_rejects_a_non_string_utterance(): - with pytest.raises(ValueError, match="utterance"): - parse_turn_response(json.dumps({"utterance": 42})) - - def test_extract_turn_reads_a_plain_string_as_no_hangup(): # LiteLLMClient returns a bare str when the model made no tool call. - assert extract_turn('{"utterance": "Still here."}') == ("Still here.", False) + assert extract_turn("Still here.") == ("Still here.", False) def test_extract_turn_detects_the_end_call_tool(): @@ -56,7 +40,7 @@ class _Call: function = _Fn() class _Message: - content = '{"utterance": "Go on."}' + content = "Go on." tool_calls = [_Call()] assert extract_turn(_Message()) == ("Go on.", False) @@ -88,7 +72,7 @@ def _make_bare_simulator() -> CascadeUserSimulator: return sim -def test_messages_flips_roles_so_the_caller_llm_sees_its_own_lines_as_assistant(): +def test_messages_flips_roles_so_the_user_simulator_llm_sees_its_own_lines_as_assistant(): sim = _make_bare_simulator() sim._history = [ {"role": "assistant", "content": "What is your email?"}, @@ -102,11 +86,82 @@ def test_messages_flips_roles_so_the_caller_llm_sees_its_own_lines_as_assistant( assert messages[2] == {"role": "assistant", "content": "It's jane@example.com."} -def test_messages_appends_a_trailing_role_reminder(): - sim = _make_bare_simulator() +class _FakeEventLogger: + def __init__(self) -> None: + self.events: list[tuple[str, dict]] = [] - messages = sim._messages() + def log_event(self, name, data): + self.events.append((name, data)) + + +class _FakeScheduler: + tick = 7 + + +def _simulator_with_buffer(committed: str = "", in_flight: str = ""): + """Build a bare simulator with just the attributes _collect_heard_text touches.""" + from eva.user_simulator.cascade.stt import TranscriptBuffer + + sim = CascadeUserSimulator.__new__(CascadeUserSimulator) + buffer = TranscriptBuffer() + buffer.committed, buffer.in_flight = committed, in_flight + sim._stt = type("_Stt", (), {"buffer": buffer})() + sim._ticks_awaiting_transcript = 0 + sim._missed_transcripts = 0 + sim.event_logger = _FakeEventLogger() + return sim + + +def test_committed_transcript_is_taken_immediately(): + sim = _simulator_with_buffer(committed="I can help with that.") + + assert sim._collect_heard_text(_FakeScheduler()) == ("I can help with that.", False) + + +def test_empty_buffer_waits_rather_than_generating_a_turn(): + # Finalization is not instant; the first empty read means "not ready yet". + sim = _simulator_with_buffer() + + assert sim._collect_heard_text(_FakeScheduler()) == ("", True) + + +def test_wait_expires_into_the_in_flight_partial(): + from eva.user_simulator.cascade.constants import TRANSCRIPT_WAIT_MS, ms_to_ticks + + sim = _simulator_with_buffer(in_flight="Please confirm your username") + for _ in range(ms_to_ticks(TRANSCRIPT_WAIT_MS)): + assert sim._collect_heard_text(_FakeScheduler()) == ("", True) + + assert sim._collect_heard_text(_FakeScheduler()) == ("Please confirm your username", False) + assert sim._stt.buffer.in_flight == "" + + +def test_hearing_nothing_at_all_is_reported_and_never_reuses_stale_history(): + from eva.user_simulator.cascade.constants import TRANSCRIPT_WAIT_MS, ms_to_ticks + + sim = _simulator_with_buffer() + for _ in range(ms_to_ticks(TRANSCRIPT_WAIT_MS)): + sim._collect_heard_text(_FakeScheduler()) + + heard, waiting = sim._collect_heard_text(_FakeScheduler()) + + assert (heard, waiting) == ("", False) + assert sim._missed_transcripts == 1 + assert [name for name, _ in sim.event_logger.events] == ["transcript_missed"] + + +def test_the_wait_counter_resets_after_a_successful_read(): + sim = _simulator_with_buffer() + sim._collect_heard_text(_FakeScheduler()) + sim._stt.buffer.committed = "Thanks, Marcus." + + sim._collect_heard_text(_FakeScheduler()) + + assert sim._ticks_awaiting_transcript == 0 + + +def test_missed_utterance_directive_forbids_repeating(): + from eva.user_simulator.cascade.simulator import MISSED_UTTERANCE_DIRECTIVE - assert messages[-1]["role"] == "system" - assert "CUSTOMER" in messages[-1]["content"] - assert "Do NOT respond as the customer service agent" in messages[-1]["content"] + assert "not repeat" in MISSED_UTTERANCE_DIRECTIVE.lower() + assert "repeat it" in MISSED_UTTERANCE_DIRECTIVE.lower() diff --git a/tests/unit/user_simulator/cascade/test_stt.py b/tests/unit/user_simulator/cascade/test_stt.py index a842ee6c..6c07f804 100644 --- a/tests/unit/user_simulator/cascade/test_stt.py +++ b/tests/unit/user_simulator/cascade/test_stt.py @@ -1,11 +1,4 @@ -import asyncio -import base64 -import json - -import pytest -from websockets.exceptions import ConnectionClosedOK - -from eva.user_simulator.cascade.stt import MAX_RECONNECT_ATTEMPTS, ScribeStreamingSTT, TranscriptBuffer +from eva.user_simulator.cascade.stt import TranscriptBuffer def test_partial_updates_replace_the_in_flight_text(): @@ -37,212 +30,9 @@ def test_successive_commits_accumulate_with_spaces(): assert buffer.committed == "First sentence. Second sentence." -def test_current_text_marks_the_incomplete_utterance(): - buffer = TranscriptBuffer() - buffer.commit("I found your booking.") - buffer.apply_partial("It leaves on Thurs") - - assert buffer.current_text() == "I found your booking. It leaves on Thurs [CURRENTLY SPEAKING, INCOMPLETE]" - - -def test_current_text_omits_the_marker_when_nothing_is_in_flight(): - buffer = TranscriptBuffer() - buffer.commit("I found your booking.") - - assert buffer.current_text() == "I found your booking." - - def test_take_committed_drains_the_buffer(): buffer = TranscriptBuffer() buffer.commit("All done.") assert buffer.take_committed() == "All done." assert buffer.committed == "" - - -class SuspendingFakeWebSocket: - """Fake websocket whose recv() genuinely suspends (awaits a future) before returning.""" - - def __init__(self, messages): - self._messages = list(messages) - self.sent: list[dict] = [] - - async def recv(self): - await asyncio.sleep(0.001) - if not self._messages: - await asyncio.Event().wait() - item = self._messages.pop(0) - if isinstance(item, Exception): - raise item - return json.dumps(item) - - async def send(self, raw: str) -> None: - self.sent.append(json.loads(raw)) - - async def close(self) -> None: - pass - - -async def _make_stt(messages) -> tuple[ScribeStreamingSTT, SuspendingFakeWebSocket]: - stt = ScribeStreamingSTT({"api_key": "test-key"}) - fake = SuspendingFakeWebSocket(messages) - stt._ws = fake - stt._receive_task = asyncio.create_task(stt._receive_loop()) - return stt, fake - - -async def test_partial_transcript_lands_in_in_flight(): - stt, _ = await _make_stt( - [ - {"message_type": "session_started", "session_id": "abc", "config": {}}, - {"message_type": "partial_transcript", "text": "It leaves on Thurs"}, - ] - ) - await asyncio.sleep(0.05) - assert stt.buffer.in_flight == "It leaves on Thurs" - await stt.stop() - - -async def test_committed_transcript_lands_in_committed(): - stt, _ = await _make_stt( - [ - {"message_type": "committed_transcript", "text": "It leaves on Thursday."}, - ] - ) - await asyncio.sleep(0.05) - assert stt.buffer.committed == "It leaves on Thursday." - await stt.stop() - - -async def test_feed_with_commit_sends_commit_true(): - stt, fake = await _make_stt([]) - await stt.feed(b"\x00\x00", commit=True) - - assert fake.sent[-1]["commit"] is True - assert base64.b64decode(fake.sent[-1]["audio_base_64"]) == b"\x00\x00" - await stt.stop() - - -async def test_feed_without_commit_omits_commit_flag(): - stt, fake = await _make_stt([]) - await stt.feed(b"\x00\x00") - - assert "commit" not in fake.sent[-1] - await stt.stop() - - -async def test_session_started_is_ignored_harmlessly(): - stt, _ = await _make_stt( - [ - {"message_type": "session_started", "session_id": "abc", "config": {}}, - ] - ) - await asyncio.sleep(0.05) - assert stt.buffer.committed == "" - assert stt.buffer.in_flight == "" - assert stt._error is None - await stt.stop() - - -async def test_recv_error_is_recorded_and_object_stays_usable(): - stt, fake = await _make_stt([RuntimeError("boom")]) - await asyncio.sleep(0.05) - - assert isinstance(stt._error, RuntimeError) - await stt.feed(b"\x00\x00") - assert fake.sent - await stt.stop() - - -class ClosingFakeWebSocket(SuspendingFakeWebSocket): - """Fake websocket whose send() fails, as a server-side close does.""" - - async def send(self, raw: str) -> None: - raise ConnectionClosedError() - - -class ConnectionClosedError(Exception): - pass - - -async def test_feed_raises_and_marks_closed_when_the_socket_send_fails(): - stt = ScribeStreamingSTT({"api_key": "test-key"}) - fake = ClosingFakeWebSocket([]) - stt._ws = fake - stt._receive_task = asyncio.create_task(stt._receive_loop()) - - with pytest.raises(RuntimeError, match="Scribe session closed"): - await stt.feed(b"\x00\x00") - - assert stt._closed is True - await stt.stop() - - -async def test_feed_raises_immediately_once_closed_without_resending(): - stt, fake = await _make_stt([]) - stt._closed = True - - with pytest.raises(RuntimeError, match="Scribe session is closed"): - await stt.feed(b"\x00\x00") - - assert fake.sent == [] - await stt.stop() - - -def _connect_stub(sockets): - remaining = list(sockets) - - async def _connect(): - return remaining.pop(0) - - return _connect - - -async def test_reconnects_after_a_clean_close_and_keeps_transcribing(): - stt = ScribeStreamingSTT({"api_key": "test-key"}) - stt.buffer.commit("Heard before the close.") - stt._ws = SuspendingFakeWebSocket([ConnectionClosedOK(None, None)]) - second_socket = SuspendingFakeWebSocket( - [{"message_type": "committed_transcript", "text": "Heard after reconnecting."}] - ) - stt._connect = _connect_stub([second_socket]) - stt._receive_task = asyncio.create_task(stt._receive_loop()) - - await asyncio.sleep(0.05) - - assert stt.buffer.committed == "Heard before the close. Heard after reconnecting." - assert stt._ws is second_socket - assert stt._error is None - assert stt._closed is False - await stt.stop() - - -async def test_reconnect_drops_in_flight_partial_but_keeps_committed_text(): - stt = ScribeStreamingSTT({"api_key": "test-key"}) - stt.buffer.commit("Already committed.") - stt.buffer.apply_partial("mid-utterance when the close happened") - stt._ws = SuspendingFakeWebSocket([ConnectionClosedOK(None, None)]) - stt._connect = _connect_stub([SuspendingFakeWebSocket([])]) - stt._receive_task = asyncio.create_task(stt._receive_loop()) - - await asyncio.sleep(0.05) - - assert stt.buffer.committed == "Already committed." - assert stt.buffer.in_flight == "" - await stt.stop() - - -async def test_persistent_clean_closes_exhaust_the_cap_and_surface_an_error(): - stt = ScribeStreamingSTT({"api_key": "test-key"}) - sockets = [SuspendingFakeWebSocket([ConnectionClosedOK(None, None)]) for _ in range(MAX_RECONNECT_ATTEMPTS + 1)] - stt._ws = sockets[0] - stt._connect = _connect_stub(sockets[1:]) - stt._receive_task = asyncio.create_task(stt._receive_loop()) - - await asyncio.sleep(0.05) - - assert isinstance(stt._error, ConnectionClosedOK) - assert stt._closed is True - with pytest.raises(RuntimeError, match="Scribe session is closed"): - await stt.feed(b"\x00\x00") - await stt.stop() diff --git a/tests/unit/user_simulator/cascade/test_stt_livekit.py b/tests/unit/user_simulator/cascade/test_stt_livekit.py new file mode 100644 index 00000000..cb68f2b7 --- /dev/null +++ b/tests/unit/user_simulator/cascade/test_stt_livekit.py @@ -0,0 +1,43 @@ +import pytest + +from eva.user_simulator.cascade.stt import TranscriptBuffer +from eva.user_simulator.cascade.stt_livekit import LiveKitStreamingSTT, build_livekit_stt + + +def test_unknown_provider_is_rejected_with_a_clear_message(): + with pytest.raises(ValueError, match="Unsupported caller STT provider"): + build_livekit_stt("nope", {}) + + +def test_elevenlabs_provider_defaults_to_the_realtime_scribe_model(): + # Only scribe_v2_realtime streams interim transcripts and honours flush(). + stt = LiveKitStreamingSTT("elevenlabs", {"api_key": "k"}) + + assert stt.model == "scribe_v2_realtime" + + +def test_explicit_model_overrides_the_default(): + stt = LiveKitStreamingSTT("elevenlabs", {"api_key": "k", "model": "scribe_v2"}) + + assert stt.model == "scribe_v2" + + +def test_buffer_starts_empty_and_is_a_transcript_buffer(): + stt = LiveKitStreamingSTT("elevenlabs", {"api_key": "k"}) + + assert isinstance(stt.buffer, TranscriptBuffer) + assert stt.buffer.committed == "" + assert stt.buffer.in_flight == "" + + +async def test_feed_before_start_is_a_noop_rather_than_an_error(): + stt = LiveKitStreamingSTT("elevenlabs", {"api_key": "k"}) + + await stt.feed(b"\x00" * 320) + + +async def test_stop_is_safe_before_start_and_twice(): + stt = LiveKitStreamingSTT("elevenlabs", {"api_key": "k"}) + + await stt.stop() + await stt.stop() diff --git a/uv.lock b/uv.lock index 4eae4cdb..c8281984 100644 --- a/uv.lock +++ b/uv.lock @@ -294,6 +294,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/16/fbe8e1e185a45042f7cd3a282def5bb8d95bb69ab9e9ef6a5368aa17e426/audioread-3.1.0-py3-none-any.whl", hash = "sha256:b30d1df6c5d3de5dcef0fb0e256f6ea17bdcf5f979408df0297d8a408e2971b4", size = 23143, upload-time = "2025-10-26T19:44:12.016Z" }, ] +[[package]] +name = "av" +version = "18.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/a4/570a5a35c8638aba01e739925846c35fdd6b0756a15526766d0a4dd3b7df/av-18.0.0.tar.gz", hash = "sha256:4ef7e72c3d3a872584a1215173b16e0226811037f40dcdbf75992631098df1ba", size = 4340222, upload-time = "2026-07-02T06:37:58.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/4a/9e3463df030e063d757fa12f0f39be6541b45b06b5bad48c2ce361b924bf/av-18.0.0-cp311-abi3-macosx_11_0_x86_64.whl", hash = "sha256:149289d40e732a6e49c9530bc245b49d9964cfd1c8c9e06778703b7d5bba6b25", size = 22499354, upload-time = "2026-07-02T06:36:58.751Z" }, + { url = "https://files.pythonhosted.org/packages/77/b3/2576a44b4f39c7462ced4c17fec04c756f7b0f3c5cb940d124173e417d6a/av-18.0.0-cp311-abi3-macosx_14_0_arm64.whl", hash = "sha256:35274c20d2ad3b4774fe632bcef2e34af79858ddf899352339cc3babbc13a484", size = 18175248, upload-time = "2026-07-02T06:37:01.741Z" }, + { url = "https://files.pythonhosted.org/packages/84/74/6732f17b96dc23fd23b876b2805435855abdc8a3b397142be4e581165de8/av-18.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4d683b7747a0ba9222b8a5f81e41db5f796e7f64473454ec4fe2548e083c2fa0", size = 33387843, upload-time = "2026-07-02T06:37:05.097Z" }, + { url = "https://files.pythonhosted.org/packages/6d/b9/7708c43fed7ae28b4a1bad060b4221e3334cd827cec24f7165902a6ac1f4/av-18.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ae56b40b6f8b067a8ad2dac664fbfbabac7f7a55b9a7bb031eb99289252bc017", size = 35536910, upload-time = "2026-07-02T06:37:08.806Z" }, + { url = "https://files.pythonhosted.org/packages/5a/94/eba99691d184f6a395a242d54dc370e2fd2265e95bbc98e2963a0fdbdd6c/av-18.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ea2e8ebbce521f21b55df9400e00d721623c9020ef158f5a188a96130be0743f", size = 38984619, upload-time = "2026-07-02T06:37:11.861Z" }, + { url = "https://files.pythonhosted.org/packages/c9/cf/0d7aee07fe16aa9ffdf96043c14bed5485a52c0dea4259de87aa306ecab4/av-18.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ef96dabb3e50dac249913145dff5424b302b257fd95dcb64be3c7b7a8aef16d1", size = 34451176, upload-time = "2026-07-02T06:37:15.154Z" }, + { url = "https://files.pythonhosted.org/packages/76/92/810da80b12680d4c4fe235bd1b4003289be9213ac7f114b77b8ecf0e3b3e/av-18.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:0f65518a184613e41536f29e8758c8e3d8293e46bf5bef108f04f925bbfa3f44", size = 36619869, upload-time = "2026-07-02T06:37:18.495Z" }, + { url = "https://files.pythonhosted.org/packages/11/85/0f121ff43dc5a70696676c98a8f1674e2fa787614c2abaacb15fa1a9bc99/av-18.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:aaf4d354d2beaa6651e4f92e54409a578bde64f79c0beef9a30b388d06f7c629", size = 27556236, upload-time = "2026-07-02T06:37:21.388Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f6/2509754d4d2356abc6fc0ea3d57c12ade29bac23a1fb7fc215a53ca518fb/av-18.0.0-cp311-abi3-win_arm64.whl", hash = "sha256:adac2b3833b6cb9bd6cb52664a522b94db453615b3675b1dbb26e13fe1c80da6", size = 20221133, upload-time = "2026-07-02T06:37:23.88Z" }, +] + [[package]] name = "azure-cognitiveservices-speech" version = "1.48.2" @@ -738,6 +755,8 @@ dependencies = [ { name = "jaconv" }, { name = "jiwer" }, { name = "litellm" }, + { name = "livekit-agents" }, + { name = "livekit-plugins-elevenlabs" }, { name = "more-itertools" }, { name = "numpy" }, { name = "onnxruntime" }, @@ -797,6 +816,8 @@ requires-dist = [ { name = "jiwer", specifier = ">=3.0.0" }, { name = "librosa", marker = "extra == 'apps'", specifier = ">=0.11" }, { name = "litellm", specifier = "==1.85.0" }, + { name = "livekit-agents", specifier = ">=1.6.8" }, + { name = "livekit-plugins-elevenlabs", specifier = ">=1.6.8" }, { name = "more-itertools", specifier = ">=10.0.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.5" }, { name = "numpy", specifier = ">=1.24" }, @@ -829,6 +850,15 @@ requires-dist = [ ] provides-extras = ["apps", "dev"] +[[package]] +name = "eval-type-backport" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/15/273a4baf8248d6d76220723c3caf039d283774b31a7c46ba686120145b76/eval_type_backport-0.4.0.tar.gz", hash = "sha256:8397d25e6524c2e67b9576bb0636be27dea2192017711220c534ec2de921e9b0", size = 10260, upload-time = "2026-06-02T13:22:06.059Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/a7/bb99bf5e6f78736ddb53480f2c3ff3702ffe2196a7c5e1661c03081d398e/eval_type_backport-0.4.0-py3-none-any.whl", hash = "sha256:ad5e2a8db71b6696a56eafb938b0f5a337d3217f256b8e158b469422b4772b20", size = 6432, upload-time = "2026-06-02T13:22:04.827Z" }, +] + [[package]] name = "fastapi" version = "0.133.1" @@ -1427,6 +1457,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, ] +[[package]] +name = "json-repair" +version = "0.60.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/a6/d69888cb4ffde30e80db1e6c32caaadd2f984a80067d5ea72c2cb3f61c3f/json_repair-0.60.1.tar.gz", hash = "sha256:841661cdd2df507c9a4e189097f38ca6bc372e06d4b4e36d72e590f68176c290", size = 49451, upload-time = "2026-06-03T17:28:44.451Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/1f/2a2b5eea8ef5762a86ad3f8fddddaaba2c0d76dd44e644b9158900868bec/json_repair-0.60.1-py3-none-any.whl", hash = "sha256:ba6ff974f2a8bef2f7768144a7f03f870a816443f03da27a49cdd0ec31a78049", size = 48045, upload-time = "2026-06-03T17:28:43.038Z" }, +] + [[package]] name = "jsonschema" version = "4.26.0" @@ -1562,6 +1601,157 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1c/38/e6a4abb062e039d18d59538cc4e6fc370c2c10cd2bff4a2e546acb69dcb9/litellm-1.85.0-py3-none-any.whl", hash = "sha256:2bb449153610691faffd76f5b94a8c29e4b66fc5394156ebf54fd4fe92759b1a", size = 16978229, upload-time = "2026-05-17T01:59:11.902Z" }, ] +[[package]] +name = "livekit" +version = "1.1.14" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiofiles" }, + { name = "numpy" }, + { name = "protobuf" }, + { name = "types-protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ab/5d/bfaf1cc73f960b40294f604d334f05e628b0a07de3c47e475d760996a8d0/livekit-1.1.14.tar.gz", hash = "sha256:47428e10ecf20d7db4ee9fde4009bf96578c003b1ae6e1c5e7e4837a55902393", size = 375000, upload-time = "2026-07-31T14:05:14.425Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/ff/a2659522b3cf860b9b4453e1ec12d4b4c7e9cfd2b672f2cf925016d73492/livekit-1.1.14-py3-none-macosx_10_15_x86_64.whl", hash = "sha256:5f671b1752c93b878cb241b84fd3f72a31f857c3927755d672cfb7656a84778c", size = 10196322, upload-time = "2026-07-31T14:05:04.167Z" }, + { url = "https://files.pythonhosted.org/packages/82/a2/89f32d369cc78cb1a50b2a9e635c653f88d86ea4338ccdfa7b2d4ca0aecd/livekit-1.1.14-py3-none-macosx_11_0_arm64.whl", hash = "sha256:efa16b9036b0b592e5399fdb858c1f04ec8a32c385184c705f030952f72174e8", size = 9019745, upload-time = "2026-07-31T14:05:06.49Z" }, + { url = "https://files.pythonhosted.org/packages/8e/5b/dda7d660fa5d5b6e228dcfc6be3664a2442d1601481686052af2da642e5e/livekit-1.1.14-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:299146efefad5f67751cd15b8225bae759be0d7ad2f0b4ae1a22c15860d93cf9", size = 10042499, upload-time = "2026-07-31T14:05:08.563Z" }, + { url = "https://files.pythonhosted.org/packages/21/e3/d9255eeaf205f090d63d762e5254097b62af394bfaa90106f71f1fb6740e/livekit-1.1.14-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:80962c4a22ddbf0e0ebd3563fc090fce42df66b39b90de68b161b7db01970f68", size = 11445915, upload-time = "2026-07-31T14:05:10.628Z" }, + { url = "https://files.pythonhosted.org/packages/f7/0a/514fb230e7c7f13ae7e53b9e39a6dd9ea1aa9ff5be9e588d55301d159a1e/livekit-1.1.14-py3-none-win_amd64.whl", hash = "sha256:b8f8d38f131956297923e520bc4375bc9ebfa255cab7f125cb7755bfca71df24", size = 10766643, upload-time = "2026-07-31T14:05:12.716Z" }, +] + +[[package]] +name = "livekit-agents" +version = "1.6.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiofiles" }, + { name = "aiohttp" }, + { name = "av" }, + { name = "certifi" }, + { name = "click" }, + { name = "colorama" }, + { name = "docstring-parser" }, + { name = "eval-type-backport" }, + { name = "json-repair" }, + { name = "livekit" }, + { name = "livekit-api" }, + { name = "livekit-blingfire" }, + { name = "livekit-local-inference" }, + { name = "livekit-protocol" }, + { name = "nest-asyncio" }, + { name = "numpy" }, + { name = "openai" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp" }, + { name = "opentelemetry-sdk" }, + { name = "prometheus-client" }, + { name = "protobuf" }, + { name = "psutil" }, + { name = "pydantic" }, + { name = "pyjwt" }, + { name = "pyyaml" }, + { name = "sounddevice" }, + { name = "typer" }, + { name = "types-protobuf" }, + { name = "typing-extensions" }, + { name = "watchfiles" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/71/168a1f61a23d652e72ea6f09bc3096f82a8b4f4d1d7f180dfdcffa6a1eba/livekit_agents-1.6.8.tar.gz", hash = "sha256:c25666b35ff44f19186cc5247690f4d0ec0737c312cc3e530af54abaea03ce7e", size = 2635925, upload-time = "2026-08-03T17:11:26.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/39/4ea6fea005f9c92cd444b04e517befcbf8f2fb8a9591fbfd5edfaa7a6a32/livekit_agents-1.6.8-py3-none-any.whl", hash = "sha256:fefe45142f398895f3fcb37f381569c2f9ce98b7eb0d9dfe6698aa9774d0f355", size = 2749031, upload-time = "2026-08-03T17:11:23.93Z" }, +] + +[package.optional-dependencies] +codecs = [ + { name = "numpy" }, +] + +[[package]] +name = "livekit-api" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "livekit-protocol" }, + { name = "protobuf" }, + { name = "pyjwt" }, + { name = "types-protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f3/19/36ff6712ec638a4b7dad4d8f03795952e401dc31db0b04cddec7892650da/livekit_api-1.2.0.tar.gz", hash = "sha256:a89817b3bca9584873786ff07209839308217537a42f95ecb2609aafaa109ddc", size = 20778, upload-time = "2026-07-11T23:20:54.781Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/e7/8926f16d4bc1b2e0ae46d4a507321bb899396d263a757f1adaabcd3b3867/livekit_api-1.2.0-py3-none-any.whl", hash = "sha256:307f8e5cfb0358c3ca091814ab768af55896022151bcd7f951954ccefa036a24", size = 26499, upload-time = "2026-07-11T23:20:53.736Z" }, +] + +[[package]] +name = "livekit-blingfire" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/09/1095ace608a41810d5c0f343eff36154505487c415acd9c653a882ff2cf1/livekit_blingfire-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0358058ba6cba59379d22a01acef6ff8a729b0facf880c0f75d13c26f1315c9d", size = 153650, upload-time = "2025-12-16T00:48:05.976Z" }, + { url = "https://files.pythonhosted.org/packages/80/a5/f4eb0e5d97334581440d37ced2a1db4fdfc8454c641c7c144e858012f1ce/livekit_blingfire-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a0741a8abcfaa1f3af2313271f15ac0f79777681a8e3ab9a782a68d8eb121c89", size = 148628, upload-time = "2025-12-16T00:48:06.998Z" }, + { url = "https://files.pythonhosted.org/packages/89/f9/dc5ad008cb8b9c2a300bb7f7d44f022cd4970a32707eb90358290a07f0e1/livekit_blingfire-1.1.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d99d7a34c9350da3a6ea738bc282a5f5b4ac4ffb7f8aa5251dfa96070ad845f6", size = 166832, upload-time = "2025-12-16T00:48:07.919Z" }, + { url = "https://files.pythonhosted.org/packages/8f/27/408c435cbed31fa3601ff32ef0499ff594cd898b483c9b4017e9df906de6/livekit_blingfire-1.1.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:815aca6c2f823fa25d7a15d8d76ce18b0295aa5ce2c988ed64fdbd9c4d3ced0a", size = 173959, upload-time = "2025-12-16T00:48:09.153Z" }, + { url = "https://files.pythonhosted.org/packages/2f/12/c826a40b32bfda29e7f826e50dfbd3c0a70726cb8c0cb5023d2311823bd2/livekit_blingfire-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:7ae045d44d8cb867fc449f44a95c0287f6e5d225e62e24f4574bac8f26ede845", size = 130006, upload-time = "2025-12-16T00:48:10.176Z" }, + { url = "https://files.pythonhosted.org/packages/dd/18/8be31c84e911218011e6e653ca466fef320a4e7bc926aa694bc4cb6625f9/livekit_blingfire-1.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9d5fb6746263529b780dc8bf7a6e6a80ff5fa7fa729e403f2b925996d041e039", size = 154567, upload-time = "2025-12-16T00:48:11.097Z" }, + { url = "https://files.pythonhosted.org/packages/03/64/bb5463d4a6a97888d52caa6256d242acab1f7eabcc59343f7874a89a30dc/livekit_blingfire-1.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d4d5642e36fc0a9f89a5154affbd12305ae008c34c7b32f00fe00127ab18d6bd", size = 148792, upload-time = "2025-12-16T00:48:12.324Z" }, + { url = "https://files.pythonhosted.org/packages/6b/9f/ec51ebce455e17b6f304044e2bda57b15b1b45fd20b2feefa6e242fa33c6/livekit_blingfire-1.1.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:502d7a41fed246ec9cc432646d523c488a05fb2e572187a754735532ba5d69b7", size = 167606, upload-time = "2025-12-16T00:48:13.611Z" }, + { url = "https://files.pythonhosted.org/packages/d1/19/a4b56e54af456f2667287497f7678ff69a82ad21a687fc540213b4f25982/livekit_blingfire-1.1.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cdec36ea4d8b0dcda2791358ac9965e832539ecf13e651011197bab9960ea156", size = 174972, upload-time = "2025-12-16T00:48:14.811Z" }, + { url = "https://files.pythonhosted.org/packages/32/29/032cbf2c88ca40bee25b8a1b5346b5cb66487e689c4f42dd19f7e745090d/livekit_blingfire-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:28d8c822616ca2ce53125040dfe09d06a6cc3e63c9055d39ca767a5c8f67ef84", size = 131026, upload-time = "2025-12-16T00:48:16.047Z" }, + { url = "https://files.pythonhosted.org/packages/81/50/46e410b935154a6bcf2d9494ee8e298b1a9c91ae33beaa78346703cf7681/livekit_blingfire-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f5f6a40e498940f5b2e53d9753f5f7fb7f909e12a93a158844c9e3e99a5486b8", size = 154623, upload-time = "2025-12-16T00:48:17.641Z" }, + { url = "https://files.pythonhosted.org/packages/de/b4/f51c25bf104e51703dc66558ff9831a9769a9effa397956268902784a3d0/livekit_blingfire-1.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:945a672a224c9a686925e9af94c2660bacdbe190ccf693d6f17cea9359426c15", size = 148846, upload-time = "2025-12-16T00:48:18.591Z" }, + { url = "https://files.pythonhosted.org/packages/e0/d2/ad95d195ed6dccb6527ed3c1e753f211c3e9509050af5cddf007608bb104/livekit_blingfire-1.1.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f3aac3207cdd88c62323e0b07c33a69aac79c544122a2ddfbecc6c721ca760c", size = 167886, upload-time = "2025-12-16T00:48:19.858Z" }, + { url = "https://files.pythonhosted.org/packages/c5/67/fc4af1bbbed319d8edc319051bce720b51fa544f5d2ebb3201240779f135/livekit_blingfire-1.1.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:839feefa2910f99d794d3f3d696f95193ee8188cc6688a8d712bade2cede7951", size = 175858, upload-time = "2025-12-16T00:48:21.144Z" }, + { url = "https://files.pythonhosted.org/packages/76/6c/9e14763826476925767b511531318a83f95f3bf9e4dbc7dc611400af6e9e/livekit_blingfire-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:1409d4c297260b60a37bfe6ba21e4fb59dd53cd929632c0a78a28d41fe424302", size = 131048, upload-time = "2025-12-16T00:48:22.17Z" }, +] + +[[package]] +name = "livekit-local-inference" +version = "0.2.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/1c/8010498321cb78111194dce2a1b400fd4548a6df9b88b469c3cf2787efd9/livekit_local_inference-0.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d31c99cfcb486b7183381289e72f6f75594c44449f5f28e04fbdd202d4200f2c", size = 34833439, upload-time = "2026-06-24T16:54:48.636Z" }, + { url = "https://files.pythonhosted.org/packages/cc/c5/c371b7f1e36dfeab9d240a3e607320918c2564b51db7d2edb8f5064073c8/livekit_local_inference-0.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56f0641306bc5451502f2f6597bac9b77e56487311d49a1ed356c5ed8ce01da6", size = 35099320, upload-time = "2026-06-24T16:54:51.761Z" }, + { url = "https://files.pythonhosted.org/packages/9a/04/914b672e43b619f0f6023641cd4ff539354ef07b12d79c2e0a538c219069/livekit_local_inference-0.2.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c684ee2d2f22c0a24ff471cdd5d873ee010135a9469d791c2cd2d3f83b219b51", size = 34821645, upload-time = "2026-06-24T16:54:57.782Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a7/2d19b23b872a8e8b80efc612c3bfcc2f4733b24ca159c9d9426eff2d4eac/livekit_local_inference-0.2.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6379f9d5ee4753919d10a2cedd2e16e1cbf634e496ad21fb54564513b731e69", size = 34865997, upload-time = "2026-06-24T16:55:01.294Z" }, + { url = "https://files.pythonhosted.org/packages/6f/66/250dbc92f4dd26b7c91c3f1c17ad238117339399006476728bfbabb5fb46/livekit_local_inference-0.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e5026b2cfd5aa2c85d677cce426b39bab88ee114acbaa814941d3aca8612ba7e", size = 34905736, upload-time = "2026-06-24T16:55:04.213Z" }, + { url = "https://files.pythonhosted.org/packages/a6/aa/7d6cfa6a2fe6baee8a443820d61b0e7df0cc7b9246762cab8f47c17ddcd3/livekit_local_inference-0.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cb11919621cc542148ebed92408f0567b758057881c7e40d852d9738415a8eae", size = 34835713, upload-time = "2026-06-24T16:55:07.043Z" }, + { url = "https://files.pythonhosted.org/packages/be/57/e284fb2663bceb78f24496edf6d9468c861797fd24e655381c17fafe300f/livekit_local_inference-0.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:274ea4046e24377e0744056db051c53976d04bd159c12f8c482246d44089ae79", size = 35100531, upload-time = "2026-06-24T16:55:10.65Z" }, + { url = "https://files.pythonhosted.org/packages/76/6f/9a580a0d3f0c4bb63e7ab627467aeaf66980d21ea52e5946eb054e6f8150/livekit_local_inference-0.2.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:425469dff5505b35a3fab587993b320c829826d041a85c17c53475f64b99f444", size = 34823583, upload-time = "2026-06-24T16:55:13.728Z" }, + { url = "https://files.pythonhosted.org/packages/72/3d/c4a73813247faa704e9eef1cbf52ebc0f8070ad08b295419033ec3bcea5c/livekit_local_inference-0.2.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0cc28d6e2e1431940b37a8c6d931e4dac914806e2c874a05513db6b65a01d7e", size = 34868325, upload-time = "2026-06-24T16:55:16.789Z" }, + { url = "https://files.pythonhosted.org/packages/c3/44/4f5a633f962142480a54eff1e4aea96848e4b54972c617da65e234d51b7d/livekit_local_inference-0.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:5dc4a0574dc2e4a0745e8fcf29c26ca9f3983b885e63ada9761d26311a778d7b", size = 34906570, upload-time = "2026-06-24T16:55:19.914Z" }, + { url = "https://files.pythonhosted.org/packages/33/ae/5080f01d9c412a0702972efbfc8e4aca48f81baca494027aefcff5e4c8cc/livekit_local_inference-0.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:63beb204b64cdcbb3bc5b1caa1dbff2a9049998b6f897cf6b68285bb6b15926e", size = 34835783, upload-time = "2026-06-24T16:55:22.927Z" }, + { url = "https://files.pythonhosted.org/packages/25/17/53ee00d6abf7b9c7ad9036356c38f40634aa83f74b8484f40678a92264ad/livekit_local_inference-0.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:26e397fe543abac838c12101267e46974afd268b6475642fdf116391eee8bd43", size = 35100553, upload-time = "2026-06-24T16:55:26.351Z" }, + { url = "https://files.pythonhosted.org/packages/df/68/1fbbc4d22cbe28800380cdb1c51b8fadbf81bfacb0e966326fc7ee5c1298/livekit_local_inference-0.2.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:188457491cd59ed201d08ec3012fbc2a35d1fa5ac7bd4f4501553b16f5fc7d5b", size = 34823608, upload-time = "2026-06-24T16:55:29.4Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7a/f62bc8dbdd6a4f32abe056bff1c4c4ad7ef30c780a43564594c72af120c2/livekit_local_inference-0.2.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10e1866f23ff6ee694a360e8a032078de3df189d100a748295c30fd1013aef34", size = 34868352, upload-time = "2026-06-24T16:55:32.611Z" }, + { url = "https://files.pythonhosted.org/packages/c8/86/cd258845ad0b52e3ba68f776f3bc40eebf03d1dde31b2bdcd4d0a1fcb46a/livekit_local_inference-0.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:69bc8976d8feef5a9c31e2c7bbd10d84e5494e1ecf85aedc75ebecef04fc9c08", size = 34906535, upload-time = "2026-06-24T16:55:35.78Z" }, +] + +[[package]] +name = "livekit-plugins-elevenlabs" +version = "1.6.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "livekit-agents", extra = ["codecs"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/d7/83b43e8ba682e27eef01a14c949779aac1bfab435dd73d5d27157b541ba7/livekit_plugins_elevenlabs-1.6.8.tar.gz", hash = "sha256:a866a689ad7041f4576f3f2c0ce60db734213a8f4ba115cc995c6782a9f36af9", size = 19224, upload-time = "2026-08-03T17:11:52.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/d1/24d0696029f99bb1ce2776958ab06866962157bca07df04fec5209733897/livekit_plugins_elevenlabs-1.6.8-py3-none-any.whl", hash = "sha256:b82c5962d9aa2f80585c358ff2c7a1d2480309ce3914ae1d780d0a40fcbea2e7", size = 21942, upload-time = "2026-08-03T17:11:51.706Z" }, +] + +[[package]] +name = "livekit-protocol" +version = "1.1.22" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, + { name = "types-protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/65/736a378c2bf89c7fb54c1ff996f0bdc2046c588029d42afa2531b04c717d/livekit_protocol-1.1.22.tar.gz", hash = "sha256:a6517fd4ecea01ccd5055a30caefedc69a3e9ee02f715a79409b671091606692", size = 122570, upload-time = "2026-08-04T20:15:36.285Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/af/343dcfc7e429fbbc30993a733efb7fbfbbae70ac035a52e44111c6f4a76b/livekit_protocol-1.1.22-py3-none-any.whl", hash = "sha256:5c2edc843a48fe21d05b82c637c3e9eb92a88a34ba1e2c39857aca0e6105a84f", size = 149401, upload-time = "2026-08-04T20:15:35.073Z" }, +] + [[package]] name = "llvmlite" version = "0.44.0" @@ -1612,7 +1802,7 @@ name = "markdown-it-py" version = "4.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mdurl", marker = "python_full_version >= '3.13'" }, + { name = "mdurl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } wheels = [ @@ -1865,6 +2055,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4b/27/20770bd6bf8fbe1e16f848ba21da9df061f38d2e6483952c29d2bb5d1d8b/narwhals-2.17.0-py3-none-any.whl", hash = "sha256:2ac5307b7c2b275a7d66eeda906b8605e3d7a760951e188dcfff86e8ebe083dd", size = 444897, upload-time = "2026-02-23T09:44:32.006Z" }, ] +[[package]] +name = "nest-asyncio" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, +] + [[package]] name = "nltk" version = "3.9.4" @@ -2014,6 +2213,118 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/1c/5d43735b2553baae2a5e899dcbcd0670a86930d993184d72ca909bf11c9b/openai-2.36.0-py3-none-any.whl", hash = "sha256:143f6194b548dbc2c921af1f1b03b9f14c85fed8a75b5b516f5bcc11a2a50c63", size = 1302361, upload-time = "2026-05-07T17:33:15.063Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f2/45/7af37fe54e5d3e66e7dcd7ba8b8aeee73f202bfac909cc94b8c4e428f9ac/opentelemetry_exporter_otlp-1.44.0.tar.gz", hash = "sha256:af1cde7c33ea8ed624bf04ac49a885730fe44c1f1ad698656e592c38f70ce106", size = 6090, upload-time = "2026-07-16T15:25:34.585Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/c3/7b466a9463944e70b37b744072a0c1b88a425dade3fff0631adec66c9bcc/opentelemetry_exporter_otlp-1.44.0-py3-none-any.whl", hash = "sha256:4a498fa8d8fd8be9e8e2d175fe5524a3fe581ccffadd8509db86526a5fb97051", size = 6727, upload-time = "2026-07-16T15:25:14.445Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/09/4d717852c1cf3f854b76c7110a5d00883bc3c99288b9b0dbcbeb9e306eb6/opentelemetry_exporter_otlp_proto_common-1.44.0.tar.gz", hash = "sha256:dc87a5a5bc58f149a56d1547e4691588fa12994cdc3bc039a694ccb3375862ac", size = 20202, upload-time = "2026-07-16T15:25:37.658Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/71/65fd9d54c10b860f87c045ccee1264cab7011268895d3528818a29c1172a/opentelemetry_exporter_otlp_proto_common-1.44.0-py3-none-any.whl", hash = "sha256:9a9fe61bba73d802904bc989f1d6b4a7b1ee40f06c40e98d6f85af65aaebb694", size = 17045, upload-time = "2026-07-16T15:25:18.201Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1f/47/80d9e9d468dc5de3af5096f5ccdb065fa4dd1470f74495cc53e59e397f47/opentelemetry_exporter_otlp_proto_grpc-1.44.0.tar.gz", hash = "sha256:40d1ae9e03fcc36de3cbac610cc99f35894938bff9cfd90fc4ec68bd85448463", size = 27225, upload-time = "2026-07-16T15:25:38.308Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/29/6ae42ba32b153ae0a44ae125f0caff2188bbe62d99c82d1768da30864e72/opentelemetry_exporter_otlp_proto_grpc-1.44.0-py3-none-any.whl", hash = "sha256:6a1a645ea182a2f59440c51fa8301d309f3324a8f9d65f8395584b064b67ee4e", size = 19624, upload-time = "2026-07-16T15:25:19.096Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/87/95e2a5aaa795b4e2260d74e16df2d5541deb2ea9de010bcd615f4dee2654/opentelemetry_exporter_otlp_proto_http-1.44.0.tar.gz", hash = "sha256:c633d7270ad6b57cd4cfbe8b0007a9e2e7c0cb50bd6c50fe2a7b245f721a09d8", size = 25806, upload-time = "2026-07-16T15:25:39.162Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/d0/fdeb1a98d8d3a6205f5f297c51b4a9bfe65126ab60339669bbe3dd54c2e2/opentelemetry_exporter_otlp_proto_http-1.44.0-py3-none-any.whl", hash = "sha256:838592fce774c1c8bb7b9a0a7facbfa82e17be5a8a4e94cef10cb84ae026bae3", size = 21850, upload-time = "2026-07-16T15:25:20.006Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/01/40ac4ae9a149263cc52c2cee200ddd80cb6d8db1a4610abf8eabce0fe771/opentelemetry_proto-1.44.0.tar.gz", hash = "sha256:c547a79c2f8c0c515d31509154682e5921c7cfd5ca67b70e1f9266e2c3e103f3", size = 46488, upload-time = "2026-07-16T15:25:45.34Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/7c/8be563d68e93bbefa5c8affb82ddcff91b3ad858ce49957ba7b16fd3e0ab/opentelemetry_proto-1.44.0-py3-none-any.whl", hash = "sha256:898b155a0e1557afd867478fb6158e8122a46329ca0bb8dc53cc55e98f017f56", size = 72483, upload-time = "2026-07-16T15:25:28.429Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/77/a6592cbc7c8d9bcc9d6757a9df45e04a7c585e3e6e7a13456da522b21109/opentelemetry_sdk-1.44.0.tar.gz", hash = "sha256:cebe7f65dc12f26ead75c6064de12fd2a9052e5060c0272d402cfa203aae123b", size = 208624, upload-time = "2026-07-16T15:25:46.078Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/23/ff077e61886ee020a17ce9c8b6fa11c601c8d8345b09ea24f605445df62a/opentelemetry_sdk-1.44.0-py3-none-any.whl", hash = "sha256:df081c4c6bcfdb1211e3e86140376792643128a25f8d72d1d27675936e7e96ad", size = 137221, upload-time = "2026-07-16T15:25:29.534Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.65b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/73/0cbdebcb4cf545fdd328da14f5137e37d0770c3f26185e478b0d15d94f50/opentelemetry_semantic_conventions-0.65b0.tar.gz", hash = "sha256:f9b2b81e9d5b64f11bc952075e7e9c7fb0aab075c7fd1c46d597f1b919852d60", size = 148774, upload-time = "2026-07-16T15:25:46.902Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, +] + [[package]] name = "packaging" version = "26.0" @@ -2241,6 +2552,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5d/19/fd3ef348460c80af7bb4669ea7926651d1f95c23ff2df18b9d24bab4f3fa/pre_commit-4.5.1-py2.py3-none-any.whl", hash = "sha256:3b3afd891e97337708c1674210f8eba659b52a38ea5f822ff142d10786221f77", size = 226437, upload-time = "2025-12-16T21:14:32.409Z" }, ] +[[package]] +name = "prometheus-client" +version = "0.26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/73/f1334c29c2af4cd9dba6c7817e61b611bd0215e2eb5565c6064a4de18802/prometheus_client-0.26.0.tar.gz", hash = "sha256:04a91bcf94e2cf74a44a1a874d651a2e853ed354b6e822f3b7487751465d5c2b", size = 92910, upload-time = "2026-07-24T19:36:41.893Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/a3/b69efbf4143b5b9859b977770bbbabcc2796b702fa69dc40271e45cd5a56/prometheus_client-0.26.0-py3-none-any.whl", hash = "sha256:fa93d06737aa02bacd05794768508bb97d2fbee28cb3bca04eaae92f0ca953d6", size = 64494, upload-time = "2026-07-24T19:36:40.854Z" }, +] + [[package]] name = "propcache" version = "0.4.1" @@ -2337,6 +2657,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, ] +[[package]] +name = "psutil" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/08/510cbdb69c25a96f4ae523f733cdc963ae654904e8db864c07585ef99875/psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b", size = 130595, upload-time = "2026-01-28T18:14:57.293Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f5/97baea3fe7a5a9af7436301f85490905379b1c6f2dd51fe3ecf24b4c5fbf/psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea", size = 131082, upload-time = "2026-01-28T18:14:59.732Z" }, + { url = "https://files.pythonhosted.org/packages/37/d6/246513fbf9fa174af531f28412297dd05241d97a75911ac8febefa1a53c6/psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63", size = 181476, upload-time = "2026-01-28T18:15:01.884Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b5/9182c9af3836cca61696dabe4fd1304e17bc56cb62f17439e1154f225dd3/psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312", size = 184062, upload-time = "2026-01-28T18:15:04.436Z" }, + { url = "https://files.pythonhosted.org/packages/16/ba/0756dca669f5a9300d0cbcbfae9a4c30e446dfc7440ffe43ded5724bfd93/psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b", size = 139893, upload-time = "2026-01-28T18:15:06.378Z" }, + { url = "https://files.pythonhosted.org/packages/1c/61/8fa0e26f33623b49949346de05ec1ddaad02ed8ba64af45f40a147dbfa97/psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9", size = 135589, upload-time = "2026-01-28T18:15:08.03Z" }, + { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, + { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, + { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, + { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, + { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, +] + [[package]] name = "pyarrow" version = "23.0.1" @@ -2532,6 +2874,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, ] +[[package]] +name = "pyjwt" +version = "2.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, +] + [[package]] name = "pyloudnorm" version = "0.2.0" @@ -2856,8 +3207,8 @@ name = "rich" version = "14.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "markdown-it-py", marker = "python_full_version >= '3.13'" }, - { name = "pygments", marker = "python_full_version >= '3.13'" }, + { name = "markdown-it-py" }, + { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b3/c6/f3b320c27991c46f43ee9d856302c70dc2d0fb2dba4842ff739d5f46b393/rich-14.3.3.tar.gz", hash = "sha256:b8daa0b9e4eef54dd8cf7c86c03713f53241884e814f4e2f5fb342fe520f639b", size = 230582, upload-time = "2026-02-19T17:23:12.474Z" } wheels = [ @@ -3114,6 +3465,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "sounddevice" +version = "0.5.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2a/f9/2592608737553638fca98e21e54bfec40bf577bb98a61b2770c912aab25e/sounddevice-0.5.5.tar.gz", hash = "sha256:22487b65198cb5bf2208755105b524f78ad173e5ab6b445bdab1c989f6698df3", size = 143191, upload-time = "2026-01-23T18:36:43.529Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/0a/478e441fd049002cf308520c0d62dd8333e7c6cc8d997f0dda07b9fbcc46/sounddevice-0.5.5-py3-none-any.whl", hash = "sha256:30ff99f6c107f49d25ad16a45cacd8d91c25a1bcdd3e81a206b921a3a6405b1f", size = 32807, upload-time = "2026-01-23T18:36:35.649Z" }, + { url = "https://files.pythonhosted.org/packages/56/f9/c037c35f6d0b6bc3bc7bfb314f1d6f1f9a341328ef47cd63fc4f850a7b27/sounddevice-0.5.5-py3-none-macosx_10_6_x86_64.macosx_10_6_universal2.whl", hash = "sha256:05eb9fd6c54c38d67741441c19164c0dae8ce80453af2d8c4ad2e7823d15b722", size = 108557, upload-time = "2026-01-23T18:36:37.41Z" }, + { url = "https://files.pythonhosted.org/packages/88/a1/d19dd9889cd4bce2e233c4fac007cd8daaf5b9fe6e6a5d432cf17be0b807/sounddevice-0.5.5-py3-none-win32.whl", hash = "sha256:1234cc9b4c9df97b6cbe748146ae0ec64dd7d6e44739e8e42eaa5b595313a103", size = 317765, upload-time = "2026-01-23T18:36:39.047Z" }, + { url = "https://files.pythonhosted.org/packages/c3/0e/002ed7c4c1c2ab69031f78989d3b789fee3a7fba9e586eb2b81688bf4961/sounddevice-0.5.5-py3-none-win_amd64.whl", hash = "sha256:cfc6b2c49fb7f555591c78cb8ecf48d6a637fd5b6e1db5fec6ed9365d64b3519", size = 365324, upload-time = "2026-01-23T18:36:40.496Z" }, + { url = "https://files.pythonhosted.org/packages/4e/39/a61d4b83a7746b70d23d9173be688c0c6bfc7173772344b7442c2c155497/sounddevice-0.5.5-py3-none-win_arm64.whl", hash = "sha256:3861901ddd8230d2e0e8ae62ac320cdd4c688d81df89da036dcb812f757bb3e6", size = 317115, upload-time = "2026-01-23T18:36:42.235Z" }, +] + [[package]] name = "soundfile" version = "0.13.1" @@ -3462,16 +3829,25 @@ name = "typer" version = "0.24.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "annotated-doc", marker = "python_full_version >= '3.13'" }, - { name = "click", marker = "python_full_version >= '3.13'" }, - { name = "rich", marker = "python_full_version >= '3.13'" }, - { name = "shellingham", marker = "python_full_version >= '3.13'" }, + { name = "annotated-doc" }, + { name = "click" }, + { name = "rich" }, + { name = "shellingham" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f5/24/cb09efec5cc954f7f9b930bf8279447d24618bb6758d4f6adf2574c41780/typer-0.24.1.tar.gz", hash = "sha256:e39b4732d65fbdcde189ae76cf7cd48aeae72919dea1fdfc16593be016256b45", size = 118613, upload-time = "2026-02-21T16:54:40.609Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/4a/91/48db081e7a63bb37284f9fbcefda7c44c277b18b0e13fbc36ea2335b71e6/typer-0.24.1-py3-none-any.whl", hash = "sha256:112c1f0ce578bfb4cab9ffdabc68f031416ebcc216536611ba21f04e9aa84c9e", size = 56085, upload-time = "2026-02-21T16:54:41.616Z" }, ] +[[package]] +name = "types-protobuf" +version = "7.34.1.20260518" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/29/59/e2b13b499d15e6720150c4b1a8d91e31fcacf716b432397475b3151ff7e4/types_protobuf-7.34.1.20260518.tar.gz", hash = "sha256:28cfaded25889cb83ebfb63cfb0a43628f0b6f3785767bec17287dc6468795f2", size = 68936, upload-time = "2026-05-18T06:01:47.332Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/1f/ec5caf72c2e3b688ca3927e0979a04ddad19e1afc4bf1c199bd743e0f419/types_protobuf-7.34.1.20260518-py3-none-any.whl", hash = "sha256:a0a5337413347166439c0e07cbc26c6164d091401c6f01b1dfd8cdb966c4dd8f", size = 85992, upload-time = "2026-05-18T06:01:45.696Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" @@ -3566,6 +3942,74 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" }, ] +[[package]] +name = "watchfiles" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/41/5e1a4bb12aac5f1493fa1bdc11154eca3b258ca4eba65d39c473fe19d8e9/watchfiles-1.2.0.tar.gz", hash = "sha256:c995fba777f1ea992f090f9236e9284cf7a5d1a0130dd5a3d82c598cacd76838", size = 108252, upload-time = "2026-05-18T04:32:04.251Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/3d/8024c801df84d1587740d0359e7fdd80afeae3d159011f3d5376dd82f18e/watchfiles-1.2.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:704fd259e332e01f9b9c178f4bce9e49027e5587cc2600eeeaf8e76e1c846201", size = 400242, upload-time = "2026-05-18T04:31:19.014Z" }, + { url = "https://files.pythonhosted.org/packages/87/5b/f4dfd45323e949984a3a7f9dc31d1cbb049921e7d98253488dda72ccdaa9/watchfiles-1.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6543cf55d170003296d185c0af981f3e1311564907e1f4e08671fc7693a890a5", size = 394562, upload-time = "2026-05-18T04:30:08.46Z" }, + { url = "https://files.pythonhosted.org/packages/98/d8/19483ef075d601c409bce8bcbb5c0f81a10876fff870400568f08ce484a1/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89d8c2394a065ca86f5d2910ff263ae67c127e1376ccc4f9fc35c71db879f80a", size = 456611, upload-time = "2026-05-18T04:30:45.723Z" }, + { url = "https://files.pythonhosted.org/packages/b1/6a/cc81fbe7ee42f2f22e661a6e12def7807e01b14b2f39e0ff83fd373fd307/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:772b80df316480d894a0e3165fdd19cf77f5d17f9a787f94029465ad0e3529d1", size = 461379, upload-time = "2026-05-18T04:31:29.292Z" }, + { url = "https://files.pythonhosted.org/packages/b1/57/7e669002082c0a0f4fb5113bb70125f7110124b846b0a11bc5ae8e90eac1/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d158cd89df6053823533e06fb1d73c549133bff5f0396170c0e53d9559340717", size = 493556, upload-time = "2026-05-18T04:30:05.44Z" }, + { url = "https://files.pythonhosted.org/packages/45/7d/f60a2b19807b21fe8281f3a8da4f59eef0d5f96825ac4680ba2d4f2ebf91/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d516b3283a758e087841aedb8031549fb41ced08f3db10aa6d2bf32dc042525b", size = 575255, upload-time = "2026-05-18T04:30:40.568Z" }, + { url = "https://files.pythonhosted.org/packages/bd/49/77f5b5e6efbcd57482f74948ebb1b97e5c0046d6b61475042d830c84b3ff/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:53b2290c92e0506d102cd448fbc610d87079553f86caa39d67440856a8b8bba5", size = 467052, upload-time = "2026-05-18T04:31:17.942Z" }, + { url = "https://files.pythonhosted.org/packages/ee/5a/73e2959af1b97fd5d556f9a8bdba017be23ceeef731869d5eaa0a753d5a3/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a711b51aec4370d0dcda5b6c09463206f133a5759341d7744b953a7b62e1100e", size = 456858, upload-time = "2026-05-18T04:30:30.182Z" }, + { url = "https://files.pythonhosted.org/packages/50/57/1bc8c27fad7e6c19bddee15d276dbb6ab72480ec01c127afff1673aee417/watchfiles-1.2.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:e2ca07fa7d89195ec0865d3d285666286740bfa83d83e5cee204043a31ecc165", size = 467579, upload-time = "2026-05-18T04:32:15.897Z" }, + { url = "https://files.pythonhosted.org/packages/09/6c/3c2e44edba3553c5e3c3b8c8a2a6dee6b9e12ae2cf4bd2378bebf9dc3038/watchfiles-1.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e0618518f282c4ebff60f5e5b1247b6d91bb8b9f4476947563a1e74acc66f3c6", size = 633253, upload-time = "2026-05-18T04:31:37.123Z" }, + { url = "https://files.pythonhosted.org/packages/30/c2/d8c84a882ab39bbefcc4915ab3e91830b7a7e990c5570b0b69075aba3faf/watchfiles-1.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0d191c054d0715c3c95c99df9b8dbf6fd096d8c1e021e8f212e1bd8bc444ccb5", size = 660713, upload-time = "2026-05-18T04:31:24.62Z" }, + { url = "https://files.pythonhosted.org/packages/a9/07/f97736a5fc605364fe67b25e9fa4a6965dfd4840d50c406ada507e9d735f/watchfiles-1.2.0-cp311-cp311-win32.whl", hash = "sha256:9342472aff9b093c5acd4f6d8f70ae0937964ab56542502bcf5579782da69ae8", size = 277222, upload-time = "2026-05-18T04:31:21.131Z" }, + { url = "https://files.pythonhosted.org/packages/cf/99/2b04981977fc2608afd60360d928c6aecf6b950292ca221d98f4005f6694/watchfiles-1.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:dbd6c97045dad81227c8d040173da044c1de08de64a5ea8b555da4aee1d5fa22", size = 290274, upload-time = "2026-05-18T04:31:45.966Z" }, + { url = "https://files.pythonhosted.org/packages/3c/74/f7f58a7075ee9cf612b0cfcddb78b8cd8234f0742d6f0075cf0da2dde1c6/watchfiles-1.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:57a2d9fa4fb4c2ecae57b13dfff2c7ab53e21a2ba674fe9f05506680fcdcc0d7", size = 283460, upload-time = "2026-05-18T04:31:39.126Z" }, + { url = "https://files.pythonhosted.org/packages/b8/2f/e42c992d2afda3108ea1c02acecc991b9f31d05c14adc2a7cee9ee211fc4/watchfiles-1.2.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:bc13eb17538be00c874699dc0abe4ee2bc8d50bb1166a6b9e175ef3fd7eb8f26", size = 400115, upload-time = "2026-05-18T04:32:02.06Z" }, + { url = "https://files.pythonhosted.org/packages/5f/8f/6af2ea19065c91d8b0ea3516fdfc8c0d349f407e8e9fbf4e5a17360de8ad/watchfiles-1.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2d95ddc1eb6914154253d239089900813f6a767e174b8e6a50e7fdacb7e4236c", size = 393659, upload-time = "2026-05-18T04:30:50.951Z" }, + { url = "https://files.pythonhosted.org/packages/13/01/b32a967c56fb3e3e5be3db52c3d3b87fa4513aa367d8ed1ad96d42952e5f/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f70d8b291ef6e88d19b1f297a6905ddb978888d9272b0d05e6f53309856bcfc", size = 453207, upload-time = "2026-05-18T04:31:04.231Z" }, + { url = "https://files.pythonhosted.org/packages/04/98/97557a812180338cb1abd32e1cffcc4588f59b5f23e0cb006b2ba95ba64a/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:56d8641cf834c2836922899105bd3ce3d0dfc69291d52edf0b4d0436829b34c0", size = 459273, upload-time = "2026-05-18T04:31:50.377Z" }, + { url = "https://files.pythonhosted.org/packages/e8/a8/b4b08dcb7653b8087c6586f7ce649505900e866bbcfe40dc9587af02e686/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2581a94056e55d7d0a31a823ea92bf73749c489ca2285bfdc0fbe6b2bb49d50c", size = 489927, upload-time = "2026-05-18T04:31:42.485Z" }, + { url = "https://files.pythonhosted.org/packages/50/94/3dceea03545d2e5ddfd839f0ddd5e1cecbf1697b5a428d5ba11cef6af95d/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:41bc1199f7523b3f82843c88cbb979180c949caef0342cf90968f178e5d49b01", size = 570476, upload-time = "2026-05-18T04:31:03.071Z" }, + { url = "https://files.pythonhosted.org/packages/cc/f2/d39a5450c3532092b91f81d274360e613c2371bc874a89c7a1a3c5e8d138/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7571e4464cb6e434958f867f7f730b8ab0b75e3f8e5eac0499168486ab3c33a8", size = 465650, upload-time = "2026-05-18T04:30:12.701Z" }, + { url = "https://files.pythonhosted.org/packages/22/24/ed72f68cbc1333ca9b9f2200aa048bb6658ae41709bc1caad4310f4bdffd/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e53a384f76b631c3ae5334ce6a52f0baa3a911eb94a4eac7f160079868b716d5", size = 456398, upload-time = "2026-05-18T04:30:13.784Z" }, + { url = "https://files.pythonhosted.org/packages/0d/64/982ef4a4e5bab5b6e5b6becc8cd5e732f6130a78b855f0abec6439a9a135/watchfiles-1.2.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:d20029a60a71a052a24c4db7673bc4de39ab89adbaccbfb5d67987c5d73f424d", size = 465140, upload-time = "2026-05-18T04:31:52.111Z" }, + { url = "https://files.pythonhosted.org/packages/a0/0c/95282abf4ed680b6096010bcfc30c5fa7a041fc5aa5a2ad17a2cc6c75bba/watchfiles-1.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:2cb93af48550faf1cea04c303107c8b75833de7013e57ce27d3b8d21d8d0f58c", size = 630259, upload-time = "2026-05-18T04:31:25.676Z" }, + { url = "https://files.pythonhosted.org/packages/30/45/607c1de1530c4bdcf2cf1d1ecc2505ddba5d96bd43ba9f2b0e79876f850f/watchfiles-1.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2995c176de7692b86a2e4c58d9ec718f753150a979cb4a754e2b4ffa38e70906", size = 659859, upload-time = "2026-05-18T04:30:24.333Z" }, + { url = "https://files.pythonhosted.org/packages/fa/08/d9e2e0f9e8e6791d33aefc694ad7eefa7f901f63caff84a81ded38692f9c/watchfiles-1.2.0-cp312-cp312-win32.whl", hash = "sha256:7a2cffd17d27d2ecbb310c2b1d8174f222a5495b1a721894afa88ec11e25b898", size = 275480, upload-time = "2026-05-18T04:30:31.307Z" }, + { url = "https://files.pythonhosted.org/packages/1c/e6/9d42569c0102645cc8cea5d8c7d8a1e9d4ada2cb7f05f75e554b8aa2202a/watchfiles-1.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:f155b3a1b2a5fc89cdc70d47ee5d54e3b75e88efa34982028a35daef9ba00379", size = 288718, upload-time = "2026-05-18T04:32:10.745Z" }, + { url = "https://files.pythonhosted.org/packages/0a/26/88e0dc6ee3898169d7fa22bb6a69cabf2502d2ee25cb8c876d1262d204f8/watchfiles-1.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:8fa585ede612ee9f9e91b18bebf9ba11b9ae29a4e3a0d0cf6fca3e382133f0d5", size = 281026, upload-time = "2026-05-18T04:30:22.23Z" }, + { url = "https://files.pythonhosted.org/packages/d1/4d/70a7feced9f87e2ff26dba42667290f41694fc64646c67261fbb8cab5d5c/watchfiles-1.2.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:01ea8d66f0693b9b60a6541c8d10263091ca9a9060d242f3c1f3143f9aad2c98", size = 399730, upload-time = "2026-05-18T04:31:38.162Z" }, + { url = "https://files.pythonhosted.org/packages/31/3a/0da302f2307aee316922806ebd5726c542cbd787c938271cf14a074c7daf/watchfiles-1.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7ba0480b9a74af058f43b337e937a451e109295c420916d68ad24e3dc02f5e44", size = 392842, upload-time = "2026-05-18T04:30:27.051Z" }, + { url = "https://files.pythonhosted.org/packages/db/ef/d5bdb705c224dbc256aa0c1ec47bf4e61ec52558f2afb44a71a1fe4d7015/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f34e26a19f91f710c08e0183429f0d1d15df734e6bc78c31e77b9ea9c433658", size = 452989, upload-time = "2026-05-18T04:31:11.945Z" }, + { url = "https://files.pythonhosted.org/packages/71/29/5495f2c1661949ef7a35e4d71111d129cfe7606414a26887a919d0a55406/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b4e77f6a55f858504069abd35d336a637555c09bca453dde1ee1e5ada8a6a1fb", size = 458978, upload-time = "2026-05-18T04:30:52.606Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8c/7f9c07c433811c2fffd93e13fdfb7135de9aab5f2ae41be08960fa0047dc/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0cb4d80e212f116474a545c21c912b445f16bb0cef9e6a73a498164223e14e2f", size = 490248, upload-time = "2026-05-18T04:31:36.003Z" }, + { url = "https://files.pythonhosted.org/packages/3c/11/d93632febc52fbc21be90231bb7c17fd5387f46c9076fd40a5f9c2ae6910/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b974946a10af379d425e2eef5b62f5c6ebeaccf91d45eaad6f5b27ecd4f91aa0", size = 571847, upload-time = "2026-05-18T04:31:10.862Z" }, + { url = "https://files.pythonhosted.org/packages/55/b4/383173e73aabb07ad1d9c7aa859d95437ac46a6d6a1e11005facda0c9d19/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86bc13c25a8d1fcd70b51d0ce7c9b65e90de5666fcbfd3e34957cc73ee19aeb5", size = 465974, upload-time = "2026-05-18T04:30:17.006Z" }, + { url = "https://files.pythonhosted.org/packages/a7/6c/89b1a230a78f57c52dd8893adb1f92f94411721b6ec12596c56d98c74356/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca148d73dea36c9763aaa351e4d7a51780ec1584217c45276f4fe8239c768b71", size = 454782, upload-time = "2026-05-18T04:30:35.656Z" }, + { url = "https://files.pythonhosted.org/packages/24/62/1732118367cfff0a9fce3bf62ff4bfded09ef5df21d9d446b858b3f70a96/watchfiles-1.2.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:c525543d91961c6955b2636b308569e84a1d1c5f5f2932041ab9ef46422f43e3", size = 465182, upload-time = "2026-05-18T04:30:20.846Z" }, + { url = "https://files.pythonhosted.org/packages/28/96/716f7e5f51339bf22963f3345f9f27d7f3b30e2eadc597e257c881dd3c53/watchfiles-1.2.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:a204794696ffb8f9b10fba6f7cb5216d42f3b2b71860ccac6b6e42f5f10973b0", size = 629841, upload-time = "2026-05-18T04:31:05.397Z" }, + { url = "https://files.pythonhosted.org/packages/4c/fe/c40783950fd771ccf66ab3ec2722d188a9af1c7f96c6e811f36e40c6e03f/watchfiles-1.2.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:10d86db20695afe7997ac9e1717637d6714a8d0220458c33f3d2061f54cec427", size = 658028, upload-time = "2026-05-18T04:31:48.22Z" }, + { url = "https://files.pythonhosted.org/packages/71/72/4508db1856d1d87fcbb3b63f4839bab1b5682cb0e8d224d122263c09654a/watchfiles-1.2.0-cp313-cp313-win32.whl", hash = "sha256:eb283ee99e21ad6443c8cdb06ac5b34b1308c329cbdf03fa02b445363714c799", size = 275183, upload-time = "2026-05-18T04:30:59.57Z" }, + { url = "https://files.pythonhosted.org/packages/f9/36/14b76ca57652e5cc5fd1c11f32a261292c08a0d19a00351013c2549cbfb2/watchfiles-1.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:a0f27f01bee51861392bb6b7c4fdb290b27d1eb194e9e28788d68102a0e898d9", size = 288059, upload-time = "2026-05-18T04:32:07.937Z" }, + { url = "https://files.pythonhosted.org/packages/1b/8d/0a85e395398d8d20fadfe5c5d32c726eee17a519e78fb356f2cf7531bffe/watchfiles-1.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:3651aa7058595e9cfb75d35dd5ada2bf9f48a5b8a0f3562821d3e210c507e077", size = 280186, upload-time = "2026-05-18T04:31:54.484Z" }, + { url = "https://files.pythonhosted.org/packages/37/68/36db056f1fdcc5f07302f56e631774d6835bcd6fa3ace402304621d5f9e5/watchfiles-1.2.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:faea288b6f0ab1902ef08f4ca6de005dccf856c4e0c4f21b8c5fce02d90a1b08", size = 399031, upload-time = "2026-05-18T04:30:44.576Z" }, + { url = "https://files.pythonhosted.org/packages/c1/64/01a9d6f66a82a5c101ce939274106cc72759d62427e153f01edd2b9f87c2/watchfiles-1.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:01859b11fd9fbca670f4d5da00fbac282cfea9bd67a2125d8b2833a3b5617ea9", size = 391205, upload-time = "2026-05-18T04:30:25.413Z" }, + { url = "https://files.pythonhosted.org/packages/84/2c/0a44fe058cb4bb7b8ede6b6670698bbb7c0400740e378d00022189b7b31d/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fff610d7bb2256a317bb1e96f0d7862c7aa8076733ee5df0fd41bbe76a24a4f4", size = 451892, upload-time = "2026-05-18T04:32:14.005Z" }, + { url = "https://files.pythonhosted.org/packages/67/a1/351e0d56cd35e6488b5c8b4fb11a809a5bc923e8fe8fed9faf8920be0c89/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b141a4891c995a039cd89e9a49e62df1dc8a559a5d1a6e4c7106d16c12777a55", size = 458867, upload-time = "2026-05-18T04:31:22.279Z" }, + { url = "https://files.pythonhosted.org/packages/d5/7d/9d09605187f1b838998624049fcf8bf47b73c1a3b76901fcac1782f62277/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f22943b7770483f6ea0721c6b11d022947a98eb0acae14694de034f4d0d38925", size = 490217, upload-time = "2026-05-18T04:31:43.657Z" }, + { url = "https://files.pythonhosted.org/packages/60/5d/a17a16eccb182f04188cd308ec24b1a71a9b5c4e7098269cf35d9fa56d02/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1bc6195825b7dcd217968bb1f801a60fd4c16e8eeab5bedc7fe917d7d5995ab4", size = 571458, upload-time = "2026-05-18T04:32:11.875Z" }, + { url = "https://files.pythonhosted.org/packages/d3/3d/4dd457062083ab1938e5dfd45032eb425cee2ac817287ca8ff4356183e5d/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4a4b147f5dca2a5d325a06a832fb43f345751adfbc63204aec30e0d9ca965a2", size = 464707, upload-time = "2026-05-18T04:30:43.492Z" }, + { url = "https://files.pythonhosted.org/packages/c6/71/ea8c57b128f5383de74d0c7d2d9c57ad7c9a65a930c451bd25d524b295b7/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4543579a9bdb0c9560039b4ffddbdb39545707659fbc430ce4c10f3f68d557f9", size = 454663, upload-time = "2026-05-18T04:30:16.061Z" }, + { url = "https://files.pythonhosted.org/packages/53/fd/2e812bf938406d7db351f0703ddd3fc6c061cf30d96153a77bc79a943a44/watchfiles-1.2.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:20aa0e708b920bde876a4aa82dc7dd6ebea228a63a67cda6632c2fc87b787efa", size = 463537, upload-time = "2026-05-18T04:31:44.9Z" }, + { url = "https://files.pythonhosted.org/packages/86/56/d17a7f1dd1bc3035f1072694a551301272f1739c2d8e319c927cb9e29b38/watchfiles-1.2.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:d413349d565dab74297f2a63e84a097936be69bf8f3b3801f27f380e32040f44", size = 629194, upload-time = "2026-05-18T04:31:14.141Z" }, + { url = "https://files.pythonhosted.org/packages/be/06/f1ff66bf5cae50aa4062779a0ecd0bbaf15e466195719074078947d9a17d/watchfiles-1.2.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:f28b2725eb8cce327b9b3ab02415c853011dc55c95832fe90de6bc56f5315f72", size = 656194, upload-time = "2026-05-18T04:31:47.14Z" }, + { url = "https://files.pythonhosted.org/packages/23/f4/7513ef1e85fc4c6331b59479d6d72661fc391fbe543678052ac72c8b6c19/watchfiles-1.2.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:4674d49eb94706dfe666c069fc0a1b646ffcf920473492e209f6d5f60d3f0cc2", size = 403050, upload-time = "2026-05-18T04:30:36.753Z" }, + { url = "https://files.pythonhosted.org/packages/27/0b/a54103cfd732bb703c7a749222011a0483ef3705948dae3b203158601119/watchfiles-1.2.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:094b9b70103d4e963499bdea001ee3c2697b144cd9ae6218a62c0f89ec9e31db", size = 396629, upload-time = "2026-05-18T04:32:03.268Z" }, + { url = "https://files.pythonhosted.org/packages/5e/2c/73f31a3b893886206c3f54d73e8ad8dee58cdb2f69ad2622e0a8a9e07f4e/watchfiles-1.2.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0ef001f8c25ad0fa9529f914c1600647ecd0f542d11c19b7894768c67b6acb7", size = 457318, upload-time = "2026-05-18T04:31:01.932Z" }, + { url = "https://files.pythonhosted.org/packages/e9/f9/45d021e4a5cc7b9dd567f7cbb06d3b75f751a690063fb6cc7ec60f4e46b7/watchfiles-1.2.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a88fc94e647bc4eec523f1caa540258eb71d14278b9daf72fa1e2658a98df0f0", size = 457771, upload-time = "2026-05-18T04:30:56.331Z" }, +] + [[package]] name = "websockets" version = "15.0.1" From 9a251648cd1b8270f066f2900bfb767b712df24e Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Fri, 7 Aug 2026 14:58:57 -0400 Subject: [PATCH 27/65] Emit audio_start/audio_end from the cascade simulator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The metrics processor numbers turns from audio_start(simulated_user) events (processor.py:117,433). BotToBotAudioBridge emitted them from inside its send loop; the cascade simulator replaced that bridge and never re-emitted them, so every conversation_trace entry collapsed onto turn_id 0 on S2S frameworks and latency_assistant_turns came out empty on every framework. Runs looked healthy because the artifacts all existed and the verification passed --metrics '[]', so nothing ever computed the metrics that were missing. TickScheduler gains caller_spoke_this_tick, which flips on the ticks audio actually enters and leaves the wire — distinct from caller_is_speaking, which is true from the moment an utterance is queued. The simulator emits the boundary events from that transition for the user and from has_assistant_speech for the assistant. These boundaries are authored rather than detected. The bridge inferred end-of-audio from a silence threshold and then back-dated the stamp to undo the lag (~600ms per its own comment), whereas the playout queue drains on a known tick. Latency measured against these stamps is therefore more accurate but not directly comparable to earlier ElevenLabs-driven runs. Verified live with metrics enabled: latency_assistant_turns 0 -> 8 populated turns, trace turn_ids 0..8 instead of all zero, strict role alternation. Co-Authored-By: Claude Opus 5 (1M context) --- src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/scheduler.py | 12 +++ src/eva/user_simulator/cascade/simulator.py | 31 ++++++- .../user_simulator/cascade/test_simulator.py | 80 +++++++++++++++++++ 4 files changed, 123 insertions(+), 2 deletions(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index ee46a22c..21c3bd07 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.14" +simulation_version = "2.0.15" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/scheduler.py b/src/eva/user_simulator/cascade/scheduler.py index c1d5bf2d..b750274b 100644 --- a/src/eva/user_simulator/cascade/scheduler.py +++ b/src/eva/user_simulator/cascade/scheduler.py @@ -34,6 +34,7 @@ def __init__(self, adapter: Adapter, *, bytes_per_tick: int = BYTES_PER_TICK) -> self._ticks_since_caller_speech = _NEVER_SPOKE self._assistant_has_spoken = False self._awaiting_reply = False + self._caller_spoke_this_tick = False @property def tick(self) -> int: @@ -54,6 +55,16 @@ def caller_is_speaking(self) -> bool: """Whether caller audio is still queued for playout.""" return bool(self._playout) + @property + def caller_spoke_this_tick(self) -> bool: + """Whether real caller audio went out on the most recent tick. + + Distinct from `caller_is_speaking`, which is true as soon as an utterance is + queued. This flips exactly on the ticks audio enters and leaves the wire, so + it dates the authored turn boundary rather than estimating it from silence. + """ + return self._caller_spoke_this_tick + def may_take_turn(self) -> bool: """Whether both silence thresholds are satisfied (tau: streaming.py:2590-2606). @@ -86,6 +97,7 @@ async def run_tick(self) -> TickResult: result = await self._adapter.run_tick(self._tick, outgoing) del self._playout[:consumed] + self._caller_spoke_this_tick = outgoing is not None self._ticks_since_caller_speech = 0 if outgoing else self._ticks_since_caller_speech + 1 self._ticks_since_assistant_speech = ( 0 if result.has_assistant_speech else self._ticks_since_assistant_speech + 1 diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py index 9d76c11c..23e6778d 100644 --- a/src/eva/user_simulator/cascade/simulator.py +++ b/src/eva/user_simulator/cascade/simulator.py @@ -12,13 +12,14 @@ from eva.user_simulator.base import AbstractUserSimulator from eva.user_simulator.cascade.adapter.realtime_ws import RealtimeWSAdapter from eva.user_simulator.cascade.constants import ( + CALLER_SAMPLE_RATE, TICK_DURATION_MS, TRANSCRIPT_WAIT_MS, - CALLER_SAMPLE_RATE, ms_to_ticks, ) from eva.user_simulator.cascade.scheduler import TickScheduler from eva.user_simulator.cascade.stt_livekit import LiveKitStreamingSTT +from eva.user_simulator.cascade.tick_result import TickResult from eva.user_simulator.cascade.tts import CartesiaTTS # Shared with the OpenAI Realtime provider so both simulators hang up on the same rules. @@ -134,6 +135,7 @@ async def _run(self) -> None: max_ticks = self.timeout * 1000 // TICK_DURATION_MS assistant_was_speaking = False + caller_was_speaking = False try: while scheduler.tick < max_ticks and not self._conversation_done.is_set(): result = await scheduler.run_tick() @@ -148,6 +150,8 @@ async def _run(self) -> None: f"(raw={result.assistant_audio_raw_bytes}B)" ) await self._stt.feed(result.assistant_audio, commit=commit) + self._log_audio_boundaries(scheduler, result, assistant_was_speaking, caller_was_speaking) + caller_was_speaking = scheduler.caller_spoke_this_tick assistant_was_speaking = result.has_assistant_speech if result.has_assistant_speech: continue @@ -166,6 +170,31 @@ async def _run(self) -> None: await adapter.stop() self.event_logger.log_connection_state("session_ended", {"reason": self._end_reason}) + def _log_audio_boundaries( + self, + scheduler: TickScheduler, + result: TickResult, + assistant_was_speaking: bool, + caller_was_speaking: bool, + ) -> None: + """Emit audio_start/audio_end for both roles, which is how metrics number turns. + + The caller's boundaries are authored rather than detected: the playout queue + drains on a known tick, so these stamp the real edges instead of a + silence-threshold estimate that has to be back-dated (see + BotToBotAudioBridge, whose end detection lags by ~600ms). + """ + seconds = result.wall_clock_ms / 1000 + caller_speaking = scheduler.caller_spoke_this_tick + if caller_speaking and not caller_was_speaking: + self.event_logger.log_audio_start("simulated_user", seconds) + elif not caller_speaking and caller_was_speaking: + self.event_logger.log_audio_end("simulated_user", seconds) + if result.has_assistant_speech and not assistant_was_speaking: + self.event_logger.log_audio_start("assistant", seconds) + elif not result.has_assistant_speech and assistant_was_speaking: + self.event_logger.log_audio_end("assistant", seconds) + def _collect_heard_text(self, scheduler: TickScheduler) -> tuple[str, bool]: """Return what the assistant said and whether to keep waiting for it. diff --git a/tests/unit/user_simulator/cascade/test_simulator.py b/tests/unit/user_simulator/cascade/test_simulator.py index 5c08b065..f127deae 100644 --- a/tests/unit/user_simulator/cascade/test_simulator.py +++ b/tests/unit/user_simulator/cascade/test_simulator.py @@ -165,3 +165,83 @@ def test_missed_utterance_directive_forbids_repeating(): assert "not repeat" in MISSED_UTTERANCE_DIRECTIVE.lower() assert "repeat it" in MISSED_UTTERANCE_DIRECTIVE.lower() + + +def _boundary_simulator(): + """Bare simulator exposing only what _log_audio_boundaries touches.""" + sim = CascadeUserSimulator.__new__(CascadeUserSimulator) + sim.event_logger = _FakeAudioEventLogger() + return sim + + +class _FakeAudioEventLogger: + def __init__(self) -> None: + self.calls: list[tuple[str, str, float]] = [] + + def log_audio_start(self, role, timestamp=None): + self.calls.append(("audio_start", role, timestamp)) + + def log_audio_end(self, role, timestamp=None): + self.calls.append(("audio_end", role, timestamp)) + + +class _Sched: + def __init__(self, spoke: bool) -> None: + self.caller_spoke_this_tick = spoke + + +def _tick(assistant_speech: bool, ms: int = 2000): + from eva.user_simulator.cascade.tick_result import TickResult + + return TickResult( + tick_number=0, + assistant_audio=b"\x00" * 8, + assistant_audio_raw_bytes=8 if assistant_speech else 0, + wall_clock_ms=ms, + ) + + +def test_caller_audio_start_is_logged_on_the_first_tick_of_playout(): + sim = _boundary_simulator() + + sim._log_audio_boundaries(_Sched(True), _tick(False), False, False) + + assert sim.event_logger.calls == [("audio_start", "simulated_user", 2.0)] + + +def test_caller_audio_end_is_logged_when_playout_stops(): + sim = _boundary_simulator() + + sim._log_audio_boundaries(_Sched(False), _tick(False), False, True) + + assert sim.event_logger.calls == [("audio_end", "simulated_user", 2.0)] + + +def test_no_event_while_the_caller_keeps_speaking(): + sim = _boundary_simulator() + + sim._log_audio_boundaries(_Sched(True), _tick(False), False, True) + + assert sim.event_logger.calls == [] + + +def test_assistant_boundaries_are_logged_too(): + # The metrics processor expects both roles, not just the user. + sim = _boundary_simulator() + + sim._log_audio_boundaries(_Sched(False), _tick(True), False, False) + sim._log_audio_boundaries(_Sched(False), _tick(False), True, False) + + assert sim.event_logger.calls == [ + ("audio_start", "assistant", 2.0), + ("audio_end", "assistant", 2.0), + ] + + +def test_timestamp_is_unix_seconds_not_milliseconds(): + # log_audio_* store the value as audio_timestamp, which metrics read as seconds. + sim = _boundary_simulator() + + sim._log_audio_boundaries(_Sched(True), _tick(False, ms=1786127928923), False, False) + + assert sim.event_logger.calls[0][2] == 1786127928.923 From 80f5e800d7954abe22b5ee4c5f42d74b10ba678a Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Fri, 7 Aug 2026 18:53:02 -0400 Subject: [PATCH 28/65] End a stalled cascade conversation with inactivity_timeout An assistant that stops replying was being treated as a transcript problem: the simulator logged a transcript_missed event, told the model it had misheard, and released the awaiting-reply gate so it could speak again. It usually repeated itself instead, producing loops of up to 14 identical utterances and minutes of dead air. Root-caused against the matrix runs: the assistant's STT drops 100% of sub-second user utterances (16/16; 0 transcribed), so a terse reply like "Aisle." never reaches its LLM and it has nothing to answer. The simulator was correctly hearing nothing. Nothing was wrong with our own transcription, and working around the assistant's STT is not this component's job. EVA already models this. conversation_valid_end treats inactivity_timeout with the user speaking last as a definitive terminal state, and ElevenLabsUserSimulator ends the call after 12 keep-alives without activity. Cascade now does the same at the same two-minute threshold, so both providers record the same end state. Removes transcript_missed, MISSED_UTTERANCE_DIRECTIVE and the ASSISTANT_UNRESPONSIVE_MS gate release. The bounded wait and partial-transcript fallback stay: those cover our own STT finalizing late, which is our concern. Co-Authored-By: Claude Opus 5 (1M context) --- src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/constants.py | 6 +- src/eva/user_simulator/cascade/scheduler.py | 15 ++-- src/eva/user_simulator/cascade/simulator.py | 46 +++++++------ .../user_simulator/cascade/test_scheduler.py | 24 ------- .../user_simulator/cascade/test_simulator.py | 69 +++++++++++++------ 6 files changed, 86 insertions(+), 76 deletions(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 21c3bd07..27025203 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.15" +simulation_version = "2.0.16" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/constants.py b/src/eva/user_simulator/cascade/constants.py index b940e81d..9adb448b 100644 --- a/src/eva/user_simulator/cascade/constants.py +++ b/src/eva/user_simulator/cascade/constants.py @@ -20,9 +20,9 @@ """How long the caller waits for the assistant's transcript to finalize before falling back to the in-flight partial, sized above the slowest measured finalization (ink-2, 1.2s).""" -ASSISTANT_UNRESPONSIVE_MS = 90000 -"""Assistant silence after which the caller stops waiting for a reply, set above the longest -legitimate inter-utterance gap measured live (220 ticks) and below the server's idle timeout.""" +INACTIVITY_TIMEOUT_MS = 120000 +"""Assistant silence that ends the conversation, matching ElevenLabsUserSimulator's 12 +keep-alives so both providers record the same inactivity_timeout terminal state.""" CALLER_SAMPLE_RATE = 16000 """PCM16 sample rate for the caller's own audio track.""" diff --git a/src/eva/user_simulator/cascade/scheduler.py b/src/eva/user_simulator/cascade/scheduler.py index b750274b..1990c271 100644 --- a/src/eva/user_simulator/cascade/scheduler.py +++ b/src/eva/user_simulator/cascade/scheduler.py @@ -4,7 +4,6 @@ from eva.user_simulator.cascade.adapter.base import Adapter from eva.user_simulator.cascade.constants import ( - ASSISTANT_UNRESPONSIVE_MS, BYTES_PER_TICK, WAIT_TO_RESPOND_OTHER_MS, WAIT_TO_RESPOND_SELF_MS, @@ -55,6 +54,11 @@ def caller_is_speaking(self) -> bool: """Whether caller audio is still queued for playout.""" return bool(self._playout) + @property + def assistant_has_spoken(self) -> bool: + """Whether the assistant has produced audio at any point in the call.""" + return self._assistant_has_spoken + @property def caller_spoke_this_tick(self) -> bool: """Whether real caller audio went out on the most recent tick. @@ -75,13 +79,10 @@ def may_take_turn(self) -> bool: Also gated on the assistant having replied since the caller's last turn: the silence thresholds alone are satisfied a fixed time after the caller stops talking regardless of whether a reply ever arrived, which lets the - caller repeat itself into a slow assistant. That gate releases only after - ASSISTANT_UNRESPONSIVE_MS, so an assistant that stops answering entirely - cannot strand the caller before it reaches its end_call turn. + caller repeat itself into a slow assistant. An assistant that stops replying + altogether is an inactivity timeout, which the simulator ends the call on. """ - if not self._assistant_has_spoken: - return False - if self._awaiting_reply and self._ticks_since_assistant_speech <= ms_to_ticks(ASSISTANT_UNRESPONSIVE_MS): + if not self._assistant_has_spoken or self._awaiting_reply: return False return self._ticks_since_assistant_speech > ms_to_ticks( WAIT_TO_RESPOND_OTHER_MS diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py index 23e6778d..2f53df44 100644 --- a/src/eva/user_simulator/cascade/simulator.py +++ b/src/eva/user_simulator/cascade/simulator.py @@ -13,6 +13,7 @@ from eva.user_simulator.cascade.adapter.realtime_ws import RealtimeWSAdapter from eva.user_simulator.cascade.constants import ( CALLER_SAMPLE_RATE, + INACTIVITY_TIMEOUT_MS, TICK_DURATION_MS, TRANSCRIPT_WAIT_MS, ms_to_ticks, @@ -30,10 +31,6 @@ _FENCE = re.compile(r"^```[a-z]*\s*|\s*```$", re.MULTILINE) -MISSED_UTTERANCE_DIRECTIVE = """You did not hear what the agent just said — the audio did not -come through. Do NOT repeat your previous message. Say briefly that you did not catch that and -ask them to repeat it, the way anyone would on a bad phone line.""" - END_CALL_TOOL = { "type": "function", "function": { @@ -105,7 +102,6 @@ def __init__( self._voice_id = self._tts.voice_for_persona(persona_config) self._history: list[dict[str, str]] = [] self._ticks_awaiting_transcript = 0 - self._missed_transcripts = 0 async def run_conversation(self) -> str: """Run the tick loop until the call ends, and return the end reason.""" @@ -155,6 +151,13 @@ async def _run(self) -> None: assistant_was_speaking = result.has_assistant_speech if result.has_assistant_speech: continue + if self._assistant_is_inactive(scheduler, result): + logger.warning( + f"tick {scheduler.tick}: assistant silent for " + f"{INACTIVITY_TIMEOUT_MS // 1000}s; ending the conversation" + ) + self._on_conversation_end("inactivity_timeout") + break if scheduler.caller_is_speaking or not scheduler.may_take_turn(): continue heard, waiting = self._collect_heard_text(scheduler) @@ -170,6 +173,19 @@ async def _run(self) -> None: await adapter.stop() self.event_logger.log_connection_state("session_ended", {"reason": self._end_reason}) + def _assistant_is_inactive(self, scheduler: TickScheduler, result: TickResult) -> bool: + """Whether the assistant has produced no audio for INACTIVITY_TIMEOUT_MS. + + Mirrors ElevenLabsUserSimulator's keep-alive rule so both providers record the + same terminal state: conversation_valid_end treats inactivity_timeout with the + user speaking last as a definitive end, not a failure. + """ + if result.has_assistant_speech: + self._ticks_assistant_silent = 0 + return False + self._ticks_assistant_silent += 1 + return scheduler.assistant_has_spoken and self._ticks_assistant_silent > ms_to_ticks(INACTIVITY_TIMEOUT_MS) + def _log_audio_boundaries( self, scheduler: TickScheduler, @@ -223,13 +239,9 @@ def _collect_heard_text(self, scheduler: TickScheduler) -> tuple[str, bool]: self.event_logger.log_event("transcript_partial_fallback", {"text": partial, "tick_index": scheduler.tick}) return partial, False - self._missed_transcripts += 1 - logger.error( - f"tick {scheduler.tick}: heard nothing at all from the assistant this turn " - f"(missed {self._missed_transcripts} so far); asking it to repeat" - ) - self.event_logger.log_event("transcript_missed", {"tick_index": scheduler.tick}) - return "", False + # Nothing was heard at all. Keep waiting rather than speaking into the void: + # an assistant that never replies is an inactivity timeout, handled in _run. + return "", True async def _take_turn(self, scheduler: TickScheduler, heard: str) -> bool: """Generate, synthesize, and queue one caller turn. Returns True to hang up.""" @@ -237,9 +249,7 @@ async def _take_turn(self, scheduler: TickScheduler, heard: str) -> bool: self._history.append({"role": "assistant", "content": heard}) self._on_assistant_speaks(heard) - message, _stats = await self._llm.complete( - messages=self._messages(missed_utterance=not heard), tools=[END_CALL_TOOL] - ) + message, _stats = await self._llm.complete(messages=self._messages(), tools=[END_CALL_TOOL]) utterance, end_call = extract_turn(message) if utterance: @@ -277,7 +287,7 @@ def _warn_unsupported_perturbation(perturbation_config: PerturbationConfig | Non "Behavior and accent perturbations are unaffected." ) - def _messages(self, *, missed_utterance: bool = False) -> list[dict[str, str]]: + def _messages(self) -> list[dict[str, str]]: """Build the message list: the shared per-domain caller prompt plus flipped history. The system prompt is `_build_prompt()` unmodified — the same per-domain prompt the other @@ -290,8 +300,4 @@ def _messages(self, *, missed_utterance: bool = False) -> list[dict[str, str]]: """ messages = [{"role": "system", "content": self._build_prompt()}] messages += [{"role": _flip_role(turn["role"]), "content": turn["content"]} for turn in self._history] - if missed_utterance: - # History is unchanged since the last turn, so without this the model would - # regenerate its previous utterance verbatim. - messages.append({"role": "system", "content": MISSED_UTTERANCE_DIRECTIVE}) return messages diff --git a/tests/unit/user_simulator/cascade/test_scheduler.py b/tests/unit/user_simulator/cascade/test_scheduler.py index d5a40fee..9e7943d5 100644 --- a/tests/unit/user_simulator/cascade/test_scheduler.py +++ b/tests/unit/user_simulator/cascade/test_scheduler.py @@ -1,5 +1,4 @@ from eva.user_simulator.cascade.adapter.base import Adapter -from eva.user_simulator.cascade.constants import ASSISTANT_UNRESPONSIVE_MS, ms_to_ticks from eva.user_simulator.cascade.scheduler import TickScheduler from eva.user_simulator.cascade.tick_result import TickResult @@ -103,29 +102,6 @@ async def test_caller_cannot_take_a_second_turn_while_awaiting_a_reply(): assert scheduler.may_take_turn() is False -async def test_caller_stops_waiting_once_the_assistant_goes_unresponsive(): - # The assistant that never answers a goodbye would otherwise hold the caller - # in awaiting-reply forever, so it could never reach its end_call turn. - scheduler = _scheduler([True]) - await scheduler.run_tick() - scheduler.enqueue_utterance(b"\x02" * BYTES_PER_TICK) - await scheduler.run_tick() - - for _ in range(ms_to_ticks(ASSISTANT_UNRESPONSIVE_MS) - 5): - await scheduler.run_tick() - assert scheduler.may_take_turn() is False - - for _ in range(10): - await scheduler.run_tick() - assert scheduler.may_take_turn() is True - - -async def test_unresponsive_threshold_clears_the_longest_observed_real_gap(): - # Longest legitimate assistant gap measured across live runs was 220 ticks; - # firing inside that would make the caller talk over a merely slow assistant. - assert ms_to_ticks(ASSISTANT_UNRESPONSIVE_MS) > 220 - - async def test_caller_may_take_a_second_turn_once_the_assistant_replies(): scheduler = _scheduler([True, False, True] + [False] * 30) await scheduler.run_tick() # tick 0: assistant greets diff --git a/tests/unit/user_simulator/cascade/test_simulator.py b/tests/unit/user_simulator/cascade/test_simulator.py index f127deae..5f71e4d4 100644 --- a/tests/unit/user_simulator/cascade/test_simulator.py +++ b/tests/unit/user_simulator/cascade/test_simulator.py @@ -136,20 +136,6 @@ def test_wait_expires_into_the_in_flight_partial(): assert sim._stt.buffer.in_flight == "" -def test_hearing_nothing_at_all_is_reported_and_never_reuses_stale_history(): - from eva.user_simulator.cascade.constants import TRANSCRIPT_WAIT_MS, ms_to_ticks - - sim = _simulator_with_buffer() - for _ in range(ms_to_ticks(TRANSCRIPT_WAIT_MS)): - sim._collect_heard_text(_FakeScheduler()) - - heard, waiting = sim._collect_heard_text(_FakeScheduler()) - - assert (heard, waiting) == ("", False) - assert sim._missed_transcripts == 1 - assert [name for name, _ in sim.event_logger.events] == ["transcript_missed"] - - def test_the_wait_counter_resets_after_a_successful_read(): sim = _simulator_with_buffer() sim._collect_heard_text(_FakeScheduler()) @@ -160,13 +146,6 @@ def test_the_wait_counter_resets_after_a_successful_read(): assert sim._ticks_awaiting_transcript == 0 -def test_missed_utterance_directive_forbids_repeating(): - from eva.user_simulator.cascade.simulator import MISSED_UTTERANCE_DIRECTIVE - - assert "not repeat" in MISSED_UTTERANCE_DIRECTIVE.lower() - assert "repeat it" in MISSED_UTTERANCE_DIRECTIVE.lower() - - def _boundary_simulator(): """Bare simulator exposing only what _log_audio_boundaries touches.""" sim = CascadeUserSimulator.__new__(CascadeUserSimulator) @@ -245,3 +224,51 @@ def test_timestamp_is_unix_seconds_not_milliseconds(): sim._log_audio_boundaries(_Sched(True), _tick(False, ms=1786127928923), False, False) assert sim.event_logger.calls[0][2] == 1786127928.923 + + +def test_hearing_nothing_keeps_waiting_instead_of_speaking_into_the_void(): + # An assistant that never replies is an inactivity timeout, not a cue to talk again. + from eva.user_simulator.cascade.constants import TRANSCRIPT_WAIT_MS, ms_to_ticks + + sim = _simulator_with_buffer() + for _ in range(ms_to_ticks(TRANSCRIPT_WAIT_MS) + 3): + assert sim._collect_heard_text(_FakeScheduler()) == ("", True) + + +class _SilenceScheduler: + tick = 0 + assistant_has_spoken = True + + +def test_inactivity_ends_the_call_after_the_shared_two_minute_threshold(): + from eva.user_simulator.cascade.constants import INACTIVITY_TIMEOUT_MS, ms_to_ticks + + sim = CascadeUserSimulator.__new__(CascadeUserSimulator) + sim._ticks_assistant_silent = 0 + silent = _tick(False) + for _ in range(ms_to_ticks(INACTIVITY_TIMEOUT_MS)): + assert sim._assistant_is_inactive(_SilenceScheduler(), silent) is False + + assert sim._assistant_is_inactive(_SilenceScheduler(), silent) is True + + +def test_assistant_speech_resets_the_inactivity_counter(): + sim = CascadeUserSimulator.__new__(CascadeUserSimulator) + sim._ticks_assistant_silent = 500 + + assert sim._assistant_is_inactive(_SilenceScheduler(), _tick(True)) is False + assert sim._ticks_assistant_silent == 0 + + +def test_inactivity_does_not_fire_before_the_assistant_ever_speaks(): + # The assistant opens the call; waiting for its greeting is not inactivity. + from eva.user_simulator.cascade.constants import INACTIVITY_TIMEOUT_MS, ms_to_ticks + + class _NeverSpoke: + tick = 0 + assistant_has_spoken = False + + sim = CascadeUserSimulator.__new__(CascadeUserSimulator) + sim._ticks_assistant_silent = 0 + for _ in range(ms_to_ticks(INACTIVITY_TIMEOUT_MS) + 5): + assert sim._assistant_is_inactive(_NeverSpoke(), _tick(False)) is False From ea6a7fda4a2a9678423dc580e4adf6ed1db960bd Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Fri, 7 Aug 2026 19:01:48 -0400 Subject: [PATCH 29/65] Fix uninitialised tick counters in the cascade simulator _ticks_assistant_silent was never initialised, so the first silent tick raised AttributeError and failed the run. The unit tests missed it because they build bare instances with __new__ and set the counters by hand, testing around the gap; both counters are now class-level defaults and a test asserts they exist without manual setup. Also restores the wait-counter reset inside _collect_heard_text, which a scripted edit removed by mistake, leaving the counter climbing across turns. Verified live: assistant stalls, conversation ends inactivity_timeout, and conversation_valid_end scores 1.0 as agent_timeout_on_user_turn. Co-Authored-By: Claude Opus 5 (1M context) --- src/eva/user_simulator/cascade/simulator.py | 5 +++-- tests/unit/user_simulator/cascade/test_simulator.py | 12 +++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py index 2f53df44..7de87b34 100644 --- a/src/eva/user_simulator/cascade/simulator.py +++ b/src/eva/user_simulator/cascade/simulator.py @@ -68,6 +68,9 @@ def extract_turn(message: object) -> tuple[str, bool]: class CascadeUserSimulator(AbstractUserSimulator): """Simulated caller built from independently chosen STT, LLM, and TTS models.""" + _ticks_awaiting_transcript = 0 + _ticks_assistant_silent = 0 + def __init__( self, current_date_time: str, @@ -101,7 +104,6 @@ def __init__( self._llm = LiteLLMClient(model=simulator_config.llm) self._voice_id = self._tts.voice_for_persona(persona_config) self._history: list[dict[str, str]] = [] - self._ticks_awaiting_transcript = 0 async def run_conversation(self) -> str: """Run the tick loop until the call ends, and return the end reason.""" @@ -228,7 +230,6 @@ def _collect_heard_text(self, scheduler: TickScheduler) -> tuple[str, bool]: if self._ticks_awaiting_transcript <= ms_to_ticks(TRANSCRIPT_WAIT_MS): return "", True - self._ticks_awaiting_transcript = 0 partial = self._stt.buffer.in_flight self._stt.buffer.in_flight = "" if partial: diff --git a/tests/unit/user_simulator/cascade/test_simulator.py b/tests/unit/user_simulator/cascade/test_simulator.py index 5f71e4d4..7dc696bc 100644 --- a/tests/unit/user_simulator/cascade/test_simulator.py +++ b/tests/unit/user_simulator/cascade/test_simulator.py @@ -138,7 +138,8 @@ def test_wait_expires_into_the_in_flight_partial(): def test_the_wait_counter_resets_after_a_successful_read(): sim = _simulator_with_buffer() - sim._collect_heard_text(_FakeScheduler()) + for _ in range(3): + sim._collect_heard_text(_FakeScheduler()) sim._stt.buffer.committed = "Thanks, Marcus." sim._collect_heard_text(_FakeScheduler()) @@ -272,3 +273,12 @@ class _NeverSpoke: sim._ticks_assistant_silent = 0 for _ in range(ms_to_ticks(INACTIVITY_TIMEOUT_MS) + 5): assert sim._assistant_is_inactive(_NeverSpoke(), _tick(False)) is False + + +def test_tick_counters_exist_without_manual_setup(): + # The unit tests build bare instances, so a counter initialised only inside __init__ + # would still pass them and then AttributeError on the first live tick. + sim = CascadeUserSimulator.__new__(CascadeUserSimulator) + + assert sim._ticks_assistant_silent == 0 + assert sim._ticks_awaiting_transcript == 0 From ecae1877f421581c339c5a54f7798a500358ec4c Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Fri, 7 Aug 2026 19:19:20 -0400 Subject: [PATCH 30/65] add out-of-turn behavior constants --- docs/changelog_cascade_out_of_turn.md | 54 +++++++++++++++++++ src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/constants.py | 18 +++++++ .../user_simulator/cascade/test_constants.py | 21 ++++++++ 4 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 docs/changelog_cascade_out_of_turn.md diff --git a/docs/changelog_cascade_out_of_turn.md b/docs/changelog_cascade_out_of_turn.md new file mode 100644 index 00000000..d1316a5b --- /dev/null +++ b/docs/changelog_cascade_out_of_turn.md @@ -0,0 +1,54 @@ +# Working log: cascade out-of-turn behaviors (Plan 2) + +Plan: `docs/superpowers/plans/2026-08-06-cascade-out-of-turn-behaviors.md` +Handoff: `docs/handoff_cascade_plan2.md` + +## Deviations from the plan, decided up front + +### Structured fields come from a separate follow-up call, not a JSON turn contract + +The plan's Tasks 10 and 12 read `self_correction` and `next_interruption` out of a JSON +turn response. That contract does not exist: Plan 1 removed it because demanding "a single +JSON object and nothing else" suppressed the `end_call` tool call and hung conversations +until timeout. + +Instead the turn call keeps its exact current shape (plain text + `END_CALL_TOOL`), and a +second call — made *after* the turn's audio is already queued — asks only for the extra +field. Reasoning: + +- It structurally cannot regress `end_call`, because the turn call is untouched. +- It costs no conversational latency. The correction is not needed until + `SELF_CORRECTION_DELAY_MS` (1200ms) after the assistant *starts replying*, which is + itself well after the caller's audio went out. +- The user additionally required that a self-correction never attach to a final turn, so + arming is gated on `end_call` being false. + +Certainty: high. This is strictly safer than the plan's version and the latency argument is +structural, not empirical. + +### Re-added `TranscriptBuffer.current_text()` + +Deleted as dead code at the end of Plan 1; Tasks 7, 9, and 12 all call it. Restored with the +`[CURRENTLY SPEAKING, INCOMPLETE]` marker the backchannel prompt's few-shot examples depend on. + +Certainty: high — the prompt examples are meaningless without the marker. + +### Task 13 stages explicitly rather than `git add -A` + +The handoff forbids `git add -A` at the repo root (many unrelated untracked files). + +## Progress + +- [ ] Task 1: behavior constants and vocabularies +- [ ] Task 2: behavior flags on the config +- [ ] Task 3: check-tick predicate +- [ ] Task 4: decision prompts (+ `current_text()`) +- [ ] Task 5: decision checks +- [ ] Task 6: phrase cache +- [ ] Task 7: backchannel behavior +- [ ] Task 8: streaming TTS +- [ ] Task 9: reactive interruption +- [ ] Task 10: self-correction +- [ ] Task 11: ambient noise mixing +- [ ] Task 12: speculative generation +- [ ] Task 13: live ablation verification diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 27025203..09d8aebf 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.16" +simulation_version = "2.0.17" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/constants.py b/src/eva/user_simulator/cascade/constants.py index 9adb448b..4a531cdd 100644 --- a/src/eva/user_simulator/cascade/constants.py +++ b/src/eva/user_simulator/cascade/constants.py @@ -35,6 +35,24 @@ SILENCE_BYTE = b"\x00" """PCM16 silence, used to pad partial ticks.""" +LISTENER_CHECK_INTERVAL_MS = 2000 +"""How often the interrupt and backchannel checks run while the assistant speaks.""" + +MAX_INTERRUPT_SLIP_MS = 1500 +"""Drop a reactive barge-in whose audio arrived this far past its intended tick.""" + +SELF_CORRECTION_DELAY_MS = 1200 +"""How long after the assistant starts replying to play a pre-authored correction.""" + +SELF_CORRECTION_RATE = 0.15 +"""Fraction of caller turns generated with a self-correction attached.""" + +BACKCHANNEL_PHRASES = ["uh-huh", "mm-hmm"] +"""Fixed continuer vocabulary (tau: voice_config.py:126). Pre-rendered at init.""" + +BARGE_IN_OPENERS = ["Wait—", "Sorry—", "Hold on—", "Actually—"] +"""Fixed barge-in openers. Pre-rendered so a decision can be voiced at zero latency.""" + def ms_to_ticks(milliseconds: int) -> int: """Convert milliseconds to whole ticks, flooring.""" diff --git a/tests/unit/user_simulator/cascade/test_constants.py b/tests/unit/user_simulator/cascade/test_constants.py index cec8808b..e34196f7 100644 --- a/tests/unit/user_simulator/cascade/test_constants.py +++ b/tests/unit/user_simulator/cascade/test_constants.py @@ -25,3 +25,24 @@ def test_threshold_constants_are_exact_multiples_of_tick_duration(): def test_ms_to_ticks_converts_and_floors_sub_tick_remainder(): assert ms_to_ticks(WAIT_TO_RESPOND_OTHER_MS) == 5 assert ms_to_ticks(150) == 0 + + +def test_listener_check_interval_is_two_seconds_in_ticks(): + from eva.user_simulator.cascade.constants import LISTENER_CHECK_INTERVAL_MS, ms_to_ticks + + assert LISTENER_CHECK_INTERVAL_MS == 2000 + assert ms_to_ticks(LISTENER_CHECK_INTERVAL_MS) == 10 + + +def test_fixed_vocabularies_are_non_empty(): + from eva.user_simulator.cascade.constants import BACKCHANNEL_PHRASES, BARGE_IN_OPENERS + + assert BACKCHANNEL_PHRASES == ["uh-huh", "mm-hmm"] + assert len(BARGE_IN_OPENERS) >= 2 + + +def test_self_correction_delay_is_shorter_than_the_check_interval(): + from eva.user_simulator.cascade.constants import LISTENER_CHECK_INTERVAL_MS, SELF_CORRECTION_DELAY_MS + + # The correction should land while the assistant is still on its first reply. + assert SELF_CORRECTION_DELAY_MS < LISTENER_CHECK_INTERVAL_MS From 89bbb64e488a3e5163c2b9b1f98de931d7661943 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Fri, 7 Aug 2026 19:20:08 -0400 Subject: [PATCH 31/65] add cascade behavior flags --- src/eva/__init__.py | 2 +- src/eva/models/config.py | 19 +++++++++++++++++ tests/unit/models/test_config_models.py | 28 +++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 09d8aebf..7afc540f 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.17" +simulation_version = "2.0.18" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/models/config.py b/src/eva/models/config.py index d094a419..c1941a42 100644 --- a/src/eva/models/config.py +++ b/src/eva/models/config.py @@ -498,6 +498,25 @@ class CascadeSimulatorConfig(BaseModel): description="Provider-native keyword arguments passed through to the TTS client.", ) + decision_llm: str = Field( + "user-llm", + description=( + "Model for the YES/NO interrupt, backchannel, and relevance checks. Defaults to the " + "caller's own deployment so it resolves through the same EVA_MODEL_LIST router; point " + "it at a cheaper deployment to cut the cost of the per-tick checks." + ), + ) + + enable_backchannel: bool = Field(False, description="Caller emits continuers while the assistant speaks.") + enable_interruptions: bool = Field(False, description="Caller may barge in reacting to the assistant mid-turn.") + enable_self_correction: bool = Field( + False, description="Caller may reverse its own prior statement, pre-authored and fired on a timer." + ) + speculative_generation: bool = Field( + False, + description="Pre-render a candidate interruption on the turn call, gated by a relevance check before firing.", + ) + UserSimulatorConfig = Annotated[ ElevenLabsSimulatorConfig | OpenAIRealtimeSimulatorConfig | CascadeSimulatorConfig, diff --git a/tests/unit/models/test_config_models.py b/tests/unit/models/test_config_models.py index 8fceec3f..6c6df440 100644 --- a/tests/unit/models/test_config_models.py +++ b/tests/unit/models/test_config_models.py @@ -1270,3 +1270,31 @@ def test_user_simulator_union_discriminates_cascade(): parsed = TypeAdapter(UserSimulatorConfig).validate_python({"provider": "cascade"}) assert isinstance(parsed, CascadeSimulatorConfig) + + +def test_cascade_behaviors_default_off(): + from eva.models.config import CascadeSimulatorConfig + + config = CascadeSimulatorConfig() + + assert config.enable_backchannel is False + assert config.enable_interruptions is False + assert config.enable_self_correction is False + assert config.speculative_generation is False + + +def test_cascade_behaviors_can_be_enabled_independently(): + from eva.models.config import CascadeSimulatorConfig + + config = CascadeSimulatorConfig(enable_backchannel=True) + + assert config.enable_backchannel is True + assert config.enable_interruptions is False + + +def test_cascade_decision_llm_defaults_to_the_caller_llm(): + from eva.models.config import CascadeSimulatorConfig + + config = CascadeSimulatorConfig() + + assert config.decision_llm == config.llm From 2fefd2fcd33e80b396af0592d1203caa8a08b0bb Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Fri, 7 Aug 2026 19:20:52 -0400 Subject: [PATCH 32/65] add check-tick predicate to the scheduler --- src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/scheduler.py | 12 ++++++ .../user_simulator/cascade/test_scheduler.py | 38 +++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 7afc540f..b00547d1 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.18" +simulation_version = "2.0.19" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/scheduler.py b/src/eva/user_simulator/cascade/scheduler.py index 1990c271..f104bee0 100644 --- a/src/eva/user_simulator/cascade/scheduler.py +++ b/src/eva/user_simulator/cascade/scheduler.py @@ -5,6 +5,7 @@ from eva.user_simulator.cascade.adapter.base import Adapter from eva.user_simulator.cascade.constants import ( BYTES_PER_TICK, + LISTENER_CHECK_INTERVAL_MS, WAIT_TO_RESPOND_OTHER_MS, WAIT_TO_RESPOND_SELF_MS, ms_to_ticks, @@ -69,6 +70,17 @@ def caller_spoke_this_tick(self) -> bool: """ return self._caller_spoke_this_tick + @property + def assistant_is_speaking(self) -> bool: + """Whether the assistant produced audio on the most recent tick.""" + return self._ticks_since_assistant_speech == 0 + + def is_check_tick(self) -> bool: + """Whether the listener-reaction checks should run now (tau: streaming.py:2514-2521).""" + if not self.assistant_is_speaking or self.caller_is_speaking: + return False + return self.tick % ms_to_ticks(LISTENER_CHECK_INTERVAL_MS) == 0 + def may_take_turn(self) -> bool: """Whether both silence thresholds are satisfied (tau: streaming.py:2590-2606). diff --git a/tests/unit/user_simulator/cascade/test_scheduler.py b/tests/unit/user_simulator/cascade/test_scheduler.py index 9e7943d5..6e6e7a43 100644 --- a/tests/unit/user_simulator/cascade/test_scheduler.py +++ b/tests/unit/user_simulator/cascade/test_scheduler.py @@ -222,3 +222,41 @@ async def test_failed_adapter_call_leaves_queue_and_tick_unadvanced(): assert scheduler.tick == 0 assert bytes(scheduler._playout) == utterance + + +async def test_check_tick_only_while_assistant_speaks_and_caller_is_silent(): + scheduler = _scheduler([True] * 40) + + # Tick 0..9 consumed; tick index 10 is the first multiple of the interval. + for _ in range(10): + await scheduler.run_tick() + + assert scheduler.is_check_tick() is True + + +async def test_not_a_check_tick_between_intervals(): + scheduler = _scheduler([True] * 40) + + for _ in range(11): + await scheduler.run_tick() + + assert scheduler.is_check_tick() is False + + +async def test_not_a_check_tick_when_the_assistant_is_silent(): + scheduler = _scheduler([True] + [False] * 40) + + for _ in range(10): + await scheduler.run_tick() + + assert scheduler.is_check_tick() is False + + +async def test_not_a_check_tick_while_the_caller_is_speaking(): + scheduler = _scheduler([True] * 40) + for _ in range(9): + await scheduler.run_tick() + scheduler.enqueue_utterance(b"\x02" * (BYTES_PER_TICK * 5)) + await scheduler.run_tick() + + assert scheduler.is_check_tick() is False From 8e2fc954cc8665a2d9ac5b41483a9e650b192436 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Fri, 7 Aug 2026 19:22:17 -0400 Subject: [PATCH 33/65] port tau interrupt and backchannel decision prompts --- configs/prompts/simulation.yaml | 77 +++++++++++++++++++ src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/stt.py | 11 +++ .../user_simulator/cascade/test_prompt.py | 19 +++++ tests/unit/user_simulator/cascade/test_stt.py | 24 ++++++ 5 files changed, 132 insertions(+), 1 deletion(-) diff --git a/configs/prompts/simulation.yaml b/configs/prompts/simulation.yaml index 4648cb62..81580366 100644 --- a/configs/prompts/simulation.yaml +++ b/configs/prompts/simulation.yaml @@ -507,3 +507,80 @@ user_simulator: For languages that use non-Latin scripts, spell out characters using their standard phonetic names in your language. IMPORTANT: Before ending the conversation, confirm with the agent that there are no outstanding actions. The end_call tool should only be called in a turn that is a brief goodbye — never in the same turn where you are providing the agent with data, an identifier, a request to transfer to a live agent, an approval to proceed, or any kind of additional information. + + interruption_decision: | + You are analyzing a conversation to decide if the user should interrupt the agent. + + Conversation history (most recent at bottom): + + + {conversation_history} + + + The agent is CURRENTLY speaking (you can see their ongoing speech in the conversation above). + + Based on the conversation so far, should the user interrupt the agent NOW? + + Consider: + - Has the user heard enough to understand what the agent is asking or saying? + - Has the user heard enough to have a response, question, or correction ready? + - Did the agent just complete the sentence which has all the pertinent information the user was looking for? + - Do NOT repeatedly interrupt the agent if it has spoken only a few words (say less than 5 words). + + Respond with ONLY "YES" if the user should interrupt now, or "NO" if they should keep listening. + + backchannel_decision: | + You simulate a natural listener who occasionally says "uh-huh" or "mm-hmm" to show they're following along. + + + {conversation_history} + + + The agent is still speaking [CURRENTLY SPEAKING, INCOMPLETE]. Ignore the trailing incomplete word/phrase — focus only on the COMPLETE sentences delivered so far in the agent's current turn. + + Continuers ("uh-huh", "mm-hmm", "yeah") are brief sounds that mean "I'm listening, keep going." They: + - Happen naturally during extended speech + - Show engagement without interrupting + - Are NOT responses to specific content — just signals of attention + + Say YES if: + - The agent has completed at least 2 full, substantive sentences in their current turn + (Short phrases like "Thanks for your patience" or "Let me check on that" don't count as substantive) + - The user hasn't spoken or backchanneled recently (check the last 3 exchanges for ANY user sound including "mm-hmm", "uh-huh", "okay") + - It would feel natural to briefly signal "I'm still here" + + Say NO if: + - The agent just started speaking (fewer than 2 substantive sentences) + - The user spoke OR backchanneled within the last 2-3 exchanges + - The agent's current turn contains or ends with a question + - The agent is wrapping up or about to finish their thought + + Frequency guidance: + - Continuers are occasional, not constant + - Even when conditions seem right, real listeners only backchannel sometimes + - Aim for roughly 1 continuer per 4-6 sentences of extended agent speech + - When in doubt, say NO — silence is also natural + - Too few continuers is better than too many + + Examples: + + AGENT: "Hi there! How can I hel [CURRENTLY SPEAKING, INCOMPLETE]" + → NO (just started) + + AGENT: "Thanks for your patience. [CURRENTLY SPEAKING, INCOMPLETE]" + → NO (only 1 short sentence, not substantive enough) + + AGENT: "Sure, I can help with that. First I'll need to verify your account. Could you provide your email or your name and zi [CURRENTLY SPEAKING, INCOMPLETE]" + → NO (agent is asking a question) + + AGENT: "No problem. We can use your name and zip code instead. Let me look that up for you. I'll check our system now and see if I can fin [CURRENTLY SPEAKING, INCOMPLETE]" + → YES (3 substantive sentences, agent explaining process) + + AGENT: "I found your order. It includes a keyboard, thermostat, and headphones. The order was delivered last Tuesday. Now for the exchange, we have a few opti [CURRENTLY SPEAKING, INCOMPLETE]" + → YES (extended explanation with specific details) + + [If user said "mm-hmm" 2 exchanges ago] + AGENT: "...and those are the available options. Now I'll need your input on which [CURRENTLY SPEAKING, INCOMPLETE]" + → NO (user backchanneled recently, don't do it again so soon) + + Respond with ONLY "YES" or "NO". diff --git a/src/eva/__init__.py b/src/eva/__init__.py index b00547d1..353b894f 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.19" +simulation_version = "2.0.20" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/stt.py b/src/eva/user_simulator/cascade/stt.py index bf11e4de..50fb5de4 100644 --- a/src/eva/user_simulator/cascade/stt.py +++ b/src/eva/user_simulator/cascade/stt.py @@ -19,6 +19,17 @@ def commit(self, text: str) -> None: self.committed = f"{self.committed} {text}".strip() if self.committed else text self.in_flight = "" + def current_text(self) -> str: + """Return everything heard so far, marking the partial as still in progress. + + The marker is load-bearing: the backchannel prompt's few-shot examples all + end in it, and it is what tells the check to judge only the complete + sentences rather than the truncated trailing word. + """ + if not self.in_flight: + return self.committed + return f"{self.committed} {self.in_flight} [CURRENTLY SPEAKING, INCOMPLETE]".strip() + def take_committed(self) -> str: """Return and clear the committed text.""" text = self.committed diff --git a/tests/unit/user_simulator/cascade/test_prompt.py b/tests/unit/user_simulator/cascade/test_prompt.py index ce7b4ae4..deaa2748 100644 --- a/tests/unit/user_simulator/cascade/test_prompt.py +++ b/tests/unit/user_simulator/cascade/test_prompt.py @@ -1,3 +1,6 @@ +from eva.utils.prompt_manager import PromptManager + + def test_cascade_reuses_the_shared_end_call_description(): # A cascade-specific copy would drift from the other providers' hang-up rules. from eva.user_simulator.cascade.simulator import END_CALL_DESCRIPTION as cascade_description @@ -12,3 +15,19 @@ def test_no_cascade_specific_prompts_remain_in_the_prompt_file(): from pathlib import Path assert "cascade_" not in Path("configs/prompts/simulation.yaml").read_text() + + +def test_interruption_decision_prompt_has_a_history_slot_and_binary_contract(): + prompt = PromptManager().get_prompt("user_simulator.interruption_decision", conversation_history="AGENT: hello") + + assert "AGENT: hello" in prompt + assert "YES" in prompt + assert "NO" in prompt + + +def test_backchannel_decision_prompt_has_a_history_slot_and_frequency_guidance(): + prompt = PromptManager().get_prompt("user_simulator.backchannel_decision", conversation_history="AGENT: hello") + + assert "AGENT: hello" in prompt + assert "CURRENTLY SPEAKING, INCOMPLETE" in prompt + assert "When in doubt, say NO" in prompt diff --git a/tests/unit/user_simulator/cascade/test_stt.py b/tests/unit/user_simulator/cascade/test_stt.py index 6c07f804..facbd2d8 100644 --- a/tests/unit/user_simulator/cascade/test_stt.py +++ b/tests/unit/user_simulator/cascade/test_stt.py @@ -36,3 +36,27 @@ def test_take_committed_drains_the_buffer(): assert buffer.take_committed() == "All done." assert buffer.committed == "" + + +def test_current_text_marks_the_in_flight_partial_as_incomplete(): + buffer = TranscriptBuffer() + buffer.commit("I found your order.") + buffer.apply_partial("It includes a keyboa") + + assert buffer.current_text() == "I found your order. It includes a keyboa [CURRENTLY SPEAKING, INCOMPLETE]" + + +def test_current_text_omits_the_marker_when_nothing_is_in_flight(): + buffer = TranscriptBuffer() + buffer.commit("I found your order.") + + assert buffer.current_text() == "I found your order." + + +def test_current_text_does_not_consume_the_committed_text(): + buffer = TranscriptBuffer() + buffer.commit("hello") + + buffer.current_text() + + assert buffer.take_committed() == "hello" From 89d355e9c873df788115f162e63f752b86401168 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Fri, 7 Aug 2026 19:23:33 -0400 Subject: [PATCH 34/65] add listener-reaction decision checks --- src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/decisions.py | 66 ++++++++++++++++ .../user_simulator/cascade/test_decisions.py | 79 +++++++++++++++++++ 3 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 src/eva/user_simulator/cascade/decisions.py create mode 100644 tests/unit/user_simulator/cascade/test_decisions.py diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 353b894f..1d2cb4a2 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.20" +simulation_version = "2.0.21" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/decisions.py b/src/eva/user_simulator/cascade/decisions.py new file mode 100644 index 00000000..81e5418e --- /dev/null +++ b/src/eva/user_simulator/cascade/decisions.py @@ -0,0 +1,66 @@ +"""Listener-reaction checks: should the caller interrupt or backchannel right now.""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from typing import Protocol + +from eva.utils.logging import get_logger + +logger = get_logger(__name__) + + +class DecisionLLM(Protocol): + """Minimal interface the checks need from a model client.""" + + async def decide(self, prompt: str) -> str: + """Return the model's raw reply to a YES/NO question.""" + ... + + +@dataclass(frozen=True) +class ListenerVerdict: + """Outcome of one check tick.""" + + should_interrupt: bool + should_backchannel: bool + + +def parse_yes_no(raw: str) -> bool: + """Read a bare YES/NO reply. Anything unrecognized counts as NO.""" + return raw.strip().upper() == "YES" + + +class ListenerDecisions: + """Runs the interrupt and backchannel checks concurrently against the partial transcript. + + Both fail closed: an exception yields "don't act", so a provider hiccup can + never inject a barge-in that the caller never actually decided to make. + """ + + def __init__(self, llm: DecisionLLM, *, interrupt_prompt: str, backchannel_prompt: str) -> None: + self._llm = llm + self._interrupt_prompt = interrupt_prompt + self._backchannel_prompt = backchannel_prompt + + async def evaluate( + self, conversation_history: str, *, allow_interrupt: bool, allow_backchannel: bool + ) -> ListenerVerdict: + """Run whichever checks are enabled. Interrupt wins ties (tau: streaming.py:2549).""" + interrupt, backchannel = await asyncio.gather( + self._check(self._interrupt_prompt, conversation_history, enabled=allow_interrupt), + self._check(self._backchannel_prompt, conversation_history, enabled=allow_backchannel), + ) + return ListenerVerdict(should_interrupt=interrupt, should_backchannel=backchannel and not interrupt) + + async def _check(self, template: str, conversation_history: str, *, enabled: bool) -> bool: + """Ask the model one YES/NO question, returning False on anything unexpected.""" + if not enabled: + return False + try: + reply = await self._llm.decide(template.format(conversation_history=conversation_history)) + except Exception as exc: + logger.warning(f"Listener check failed, defaulting to no action: {exc}") + return False + return parse_yes_no(reply) diff --git a/tests/unit/user_simulator/cascade/test_decisions.py b/tests/unit/user_simulator/cascade/test_decisions.py new file mode 100644 index 00000000..206c4437 --- /dev/null +++ b/tests/unit/user_simulator/cascade/test_decisions.py @@ -0,0 +1,79 @@ +from eva.user_simulator.cascade.decisions import ListenerDecisions, parse_yes_no + + +def test_parse_yes_no_accepts_plain_yes(): + assert parse_yes_no("YES") is True + + +def test_parse_yes_no_is_case_and_whitespace_insensitive(): + assert parse_yes_no(" yes\n") is True + + +def test_parse_yes_no_treats_anything_else_as_no(): + assert parse_yes_no("NO") is False + assert parse_yes_no("maybe") is False + assert parse_yes_no("") is False + + +class FakeLLM: + """Returns scripted replies, or raises when configured to.""" + + def __init__(self, replies: list[str] | None = None, error: Exception | None = None) -> None: + self.replies = replies or [] + self.error = error + self.calls = 0 + + async def decide(self, prompt: str) -> str: + self.calls += 1 + if self.error is not None: + raise self.error + return self.replies.pop(0) if self.replies else "NO" + + +async def test_both_checks_run_and_interrupt_wins_ties(): + llm = FakeLLM(["YES", "YES"]) + decisions = ListenerDecisions( + llm, interrupt_prompt="i {conversation_history}", backchannel_prompt="b {conversation_history}" + ) + + verdict = await decisions.evaluate("AGENT: hello", allow_interrupt=True, allow_backchannel=True) + + assert verdict.should_interrupt is True + assert verdict.should_backchannel is False + assert llm.calls == 2 + + +async def test_backchannel_alone_when_interrupt_declines(): + llm = FakeLLM(["NO", "YES"]) + decisions = ListenerDecisions( + llm, interrupt_prompt="i {conversation_history}", backchannel_prompt="b {conversation_history}" + ) + + verdict = await decisions.evaluate("AGENT: hello", allow_interrupt=True, allow_backchannel=True) + + assert verdict.should_interrupt is False + assert verdict.should_backchannel is True + + +async def test_disabled_behaviors_are_not_called_at_all(): + llm = FakeLLM(["YES"]) + decisions = ListenerDecisions( + llm, interrupt_prompt="i {conversation_history}", backchannel_prompt="b {conversation_history}" + ) + + verdict = await decisions.evaluate("AGENT: hello", allow_interrupt=False, allow_backchannel=True) + + assert verdict.should_interrupt is False + assert llm.calls == 1 + + +async def test_a_failing_check_fails_closed(): + llm = FakeLLM(error=RuntimeError("provider down")) + decisions = ListenerDecisions( + llm, interrupt_prompt="i {conversation_history}", backchannel_prompt="b {conversation_history}" + ) + + verdict = await decisions.evaluate("AGENT: hello", allow_interrupt=True, allow_backchannel=True) + + assert verdict.should_interrupt is False + assert verdict.should_backchannel is False From fddb3dcfe8b60adcb56670e849ffcb7c682b8d2e Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Fri, 7 Aug 2026 19:24:10 -0400 Subject: [PATCH 35/65] add pre-rendered phrase cache --- src/eva/__init__.py | 2 +- .../user_simulator/cascade/phrase_cache.py | 50 +++++++++++++++++ .../cascade/test_phrase_cache.py | 53 +++++++++++++++++++ 3 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 src/eva/user_simulator/cascade/phrase_cache.py create mode 100644 tests/unit/user_simulator/cascade/test_phrase_cache.py diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 1d2cb4a2..78ff07d2 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.21" +simulation_version = "2.0.22" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/phrase_cache.py b/src/eva/user_simulator/cascade/phrase_cache.py new file mode 100644 index 00000000..7621a0fa --- /dev/null +++ b/src/eva/user_simulator/cascade/phrase_cache.py @@ -0,0 +1,50 @@ +"""Pre-rendered audio for the caller's fixed phrase vocabularies.""" + +from __future__ import annotations + +import asyncio +import random +from typing import Protocol + +from eva.utils.logging import get_logger + +logger = get_logger(__name__) + + +class SpeechSynthesizer(Protocol): + """Minimal interface the cache needs from a TTS client.""" + + async def synthesize(self, text: str, *, voice_id: str) -> bytes: + """Render text to PCM16 audio.""" + ... + + +class PhraseCache: + """Renders a fixed vocabulary once at init so it can be voiced at zero latency. + + Backchannels and barge-in openers are short, fixed, and voice-stable, which + makes them cacheable — and caching is the only way a 300ms "mm-hmm" reliably + lands on the tick the decision chose. + """ + + def __init__(self, tts: SpeechSynthesizer, *, voice_id: str, seed: int = 0) -> None: + self._tts = tts + self._voice_id = voice_id + self._rng = random.Random(seed) + self._audio: dict[str, bytes] = {} + + async def prerender(self, phrases: list[str]) -> None: + """Synthesize every phrase concurrently and hold the audio in memory.""" + rendered = await asyncio.gather(*(self._tts.synthesize(p, voice_id=self._voice_id) for p in phrases)) + self._audio.update(dict(zip(phrases, rendered, strict=True))) + logger.info(f"Pre-rendered {len(phrases)} caller phrases") + + def get(self, phrase: str) -> bytes: + """Return cached audio for a phrase.""" + if phrase not in self._audio: + raise KeyError(f"Phrase not pre-rendered: {phrase!r}") + return self._audio[phrase] + + def choose(self, phrases: list[str]) -> str: + """Pick a phrase using the cache's seeded RNG, so runs stay reproducible.""" + return self._rng.choice(phrases) diff --git a/tests/unit/user_simulator/cascade/test_phrase_cache.py b/tests/unit/user_simulator/cascade/test_phrase_cache.py new file mode 100644 index 00000000..d2ff72f9 --- /dev/null +++ b/tests/unit/user_simulator/cascade/test_phrase_cache.py @@ -0,0 +1,53 @@ +import pytest + +from eva.user_simulator.cascade.phrase_cache import PhraseCache + + +class FakeTTS: + """Counts synthesis calls and returns deterministic audio per phrase.""" + + def __init__(self) -> None: + self.calls: list[str] = [] + + async def synthesize(self, text: str, *, voice_id: str) -> bytes: + self.calls.append(text) + return text.encode() + + +async def test_prerender_synthesizes_every_phrase_once(): + tts = FakeTTS() + cache = PhraseCache(tts, voice_id="voice-f") + + await cache.prerender(["uh-huh", "mm-hmm"]) + + assert sorted(tts.calls) == ["mm-hmm", "uh-huh"] + + +async def test_cached_audio_is_returned_without_further_synthesis(): + tts = FakeTTS() + cache = PhraseCache(tts, voice_id="voice-f") + await cache.prerender(["uh-huh"]) + + audio = cache.get("uh-huh") + + assert audio == b"uh-huh" + assert len(tts.calls) == 1 + + +async def test_choose_returns_a_phrase_from_the_cache_deterministically(): + tts = FakeTTS() + cache = PhraseCache(tts, voice_id="voice-f", seed=7) + await cache.prerender(["uh-huh", "mm-hmm"]) + + first = cache.choose(["uh-huh", "mm-hmm"]) + replay = PhraseCache(FakeTTS(), voice_id="voice-f", seed=7) + await replay.prerender(["uh-huh", "mm-hmm"]) + + assert first == replay.choose(["uh-huh", "mm-hmm"]) + + +async def test_requesting_an_unrendered_phrase_raises(): + cache = PhraseCache(FakeTTS(), voice_id="voice-f") + + with pytest.raises(KeyError, match="not pre-rendered"): + cache.get("never-rendered") From 0f664b87a097d11547ee87e63c42dc7376b513e8 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Fri, 7 Aug 2026 19:26:12 -0400 Subject: [PATCH 36/65] add backchannel behavior --- src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/simulator.py | 71 +++++++++++++++++++ .../user_simulator/cascade/test_simulator.py | 40 +++++++++++ 3 files changed, 112 insertions(+), 1 deletion(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 78ff07d2..46956263 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.22" +simulation_version = "2.0.23" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py index 7de87b34..fb29bfb6 100644 --- a/src/eva/user_simulator/cascade/simulator.py +++ b/src/eva/user_simulator/cascade/simulator.py @@ -12,12 +12,16 @@ from eva.user_simulator.base import AbstractUserSimulator from eva.user_simulator.cascade.adapter.realtime_ws import RealtimeWSAdapter from eva.user_simulator.cascade.constants import ( + BACKCHANNEL_PHRASES, + BARGE_IN_OPENERS, CALLER_SAMPLE_RATE, INACTIVITY_TIMEOUT_MS, TICK_DURATION_MS, TRANSCRIPT_WAIT_MS, ms_to_ticks, ) +from eva.user_simulator.cascade.decisions import ListenerDecisions +from eva.user_simulator.cascade.phrase_cache import PhraseCache from eva.user_simulator.cascade.scheduler import TickScheduler from eva.user_simulator.cascade.stt_livekit import LiveKitStreamingSTT from eva.user_simulator.cascade.tick_result import TickResult @@ -26,6 +30,7 @@ # Shared with the OpenAI Realtime provider so both simulators hang up on the same rules. from eva.user_simulator.openai_realtime import END_CALL_DESCRIPTION from eva.utils.logging import get_logger +from eva.utils.prompt_manager import PromptManager logger = get_logger(__name__) @@ -65,6 +70,27 @@ def extract_turn(message: object) -> tuple[str, bool]: return parse_turn_response(content), end_call +def play_backchannel(scheduler, cache, phrases: list[str]) -> str: + """Queue a cached continuer and return the phrase that was chosen.""" + phrase = cache.choose(phrases) + scheduler.enqueue_utterance(cache.get(phrase)) + return phrase + + +class _DecisionClient: + """Adapts LiteLLMClient to the single-prompt interface the checks expect.""" + + def __init__(self, client: LiteLLMClient) -> None: + self._client = client + + async def decide(self, prompt: str) -> str: + """Ask one YES/NO question and return the raw reply.""" + message, _stats = await self._client.complete(messages=[{"role": "user", "content": prompt}]) + if isinstance(message, str): + return message + return getattr(message, "content", None) or "" + + class CascadeUserSimulator(AbstractUserSimulator): """Simulated caller built from independently chosen STT, LLM, and TTS models.""" @@ -104,6 +130,10 @@ def __init__( self._llm = LiteLLMClient(model=simulator_config.llm) self._voice_id = self._tts.voice_for_persona(persona_config) self._history: list[dict[str, str]] = [] + # Shared by the listener checks and the relevance gate so both cost one client. + self._decision_client = _DecisionClient(LiteLLMClient(model=simulator_config.decision_llm)) + self._phrase_cache: PhraseCache | None = None + self._decisions: ListenerDecisions | None = None async def run_conversation(self) -> str: """Run the tick loop until the call ends, and return the end reason.""" @@ -129,6 +159,7 @@ async def _run(self) -> None: await adapter.start() await self._stt.start() + await self._prepare_listener_behaviors() self.event_logger.log_connection_state("connected", {"server_url": self.server_url}) max_ticks = self.timeout * 1000 // TICK_DURATION_MS @@ -152,6 +183,8 @@ async def _run(self) -> None: caller_was_speaking = scheduler.caller_spoke_this_tick assistant_was_speaking = result.has_assistant_speech if result.has_assistant_speech: + if scheduler.is_check_tick(): + await self._run_checks(scheduler) continue if self._assistant_is_inactive(scheduler, result): logger.warning( @@ -175,6 +208,44 @@ async def _run(self) -> None: await adapter.stop() self.event_logger.log_connection_state("session_ended", {"reason": self._end_reason}) + async def _prepare_listener_behaviors(self) -> None: + """Pre-render the fixed vocabularies and build the checks, when any is enabled. + + Rendering happens once at connect time rather than on demand: a cached phrase + is the only way a 300ms reaction lands on the tick its check chose. + """ + vocabulary: list[str] = [] + if self._config.enable_backchannel: + vocabulary += BACKCHANNEL_PHRASES + if self._config.enable_interruptions: + vocabulary += BARGE_IN_OPENERS + if not vocabulary: + return + + self._phrase_cache = PhraseCache(self._tts, voice_id=self._voice_id) + await self._phrase_cache.prerender(vocabulary) + prompts = PromptManager() + self._decisions = ListenerDecisions( + self._decision_client, + interrupt_prompt=prompts.get_template("user_simulator.interruption_decision"), + backchannel_prompt=prompts.get_template("user_simulator.backchannel_decision"), + ) + + async def _run_checks(self, scheduler: TickScheduler) -> None: + """Run the listener-reaction checks and act on the verdict.""" + if self._decisions is None or self._phrase_cache is None: + return + verdict = await self._decisions.evaluate( + self._stt.buffer.current_text(), + allow_interrupt=self._config.enable_interruptions, + allow_backchannel=self._config.enable_backchannel, + ) + if verdict.should_backchannel: + phrase = play_backchannel(scheduler, self._phrase_cache, BACKCHANNEL_PHRASES) + # Recorded too, or the saved clean track diverges from what went on the wire. + self._record_audio("user_clean", self._phrase_cache.get(phrase)) + self.event_logger.log_event("backchannel", {"text": phrase, "tick_index": scheduler.tick}) + def _assistant_is_inactive(self, scheduler: TickScheduler, result: TickResult) -> bool: """Whether the assistant has produced no audio for INACTIVITY_TIMEOUT_MS. diff --git a/tests/unit/user_simulator/cascade/test_simulator.py b/tests/unit/user_simulator/cascade/test_simulator.py index 7dc696bc..d28b6efc 100644 --- a/tests/unit/user_simulator/cascade/test_simulator.py +++ b/tests/unit/user_simulator/cascade/test_simulator.py @@ -282,3 +282,43 @@ def test_tick_counters_exist_without_manual_setup(): assert sim._ticks_assistant_silent == 0 assert sim._ticks_awaiting_transcript == 0 + + +class RecordingScheduler: + """Captures what the simulator queues for playout.""" + + def __init__(self) -> None: + self.queued: list[bytes] = [] + self.tick = 0 + + def enqueue_utterance(self, audio: bytes) -> None: + self.queued.append(audio) + + +class StubCache: + """Phrase cache stand-in that always picks the first phrase.""" + + def choose(self, phrases): + return phrases[0] + + def get(self, phrase): + return b"CACHED" + + +async def test_backchannel_queues_cached_audio_without_synthesis(): + from eva.user_simulator.cascade import simulator as module + + scheduler = RecordingScheduler() + played = module.play_backchannel(scheduler, StubCache(), ["uh-huh", "mm-hmm"]) + + assert played == "uh-huh" + assert scheduler.queued == [b"CACHED"] + + +def test_verdict_with_no_action_queues_nothing(): + from eva.user_simulator.cascade.decisions import ListenerVerdict + + verdict = ListenerVerdict(should_interrupt=False, should_backchannel=False) + + assert verdict.should_interrupt is False + assert verdict.should_backchannel is False From 6d7bb39d40b720cf94cdb1a5da4f992986e7f0a4 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Fri, 7 Aug 2026 19:27:02 -0400 Subject: [PATCH 37/65] add streaming synthesis to the TTS client --- src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/tts.py | 19 ++++++++++++------ tests/unit/user_simulator/cascade/test_tts.py | 20 +++++++++++++++++++ 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 46956263..cbcfa99b 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.23" +simulation_version = "2.0.24" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/tts.py b/src/eva/user_simulator/cascade/tts.py index 231ebd99..c38e82e4 100644 --- a/src/eva/user_simulator/cascade/tts.py +++ b/src/eva/user_simulator/cascade/tts.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +from collections.abc import AsyncIterator from typing import Any import httpx @@ -37,10 +38,10 @@ def voice_for_persona(self, persona_config: dict[str, Any]) -> str: return self._female_voice return self._male_voice - async def synthesize(self, text: str, *, voice_id: str) -> bytes: - """Render text to raw PCM16 mono at CALLER_SAMPLE_RATE.""" + async def stream(self, text: str, *, voice_id: str) -> AsyncIterator[bytes]: + """Yield PCM16 chunks as they render, so playout can start before the tail exists.""" if not text: - return b"" + return if not self._api_key: raise ValueError("Cartesia API key missing: set tts_params.api_key or CARTESIA_API_KEY") @@ -58,6 +59,12 @@ async def synthesize(self, text: str, *, voice_id: str) -> bytes: headers = {"X-API-Key": self._api_key, "Cartesia-Version": CARTESIA_VERSION} async with httpx.AsyncClient(timeout=30.0) as client: - response = await client.post(CARTESIA_URL, json=body, headers=headers) - response.raise_for_status() - return response.content + async with client.stream("POST", CARTESIA_URL, json=body, headers=headers) as response: + response.raise_for_status() + async for chunk in response.aiter_bytes(): + if chunk: + yield chunk + + async def synthesize(self, text: str, *, voice_id: str) -> bytes: + """Render text to raw PCM16 mono at CALLER_SAMPLE_RATE.""" + return b"".join([chunk async for chunk in self.stream(text, voice_id=voice_id)]) diff --git a/tests/unit/user_simulator/cascade/test_tts.py b/tests/unit/user_simulator/cascade/test_tts.py index 1973768d..e66985ec 100644 --- a/tests/unit/user_simulator/cascade/test_tts.py +++ b/tests/unit/user_simulator/cascade/test_tts.py @@ -33,3 +33,23 @@ async def test_missing_api_key_raises_a_clear_error(monkeypatch): with pytest.raises(ValueError, match="Cartesia API key"): await tts.synthesize("hello", voice_id="voice-f") + + +async def test_stream_yields_nothing_for_empty_text(): + tts = CartesiaTTS({"model": "sonic-3.5", "api_key": "k"}) + + chunks = [chunk async for chunk in tts.stream("", voice_id="voice-f")] + + assert chunks == [] + + +async def test_synthesize_concatenates_the_stream(monkeypatch): + tts = CartesiaTTS({"model": "sonic-3.5", "api_key": "k"}) + + async def fake_stream(text, *, voice_id): + for piece in (b"ab", b"cd"): + yield piece + + monkeypatch.setattr(tts, "stream", fake_stream) + + assert await tts.synthesize("hello", voice_id="voice-f") == b"abcd" From 8d3680bf9725153c47a38560ef6b0e25c6ccb9cf Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Fri, 7 Aug 2026 19:28:23 -0400 Subject: [PATCH 38/65] add reactive interruption with slip tracking --- src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/simulator.py | 63 +++++++++++++++++++ .../user_simulator/cascade/test_simulator.py | 31 +++++++++ 3 files changed, 95 insertions(+), 1 deletion(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index cbcfa99b..480a8e12 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.24" +simulation_version = "2.0.25" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py index fb29bfb6..4b39785b 100644 --- a/src/eva/user_simulator/cascade/simulator.py +++ b/src/eva/user_simulator/cascade/simulator.py @@ -16,6 +16,7 @@ BARGE_IN_OPENERS, CALLER_SAMPLE_RATE, INACTIVITY_TIMEOUT_MS, + MAX_INTERRUPT_SLIP_MS, TICK_DURATION_MS, TRANSCRIPT_WAIT_MS, ms_to_ticks, @@ -70,6 +71,16 @@ def extract_turn(message: object) -> tuple[str, bool]: return parse_turn_response(content), end_call +def interrupt_slip_ms(*, intended_tick: int, actual_tick: int) -> int: + """How far past its intended tick a barge-in actually landed.""" + return max(0, actual_tick - intended_tick) * TICK_DURATION_MS + + +def should_drop_interrupt(*, slip_ms: int, assistant_still_speaking: bool) -> bool: + """Whether a barge-in has gone stale and should be abandoned.""" + return slip_ms > MAX_INTERRUPT_SLIP_MS or not assistant_still_speaking + + def play_backchannel(scheduler, cache, phrases: list[str]) -> str: """Queue a cached continuer and return the phrase that was chosen.""" phrase = cache.choose(phrases) @@ -240,12 +251,64 @@ async def _run_checks(self, scheduler: TickScheduler) -> None: allow_interrupt=self._config.enable_interruptions, allow_backchannel=self._config.enable_backchannel, ) + if verdict.should_interrupt: + await self._play_interruption(scheduler) + return if verdict.should_backchannel: phrase = play_backchannel(scheduler, self._phrase_cache, BACKCHANNEL_PHRASES) # Recorded too, or the saved clean track diverges from what went on the wire. self._record_audio("user_clean", self._phrase_cache.get(phrase)) self.event_logger.log_event("backchannel", {"text": phrase, "tick_index": scheduler.tick}) + async def _play_interruption(self, scheduler: TickScheduler) -> None: + """Voice a cached opener immediately, then stream the real content behind it. + + The opener buys the lead time the content generation costs. If the content + still arrives too late to be a barge-in, it is dropped rather than emitted + stale — and both ticks are logged either way, because that gap is the + empirical risk this design carries. + """ + if self._phrase_cache is None: + return + intended_tick = scheduler.tick + opener = self._phrase_cache.choose(BARGE_IN_OPENERS) + opener_audio = self._phrase_cache.get(opener) + scheduler.enqueue_utterance(opener_audio) + self._record_audio("user_clean", opener_audio) + + # Consumed, not peeked: leaving it in the buffer would re-append the same + # assistant prefix at the next ordinary turn and duplicate it in the history. + heard = self._stt.buffer.current_text() + self._stt.buffer.take_committed() + self._stt.buffer.in_flight = "" + if heard: + self._history.append({"role": "assistant", "content": heard}) + self._on_assistant_speaks(heard) + message, _stats = await self._llm.complete(messages=self._messages(), tools=[END_CALL_TOOL]) + utterance, _end_call = extract_turn(message) + + slip = interrupt_slip_ms(intended_tick=intended_tick, actual_tick=scheduler.tick) + dropped = should_drop_interrupt(slip_ms=slip, assistant_still_speaking=scheduler.assistant_is_speaking) + self.event_logger.log_event( + "interruption", + { + "text": utterance, + "opener": opener, + "intended_tick": intended_tick, + "actual_tick": scheduler.tick, + "slip_ms": slip, + "dropped": dropped, + }, + ) + if dropped or not utterance: + return + + self._history.append({"role": "user", "content": utterance}) + self._on_user_speaks(utterance) + async for chunk in self._tts.stream(utterance, voice_id=self._voice_id): + self._record_audio("user_clean", chunk) + scheduler.enqueue_utterance(chunk) + def _assistant_is_inactive(self, scheduler: TickScheduler, result: TickResult) -> bool: """Whether the assistant has produced no audio for INACTIVITY_TIMEOUT_MS. diff --git a/tests/unit/user_simulator/cascade/test_simulator.py b/tests/unit/user_simulator/cascade/test_simulator.py index d28b6efc..e7520e54 100644 --- a/tests/unit/user_simulator/cascade/test_simulator.py +++ b/tests/unit/user_simulator/cascade/test_simulator.py @@ -322,3 +322,34 @@ def test_verdict_with_no_action_queues_nothing(): assert verdict.should_interrupt is False assert verdict.should_backchannel is False + + +def test_slip_is_measured_in_ticks_converted_to_ms(): + from eva.user_simulator.cascade.simulator import interrupt_slip_ms + + assert interrupt_slip_ms(intended_tick=10, actual_tick=15) == 1000 + + +def test_no_slip_when_the_interrupt_lands_on_its_intended_tick(): + from eva.user_simulator.cascade.simulator import interrupt_slip_ms + + assert interrupt_slip_ms(intended_tick=10, actual_tick=10) == 0 + + +def test_interrupt_kept_when_slip_is_within_budget(): + from eva.user_simulator.cascade.simulator import should_drop_interrupt + + assert should_drop_interrupt(slip_ms=800, assistant_still_speaking=True) is False + + +def test_interrupt_dropped_when_slip_exceeds_budget(): + from eva.user_simulator.cascade.simulator import should_drop_interrupt + + assert should_drop_interrupt(slip_ms=2000, assistant_still_speaking=True) is True + + +def test_interrupt_dropped_when_the_assistant_already_stopped(): + # No longer an interruption — it would land as an ordinary reply. + from eva.user_simulator.cascade.simulator import should_drop_interrupt + + assert should_drop_interrupt(slip_ms=100, assistant_still_speaking=False) is True From 1f725530fc92e3522be52afeabe6e8252b0029c6 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Fri, 7 Aug 2026 19:31:48 -0400 Subject: [PATCH 39/65] add self-correction behavior --- configs/prompts/simulation.yaml | 29 ++++ src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/simulator.py | 96 +++++++++++++ .../user_simulator/cascade/test_prompt.py | 26 +++- .../user_simulator/cascade/test_simulator.py | 130 ++++++++++++++++++ 5 files changed, 279 insertions(+), 4 deletions(-) diff --git a/configs/prompts/simulation.yaml b/configs/prompts/simulation.yaml index 81580366..23e9d149 100644 --- a/configs/prompts/simulation.yaml +++ b/configs/prompts/simulation.yaml @@ -584,3 +584,32 @@ user_simulator: → NO (user backchanneled recently, don't do it again so soon) Respond with ONLY "YES" or "NO". + + cascade_self_correction: | + You are about to say this line to the agent: + + + {utterance} + + + On this turn only, you will misspeak first and then correct yourself, the way + people do on real phone calls. The line above is the CORRECT one — it is what + you will say as the correction. Your job here is to write the SLIP that comes + just before it. + + Write the version of that line that you say by mistake. + + Rules, all of which matter: + - It must state a decision that is WRONG but plausibly close to the intended + line. A different day, a different one of two options, a slightly wrong + number. Never something absurd, and never a different topic. + - Change exactly one detail. Everything else stays as it was. + - It must stand on its own as a natural thing to say. Do not hedge, do not + signal that it is wrong, and do not correct yourself inside it — the + self_correction is delivered separately, a moment later. + - Never contradict or drop a fact the agent needs; the correction that follows + is what satisfies your must_have_criteria, so the slip must be safely + reversible by it. + - Reply with ONLY the spoken line, nothing else. No quotes, no labels. + - If the intended line contains no detail that could plausibly be misspoken + (a greeting, a thank-you, a yes/no acknowledgement), reply with exactly NONE. diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 480a8e12..63046195 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.25" +simulation_version = "2.0.26" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py index 4b39785b..618025ce 100644 --- a/src/eva/user_simulator/cascade/simulator.py +++ b/src/eva/user_simulator/cascade/simulator.py @@ -2,6 +2,7 @@ from __future__ import annotations +import random import re from pathlib import Path @@ -17,6 +18,8 @@ CALLER_SAMPLE_RATE, INACTIVITY_TIMEOUT_MS, MAX_INTERRUPT_SLIP_MS, + SELF_CORRECTION_DELAY_MS, + SELF_CORRECTION_RATE, TICK_DURATION_MS, TRANSCRIPT_WAIT_MS, ms_to_ticks, @@ -71,6 +74,25 @@ def extract_turn(message: object) -> tuple[str, bool]: return parse_turn_response(content), end_call +def extract_correction(message: object) -> str: + """Read the self-correction line from its own dedicated call. + + It is a bare spoken line, not a JSON field: the caller's turn call keeps the + plain-text contract Plan 1 settled on, since demanding JSON there suppressed + the end_call tool call entirely. + """ + content = message if isinstance(message, str) else (getattr(message, "content", None) or "") + line = _FENCE.sub("", content).strip() + return "" if line.upper() == "NONE" else line + + +def should_fire_self_correction(*, ticks_since_assistant_started: int, assistant_speaking: bool) -> bool: + """Whether an armed correction should play now.""" + if not assistant_speaking: + return False + return ticks_since_assistant_started >= ms_to_ticks(SELF_CORRECTION_DELAY_MS) + + def interrupt_slip_ms(*, intended_tick: int, actual_tick: int) -> int: """How far past its intended tick a barge-in actually landed.""" return max(0, actual_tick - intended_tick) * TICK_DURATION_MS @@ -107,6 +129,7 @@ class CascadeUserSimulator(AbstractUserSimulator): _ticks_awaiting_transcript = 0 _ticks_assistant_silent = 0 + _ticks_since_assistant_started = 0 def __init__( self, @@ -145,6 +168,9 @@ def __init__( self._decision_client = _DecisionClient(LiteLLMClient(model=simulator_config.decision_llm)) self._phrase_cache: PhraseCache | None = None self._decisions: ListenerDecisions | None = None + self._rng = random.Random(0) + self._armed_correction: bytes = b"" + self._armed_correction_text = "" async def run_conversation(self) -> str: """Run the tick loop until the call ends, and return the end reason.""" @@ -194,9 +220,17 @@ async def _run(self) -> None: caller_was_speaking = scheduler.caller_spoke_this_tick assistant_was_speaking = result.has_assistant_speech if result.has_assistant_speech: + self._ticks_since_assistant_started += 1 + if self._armed_correction and should_fire_self_correction( + ticks_since_assistant_started=self._ticks_since_assistant_started, + assistant_speaking=True, + ): + self._fire_self_correction(scheduler) + continue if scheduler.is_check_tick(): await self._run_checks(scheduler) continue + self._ticks_since_assistant_started = 0 if self._assistant_is_inactive(scheduler, result): logger.warning( f"tick {scheduler.tick}: assistant silent for " @@ -384,9 +418,13 @@ async def _take_turn(self, scheduler: TickScheduler, heard: str) -> bool: self._history.append({"role": "assistant", "content": heard}) self._on_assistant_speaks(heard) + self._drop_stale_correction() message, _stats = await self._llm.complete(messages=self._messages(), tools=[END_CALL_TOOL]) utterance, end_call = extract_turn(message) + if utterance and not end_call: + utterance = await self._maybe_arm_self_correction(utterance) + if utterance: self._history.append({"role": "user", "content": utterance}) self._on_user_speaks(utterance) @@ -400,6 +438,64 @@ async def _take_turn(self, scheduler: TickScheduler, heard: str) -> bool: return True return False + def _fire_self_correction(self, scheduler: TickScheduler) -> None: + """Play the armed correction over the assistant's reply and clear the arming.""" + scheduler.enqueue_utterance(self._armed_correction) + self._record_audio("user_clean", self._armed_correction) + self._history.append({"role": "user", "content": self._armed_correction_text}) + self._on_user_speaks(self._armed_correction_text) + self.event_logger.log_event( + "self_correction", + {"text": self._armed_correction_text, "tick_index": scheduler.tick}, + ) + self._armed_correction = b"" + self._armed_correction_text = "" + + def _drop_stale_correction(self) -> None: + """Abandon an armed correction whose assistant turn never arrived. + + Carrying it into a later turn would read as a non-sequitur, since it refers + to an utterance that is now several exchanges back. + """ + if not self._armed_correction: + return + self.event_logger.log_event("self_correction_dropped", {"text": self._armed_correction_text}) + self._armed_correction = b"" + self._armed_correction_text = "" + + async def _maybe_arm_self_correction(self, utterance: str) -> str: + """Maybe misspeak: return a wrong variant to say now, arming `utterance` as the fix. + + The wrong-then-right ordering is the design, not a detail. The generated slip + is spoken first and the model's own goal-consistent line lands as the + correction, so the conversation's end state still satisfies must_have_criteria + by construction and this behavior cannot make a record unachievable. + + Asked as its own call rather than as an extra JSON field on the turn call: a + JSON contract there suppressed the end_call tool entirely (Plan 1), and this + way a failure degrades to an ordinary turn instead of a broken one. + """ + if not self._config.enable_self_correction or self._rng.random() >= SELF_CORRECTION_RATE: + return utterance + + prompt = PromptManager().get_prompt("user_simulator.cascade_self_correction", utterance=utterance) + try: + message, _stats = await self._llm.complete( + messages=[*self._messages(), {"role": "user", "content": prompt}] + ) + except Exception as exc: + logger.warning(f"Self-correction generation failed, speaking the turn unchanged: {exc}") + return utterance + + slip = extract_correction(message) + if not slip or slip == utterance: + return utterance + + self._armed_correction = await self._tts.synthesize(utterance, voice_id=self._voice_id) + self._armed_correction_text = utterance + self.event_logger.log_event("self_correction_armed", {"slip": slip, "correction": utterance}) + return slip + @staticmethod def _warn_unsupported_perturbation(perturbation_config: PerturbationConfig | None) -> None: """Warn when outbound-audio perturbations are configured but unsupported by cascade. diff --git a/tests/unit/user_simulator/cascade/test_prompt.py b/tests/unit/user_simulator/cascade/test_prompt.py index deaa2748..ea7c91e6 100644 --- a/tests/unit/user_simulator/cascade/test_prompt.py +++ b/tests/unit/user_simulator/cascade/test_prompt.py @@ -9,12 +9,18 @@ def test_cascade_reuses_the_shared_end_call_description(): assert cascade_description is shared_description -def test_no_cascade_specific_prompts_remain_in_the_prompt_file(): +def test_the_turn_call_carries_no_cascade_specific_contract(): # The per-domain user_simulator prompt already carries persona, goal and end_call rules; # layering a cascade-only contract on top is what suppressed the end_call tool call. - from pathlib import Path + # Out-of-turn behavior prompts exist, but they are only ever used in their own + # standalone calls — never appended to the system prompt of the turn call. + from eva.user_simulator.cascade.simulator import CascadeUserSimulator - assert "cascade_" not in Path("configs/prompts/simulation.yaml").read_text() + sim = object.__new__(CascadeUserSimulator) + sim._build_prompt = lambda: "SYSTEM PROMPT" + sim._history = [] + + assert sim._messages()[0]["content"] == "SYSTEM PROMPT" def test_interruption_decision_prompt_has_a_history_slot_and_binary_contract(): @@ -31,3 +37,17 @@ def test_backchannel_decision_prompt_has_a_history_slot_and_frequency_guidance() assert "AGENT: hello" in prompt assert "CURRENTLY SPEAKING, INCOMPLETE" in prompt assert "When in doubt, say NO" in prompt + + +def test_self_correction_prompt_states_the_wrong_then_right_ordering(): + prompt = PromptManager().get_template("user_simulator.cascade_self_correction") + + assert "self_correction" in prompt + assert "must_have_criteria" in prompt + + +def test_self_correction_prompt_never_mentions_ending_the_call(): + # It runs as its own call; if it could elicit a hang-up it would race the turn call. + prompt = PromptManager().get_template("user_simulator.cascade_self_correction") + + assert "end_call" not in prompt diff --git a/tests/unit/user_simulator/cascade/test_simulator.py b/tests/unit/user_simulator/cascade/test_simulator.py index e7520e54..91cd3e58 100644 --- a/tests/unit/user_simulator/cascade/test_simulator.py +++ b/tests/unit/user_simulator/cascade/test_simulator.py @@ -353,3 +353,133 @@ def test_interrupt_dropped_when_the_assistant_already_stopped(): from eva.user_simulator.cascade.simulator import should_drop_interrupt assert should_drop_interrupt(slip_ms=100, assistant_still_speaking=False) is True + + +def test_correction_fires_once_the_delay_has_elapsed(): + from eva.user_simulator.cascade.simulator import should_fire_self_correction + + assert should_fire_self_correction(ticks_since_assistant_started=6, assistant_speaking=True) is True + + +def test_correction_does_not_fire_before_the_delay(): + from eva.user_simulator.cascade.simulator import should_fire_self_correction + + assert should_fire_self_correction(ticks_since_assistant_started=2, assistant_speaking=True) is False + + +def test_correction_is_abandoned_if_the_assistant_never_replied(): + from eva.user_simulator.cascade.simulator import should_fire_self_correction + + assert should_fire_self_correction(ticks_since_assistant_started=6, assistant_speaking=False) is False + + +def test_extract_correction_reads_a_plain_line(): + from eva.user_simulator.cascade.simulator import extract_correction + + assert extract_correction("Actually, wait — I said Thursday, I meant Friday.") == ( + "Actually, wait — I said Thursday, I meant Friday." + ) + + +def test_extract_correction_strips_a_code_fence(): + from eva.user_simulator.cascade.simulator import extract_correction + + assert extract_correction("```\nI meant Friday.\n```") == "I meant Friday." + + +def test_extract_correction_is_empty_for_an_empty_reply(): + from eva.user_simulator.cascade.simulator import extract_correction + + assert extract_correction("") == "" + + +def test_extract_correction_reads_through_a_message_object(): + from eva.user_simulator.cascade.simulator import extract_correction + + class _Message: + content = "I meant Friday." + tool_calls: list = [] + + assert extract_correction(_Message()) == "I meant Friday." + + +def test_extract_correction_rejects_a_refusal_style_non_answer(): + # The prompt allows the model to decline by returning NONE. + from eva.user_simulator.cascade.simulator import extract_correction + + assert extract_correction("NONE") == "" + + +def _correcting_simulator(reply: str, *, rate_roll: float = 0.0): + """Bare simulator wired for _maybe_arm_self_correction only.""" + from eva.models.config import CascadeSimulatorConfig + + sim = CascadeUserSimulator.__new__(CascadeUserSimulator) + sim._config = CascadeSimulatorConfig(enable_self_correction=True) + sim._rng = type("_Rng", (), {"random": staticmethod(lambda: rate_roll)})() + sim._build_prompt = lambda: "SYSTEM PROMPT" + sim._history = [] + sim._voice_id = "voice-f" + sim.event_logger = _FakeEventLogger() + sim._armed_correction = b"" + sim._armed_correction_text = "" + + class _Llm: + async def complete(self, messages, tools=None): + return reply, {} + + class _Tts: + async def synthesize(self, text, *, voice_id): + return text.encode() + + sim._llm, sim._tts = _Llm(), _Tts() + return sim + + +async def test_the_slip_is_spoken_and_the_original_line_is_armed_as_the_correction(): + # Wrong-then-right: the goal-consistent line must be what lands last. + sim = _correcting_simulator("Book me Thursday.") + + spoken = await sim._maybe_arm_self_correction("Book me Friday.") + + assert spoken == "Book me Thursday." + assert sim._armed_correction_text == "Book me Friday." + + +async def test_no_correction_is_armed_when_the_rate_gate_declines(): + sim = _correcting_simulator("Book me Thursday.", rate_roll=0.99) + + spoken = await sim._maybe_arm_self_correction("Book me Friday.") + + assert spoken == "Book me Friday." + assert sim._armed_correction == b"" + + +async def test_a_none_reply_leaves_the_turn_unchanged(): + sim = _correcting_simulator("NONE") + + spoken = await sim._maybe_arm_self_correction("Thanks, goodbye.") + + assert spoken == "Thanks, goodbye." + assert sim._armed_correction == b"" + + +async def test_a_failed_correction_call_degrades_to_an_ordinary_turn(): + sim = _correcting_simulator("unused") + + class _Failing: + async def complete(self, messages, tools=None): + raise RuntimeError("provider down") + + sim._llm = _Failing() + + assert await sim._maybe_arm_self_correction("Book me Friday.") == "Book me Friday." + + +async def test_self_correction_is_skipped_when_the_behavior_is_disabled(): + from eva.models.config import CascadeSimulatorConfig + + sim = _correcting_simulator("Book me Thursday.") + sim._config = CascadeSimulatorConfig() + + assert await sim._maybe_arm_self_correction("Book me Friday.") == "Book me Friday." From a2acc3f94a31a7d6d46d06da70c4a6433894ac33 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Fri, 7 Aug 2026 19:34:56 -0400 Subject: [PATCH 40/65] add per-tick ambient noise mixing --- src/eva/__init__.py | 2 +- .../cascade/adapter/realtime_ws.py | 30 +++++++++- src/eva/user_simulator/cascade/simulator.py | 24 +------- .../cascade/test_realtime_ws_adapter.py | 58 ++++++++++++++++++- .../user_simulator/cascade/test_simulator.py | 23 ++------ 5 files changed, 91 insertions(+), 46 deletions(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 63046195..a9141aa6 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.26" +simulation_version = "2.0.27" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/adapter/realtime_ws.py b/src/eva/user_simulator/cascade/adapter/realtime_ws.py index 7ed14962..2e1ca512 100644 --- a/src/eva/user_simulator/cascade/adapter/realtime_ws.py +++ b/src/eva/user_simulator/cascade/adapter/realtime_ws.py @@ -44,10 +44,18 @@ class RealtimeWSAdapter(Adapter): `audio_utils` cannot express that and are not reused here for that reason. """ - def __init__(self, *, websocket, conversation_id: str, bytes_per_tick: int = BYTES_PER_TICK) -> None: + def __init__( + self, + *, + websocket, + conversation_id: str, + bytes_per_tick: int = BYTES_PER_TICK, + perturbator=None, + ) -> None: self._ws = websocket self._conversation_id = conversation_id self._bytes_per_tick = bytes_per_tick + self._perturbator = perturbator self._inbound = bytearray() self._receive_task: asyncio.Task | None = None self._inbound_resample_state = None @@ -74,7 +82,8 @@ async def run_tick(self, tick_number: int, outgoing_audio: bytes | None) -> Tick await self._send_speech_event("user_speech_stop") self._caller_speaking = is_speaking - await self._send_tick_audio(outgoing_audio or SILENCE_BYTE * self._bytes_per_tick) + outgoing = self._apply_perturbation(outgoing_audio) + await self._send_tick_audio(outgoing or SILENCE_BYTE * self._bytes_per_tick) raw = bytes(self._inbound[: self._bytes_per_tick]) del self._inbound[: len(raw)] @@ -96,6 +105,23 @@ async def run_tick(self, tick_number: int, outgoing_audio: bytes | None) -> Tick return result + def _apply_perturbation(self, outgoing_audio: bytes | None) -> bytes | None: + """Mix ambient noise into this tick's outgoing audio. + + Real-time path: the mic is always open, so noise is emitted even when the + caller is silent — it replaces the silence this adapter already sends every + tick rather than adding frames. Never call this for a tick that emits nothing + on the tick-driven path (Plan 3): audio sent during a stall would advance the + assistant's VAD and break the freeze. + """ + if self._perturbator is None: + return outgoing_audio + if outgoing_audio: + return self._perturbator.apply(outgoing_audio) + if getattr(self._perturbator, "has_ambient_noise", False): + return self._perturbator.get_ambient_chunk(self._bytes_per_tick) + return outgoing_audio + async def stop(self) -> None: """Send stop, cancel the receive loop, and close the socket. Safe to call twice.""" if self._receive_task is not None: diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py index 618025ce..31da22f2 100644 --- a/src/eva/user_simulator/cascade/simulator.py +++ b/src/eva/user_simulator/cascade/simulator.py @@ -157,7 +157,6 @@ def __init__( language=language, provider="cascade", ) - self._warn_unsupported_perturbation(perturbation_config) self._config = simulator_config self._stt = LiveKitStreamingSTT(simulator_config.stt, simulator_config.stt_params, language=language) self._tts = CartesiaTTS(simulator_config.tts_params, language=language) @@ -191,6 +190,7 @@ async def _run(self) -> None: adapter = RealtimeWSAdapter( websocket=websocket, conversation_id=self._record_id or "cascade", + perturbator=self._perturbator, ) scheduler = TickScheduler(adapter) @@ -496,28 +496,6 @@ async def _maybe_arm_self_correction(self, utterance: str) -> str: self.event_logger.log_event("self_correction_armed", {"slip": slip, "correction": utterance}) return slip - @staticmethod - def _warn_unsupported_perturbation(perturbation_config: PerturbationConfig | None) -> None: - """Warn when outbound-audio perturbations are configured but unsupported by cascade. - - `RealtimeWSAdapter` does not yet apply perturbation to outbound audio, so - `background_noise` and `connection_degradation` are silently dropped without this. - `snr_db` only has an effect alongside `background_noise` and defaults to 15.0, so it - is flagged only when `background_noise` is also set, not on its default value alone. - """ - if perturbation_config is None: - return - unsupported = [] - if perturbation_config.background_noise is not None: - unsupported.extend(["background_noise", "snr_db"]) - if perturbation_config.connection_degradation: - unsupported.append("connection_degradation") - if unsupported: - logger.warning( - f"Cascade simulator does not yet apply audio perturbation: ignoring {', '.join(unsupported)}. " - "Behavior and accent perturbations are unaffected." - ) - def _messages(self) -> list[dict[str, str]]: """Build the message list: the shared per-domain caller prompt plus flipped history. diff --git a/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py b/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py index 15afc094..157fa165 100644 --- a/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py +++ b/tests/unit/user_simulator/cascade/test_realtime_ws_adapter.py @@ -9,7 +9,7 @@ except ImportError: # pragma: no cover - Python 3.13+ import audioop_lts as audioop -from eva.user_simulator.cascade.adapter.realtime_ws import RealtimeWSAdapter +from eva.user_simulator.cascade.adapter.realtime_ws import FRAMES_PER_TICK, RealtimeWSAdapter BYTES_PER_TICK = 6400 _SETTLE_ROUNDS = 300 @@ -306,3 +306,59 @@ async def test_user_speech_stop_emitted_once_on_audio_to_silence_transition(): assert isinstance(stops[0]["timestamp_ms"], str) await adapter.stop() + + +class StubPerturbator: + """Stands in for AudioPerturbator with a constant, recognizable noise floor.""" + + has_ambient_noise = True + + def get_ambient_chunk(self, size: int) -> bytes: + return b"\x11" * size + + def apply(self, audio: bytes) -> bytes: + return b"\x22" * len(audio) + + +def _media_payloads(ws) -> list[bytes]: + import base64 + + return [ + base64.b64decode(json.loads(m)["media"]["payload"]) for m in ws.sent if json.loads(m).get("event") == "media" + ] + + +async def test_ambient_noise_replaces_silence_when_the_caller_is_not_speaking(): + ws = FakeWebSocket() + adapter = RealtimeWSAdapter( + websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK, perturbator=StubPerturbator() + ) + + await adapter.run_tick(0, None) + + payloads = _media_payloads(ws) + assert len(payloads) == FRAMES_PER_TICK + # Silence would encode to a constant mulaw byte; ambient noise must not. + assert set(b"".join(payloads)) != {0xFF} + + +async def test_ambient_noise_is_mixed_into_caller_speech(): + ws = FakeWebSocket() + perturbator = StubPerturbator() + adapter = RealtimeWSAdapter( + websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK, perturbator=perturbator + ) + + await adapter.run_tick(0, b"\x01" * BYTES_PER_TICK) + + assert len(_media_payloads(ws)) == FRAMES_PER_TICK + + +async def test_silence_is_still_sent_every_tick_without_a_perturbator(): + # Plan 1: a tick that sends no frames at all makes the assistant's turn detection misfire. + ws = FakeWebSocket() + adapter = RealtimeWSAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + + await adapter.run_tick(0, None) + + assert len(_media_payloads(ws)) == FRAMES_PER_TICK diff --git a/tests/unit/user_simulator/cascade/test_simulator.py b/tests/unit/user_simulator/cascade/test_simulator.py index 91cd3e58..ce57788d 100644 --- a/tests/unit/user_simulator/cascade/test_simulator.py +++ b/tests/unit/user_simulator/cascade/test_simulator.py @@ -1,6 +1,3 @@ -import logging - -from eva.models.config import PerturbationConfig from eva.user_simulator.cascade.simulator import CascadeUserSimulator, extract_turn, parse_turn_response @@ -46,22 +43,10 @@ class _Message: assert extract_turn(_Message()) == ("Go on.", False) -def test_warn_unsupported_perturbation_fires_for_background_noise(caplog): - with caplog.at_level(logging.WARNING): - CascadeUserSimulator._warn_unsupported_perturbation(PerturbationConfig(background_noise="road_noise")) - assert any("background_noise" in record.message for record in caplog.records) - - -def test_warn_unsupported_perturbation_is_silent_for_a_default_config(caplog): - with caplog.at_level(logging.WARNING): - CascadeUserSimulator._warn_unsupported_perturbation(PerturbationConfig()) - assert caplog.records == [] - - -def test_warn_unsupported_perturbation_is_silent_for_none(caplog): - with caplog.at_level(logging.WARNING): - CascadeUserSimulator._warn_unsupported_perturbation(None) - assert caplog.records == [] +def test_outbound_perturbation_reaches_the_adapter(): + # background_noise / connection_degradation are applied per tick by RealtimeWSAdapter, + # so the cascade simulator no longer warns that it drops them. + assert not hasattr(CascadeUserSimulator, "_warn_unsupported_perturbation") def _make_bare_simulator() -> CascadeUserSimulator: From 90e29f24b2a11c289b6ede12f09ae4afcbad1612 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Fri, 7 Aug 2026 19:37:16 -0400 Subject: [PATCH 41/65] add speculative interruption generation with relevance gate --- configs/prompts/simulation.yaml | 38 ++++++++++ src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/simulator.py | 74 ++++++++++++++++++- .../user_simulator/cascade/test_simulator.py | 27 +++++++ 4 files changed, 139 insertions(+), 2 deletions(-) diff --git a/configs/prompts/simulation.yaml b/configs/prompts/simulation.yaml index 23e9d149..e06fb340 100644 --- a/configs/prompts/simulation.yaml +++ b/configs/prompts/simulation.yaml @@ -613,3 +613,41 @@ user_simulator: - Reply with ONLY the spoken line, nothing else. No quotes, no labels. - If the intended line contains no detail that could plausibly be misspoken (a greeting, a thank-you, a yes/no acknowledgement), reply with exactly NONE. + + cascade_next_interruption: | + You just said this line to the agent: + + + {utterance} + + + Write the single line you would say to cut the agent off mid-reply, if its + answer turns out to need correcting or clarifying. + + Rules: + - Write it as a natural interruption, not a full turn. Short. + - It must follow from your own goal, not from any specific thing the agent + might say — you have not heard the reply yet. + - Reply with ONLY the spoken line, nothing else. No quotes, no labels. + - If you would have no reason to interrupt whatever the agent says next, + reply with exactly NONE. + + cascade_relevance_gate: | + The caller prepared this line a moment ago, before hearing the rest of what + the agent is currently saying: + + + {candidate} + + + Here is what the agent has said so far in its current turn: + + + {heard} + + + Would saying the prepared line right now still make sense as a natural + interruption? Answer NO if it has become irrelevant, already been addressed, + or would read as a non-sequitur. + + Respond with ONLY "YES" or "NO". diff --git a/src/eva/__init__.py b/src/eva/__init__.py index a9141aa6..8bc6f8e7 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.27" +simulation_version = "2.0.28" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py index 31da22f2..7c2bec1f 100644 --- a/src/eva/user_simulator/cascade/simulator.py +++ b/src/eva/user_simulator/cascade/simulator.py @@ -24,7 +24,7 @@ TRANSCRIPT_WAIT_MS, ms_to_ticks, ) -from eva.user_simulator.cascade.decisions import ListenerDecisions +from eva.user_simulator.cascade.decisions import ListenerDecisions, parse_yes_no from eva.user_simulator.cascade.phrase_cache import PhraseCache from eva.user_simulator.cascade.scheduler import TickScheduler from eva.user_simulator.cascade.stt_livekit import LiveKitStreamingSTT @@ -103,6 +103,21 @@ def should_drop_interrupt(*, slip_ms: int, assistant_still_speaking: bool) -> bo return slip_ms > MAX_INTERRUPT_SLIP_MS or not assistant_still_speaking +async def candidate_is_relevant(llm, *, candidate: str, heard: str) -> bool: + """Whether a pre-generated interruption still fits what the assistant is saying. + + Fails closed: an unusable answer means we fall back to generating fresh + content rather than firing something that may have gone stale. + """ + prompt = PromptManager().get_prompt("user_simulator.cascade_relevance_gate", candidate=candidate, heard=heard) + try: + reply = await llm.decide(prompt) + except Exception as exc: + logger.warning(f"Relevance gate failed, discarding candidate: {exc}") + return False + return parse_yes_no(reply) + + def play_backchannel(scheduler, cache, phrases: list[str]) -> str: """Queue a cached continuer and return the phrase that was chosen.""" phrase = cache.choose(phrases) @@ -170,6 +185,8 @@ def __init__( self._rng = random.Random(0) self._armed_correction: bytes = b"" self._armed_correction_text = "" + self._candidate_text = "" + self._candidate_audio = b"" async def run_conversation(self) -> str: """Run the tick loop until the call ends, and return the end reason.""" @@ -310,6 +327,31 @@ async def _play_interruption(self, scheduler: TickScheduler) -> None: scheduler.enqueue_utterance(opener_audio) self._record_audio("user_clean", opener_audio) + if self._config.speculative_generation and self._candidate_audio: + candidate, audio = self._candidate_text, self._candidate_audio + self._candidate_text, self._candidate_audio = "", b"" + if await candidate_is_relevant( + self._decision_client, candidate=candidate, heard=self._stt.buffer.current_text() + ): + scheduler.enqueue_utterance(audio) + self._record_audio("user_clean", audio) + self._history.append({"role": "user", "content": candidate}) + self._on_user_speaks(candidate) + self.event_logger.log_event( + "interruption", + { + "text": candidate, + "opener": opener, + "intended_tick": intended_tick, + "actual_tick": scheduler.tick, + "slip_ms": interrupt_slip_ms(intended_tick=intended_tick, actual_tick=scheduler.tick), + "speculative": True, + "dropped": False, + }, + ) + return + self.event_logger.log_event("interruption_candidate_rejected", {"text": candidate}) + # Consumed, not peeked: leaving it in the buffer would re-append the same # assistant prefix at the next ordinary turn and duplicate it in the history. heard = self._stt.buffer.current_text() @@ -436,6 +478,11 @@ async def _take_turn(self, scheduler: TickScheduler, heard: str) -> bool: if end_call: self._on_conversation_end("goodbye") return True + + # After the audio is queued: the caller is now speaking, so this generation + # runs behind its own outgoing audio and costs no conversational latency. + if utterance: + await self._prerender_candidate(utterance) return False def _fire_self_correction(self, scheduler: TickScheduler) -> None: @@ -451,6 +498,31 @@ def _fire_self_correction(self, scheduler: TickScheduler) -> None: self._armed_correction = b"" self._armed_correction_text = "" + async def _prerender_candidate(self, utterance: str) -> None: + """Pre-generate and pre-render the line the caller would barge in with. + + Done on the caller's own turn, where the latency is already hidden behind + its outgoing audio, so a later barge-in can fire without waiting on + generation. The relevance gate is what keeps this from degrading into a + scripted interruption that lands as a non-sequitur. + """ + self._candidate_text, self._candidate_audio = "", b"" + if not self._config.speculative_generation: + return + prompt = PromptManager().get_prompt("user_simulator.cascade_next_interruption", utterance=utterance) + try: + message, _stats = await self._llm.complete( + messages=[*self._messages(), {"role": "user", "content": prompt}] + ) + except Exception as exc: + logger.warning(f"Speculative interruption generation failed: {exc}") + return + candidate = extract_correction(message) + if not candidate: + return + self._candidate_text = candidate + self._candidate_audio = await self._tts.synthesize(candidate, voice_id=self._voice_id) + def _drop_stale_correction(self) -> None: """Abandon an armed correction whose assistant turn never arrived. diff --git a/tests/unit/user_simulator/cascade/test_simulator.py b/tests/unit/user_simulator/cascade/test_simulator.py index ce57788d..bf7f7523 100644 --- a/tests/unit/user_simulator/cascade/test_simulator.py +++ b/tests/unit/user_simulator/cascade/test_simulator.py @@ -468,3 +468,30 @@ async def test_self_correction_is_skipped_when_the_behavior_is_disabled(): sim._config = CascadeSimulatorConfig() assert await sim._maybe_arm_self_correction("Book me Friday.") == "Book me Friday." + + +async def test_relevance_gate_allows_a_still_relevant_candidate(): + from eva.user_simulator.cascade.simulator import candidate_is_relevant + from tests.unit.user_simulator.cascade.test_decisions import FakeLLM + + llm = FakeLLM(["YES"]) + + assert await candidate_is_relevant(llm, candidate="I wanted Friday.", heard="Booking Thursday now") is True + + +async def test_relevance_gate_rejects_a_stale_candidate(): + from eva.user_simulator.cascade.simulator import candidate_is_relevant + from tests.unit.user_simulator.cascade.test_decisions import FakeLLM + + llm = FakeLLM(["NO"]) + + assert await candidate_is_relevant(llm, candidate="I wanted Friday.", heard="What is your name?") is False + + +async def test_relevance_gate_fails_closed(): + from eva.user_simulator.cascade.simulator import candidate_is_relevant + from tests.unit.user_simulator.cascade.test_decisions import FakeLLM + + llm = FakeLLM(error=RuntimeError("down")) + + assert await candidate_is_relevant(llm, candidate="x", heard="y") is False From cb834b75c15cd0d819e3c2d30ae0d713e83c8051 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Sat, 8 Aug 2026 04:49:54 -0400 Subject: [PATCH 42/65] stop a backchannel from consuming the caller's turn --- src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/scheduler.py | 18 ++++++++- src/eva/user_simulator/cascade/simulator.py | 8 +++- .../user_simulator/cascade/test_scheduler.py | 38 +++++++++++++++++++ .../user_simulator/cascade/test_simulator.py | 8 +++- 5 files changed, 69 insertions(+), 5 deletions(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 8bc6f8e7..e94e6a08 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.28" +simulation_version = "2.0.29" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/scheduler.py b/src/eva/user_simulator/cascade/scheduler.py index f104bee0..b57cc51f 100644 --- a/src/eva/user_simulator/cascade/scheduler.py +++ b/src/eva/user_simulator/cascade/scheduler.py @@ -35,6 +35,7 @@ def __init__(self, adapter: Adapter, *, bytes_per_tick: int = BYTES_PER_TICK) -> self._assistant_has_spoken = False self._awaiting_reply = False self._caller_spoke_this_tick = False + self._backchannel_bytes = 0 @property def tick(self) -> int: @@ -50,6 +51,16 @@ def enqueue_utterance(self, audio: bytes) -> None: """ self._playout.extend(audio) + def enqueue_backchannel(self, audio: bytes) -> None: + """Append a continuer, which sounds but does not take the caller's turn. + + A backchannel earns no reply, so counting it as a turn leaves the caller + waiting for one that never comes and the call dies at the inactivity + timeout instead of reaching a goodbye. + """ + self._backchannel_bytes += len(audio) + self._playout.extend(audio) + @property def caller_is_speaking(self) -> bool: """Whether caller audio is still queued for playout.""" @@ -110,6 +121,11 @@ async def run_tick(self) -> TickResult: result = await self._adapter.run_tick(self._tick, outgoing) del self._playout[:consumed] + # Backchannel bytes sit at the head of the queue, so this tick is a continuer + # only while they remain. Anything past them is real speech and takes the turn. + was_backchannel = consumed > 0 and self._backchannel_bytes > 0 + self._backchannel_bytes = max(0, self._backchannel_bytes - consumed) + self._caller_spoke_this_tick = outgoing is not None self._ticks_since_caller_speech = 0 if outgoing else self._ticks_since_caller_speech + 1 self._ticks_since_assistant_speech = ( @@ -118,7 +134,7 @@ async def run_tick(self) -> TickResult: self._assistant_has_spoken = self._assistant_has_spoken or result.has_assistant_speech if result.has_assistant_speech: self._awaiting_reply = False - if outgoing: + if outgoing and not was_backchannel: self._awaiting_reply = True self._tick += 1 diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py index 7c2bec1f..224b16aa 100644 --- a/src/eva/user_simulator/cascade/simulator.py +++ b/src/eva/user_simulator/cascade/simulator.py @@ -119,9 +119,13 @@ async def candidate_is_relevant(llm, *, candidate: str, heard: str) -> bool: def play_backchannel(scheduler, cache, phrases: list[str]) -> str: - """Queue a cached continuer and return the phrase that was chosen.""" + """Queue a cached continuer and return the phrase that was chosen. + + Queued as a backchannel, not an utterance: it must not consume the caller's + turn, or the caller waits for a reply the continuer never earns. + """ phrase = cache.choose(phrases) - scheduler.enqueue_utterance(cache.get(phrase)) + scheduler.enqueue_backchannel(cache.get(phrase)) return phrase diff --git a/tests/unit/user_simulator/cascade/test_scheduler.py b/tests/unit/user_simulator/cascade/test_scheduler.py index 6e6e7a43..1d236526 100644 --- a/tests/unit/user_simulator/cascade/test_scheduler.py +++ b/tests/unit/user_simulator/cascade/test_scheduler.py @@ -260,3 +260,41 @@ async def test_not_a_check_tick_while_the_caller_is_speaking(): await scheduler.run_tick() assert scheduler.is_check_tick() is False + + +async def test_a_backchannel_does_not_consume_the_callers_turn(): + # A continuer earns no reply, so treating it as a turn deadlocks may_take_turn() + # until the inactivity timeout — 7/8 live conversations died this way. + scheduler = _scheduler([True] + [False] * 60) + await scheduler.run_tick() # tick 0: assistant greets + scheduler.enqueue_backchannel(b"\x02" * BYTES_PER_TICK) + await scheduler.run_tick() # caller says "mm-hmm" + + for _ in range(40): + await scheduler.run_tick() + + assert scheduler.may_take_turn() is True + + +async def test_a_real_utterance_still_consumes_the_turn(): + scheduler = _scheduler([True] + [False] * 60) + await scheduler.run_tick() + scheduler.enqueue_utterance(b"\x02" * BYTES_PER_TICK) + await scheduler.run_tick() + + for _ in range(40): + await scheduler.run_tick() + + assert scheduler.may_take_turn() is False + + +async def test_an_utterance_queued_after_a_backchannel_still_consumes_the_turn(): + scheduler = _scheduler([True] + [False] * 60) + await scheduler.run_tick() + scheduler.enqueue_backchannel(b"\x02" * BYTES_PER_TICK) + scheduler.enqueue_utterance(b"\x03" * BYTES_PER_TICK) + + for _ in range(40): + await scheduler.run_tick() + + assert scheduler.may_take_turn() is False diff --git a/tests/unit/user_simulator/cascade/test_simulator.py b/tests/unit/user_simulator/cascade/test_simulator.py index bf7f7523..dd3b138c 100644 --- a/tests/unit/user_simulator/cascade/test_simulator.py +++ b/tests/unit/user_simulator/cascade/test_simulator.py @@ -274,11 +274,15 @@ class RecordingScheduler: def __init__(self) -> None: self.queued: list[bytes] = [] + self.backchannels: list[bytes] = [] self.tick = 0 def enqueue_utterance(self, audio: bytes) -> None: self.queued.append(audio) + def enqueue_backchannel(self, audio: bytes) -> None: + self.backchannels.append(audio) + class StubCache: """Phrase cache stand-in that always picks the first phrase.""" @@ -297,7 +301,9 @@ async def test_backchannel_queues_cached_audio_without_synthesis(): played = module.play_backchannel(scheduler, StubCache(), ["uh-huh", "mm-hmm"]) assert played == "uh-huh" - assert scheduler.queued == [b"CACHED"] + # Queued as a backchannel so it does not consume the caller's turn. + assert scheduler.backchannels == [b"CACHED"] + assert scheduler.queued == [] def test_verdict_with_no_action_queues_nothing(): From 7dba2ba71ca3ada21912f49a9c6a9301a38c56a9 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Tue, 18 Aug 2026 20:32:54 -0700 Subject: [PATCH 43/65] fix swallowed hang-up and unreachable self-correction in the cascade caller - _play_interruption discarded extract_turn's end_call flag, so a caller that decided to hang up mid-assistant-turn emitted nothing and the conversation ran on to the inactivity timeout. Observed live: six empty interruptions after the caller said goodbye, with the assistant looping. A hang-up is also never treated as stale. - Slip was computed from tick deltas, but run_tick is only pumped by _run, so no tick can advance during the generation await: 525/525 interruptions reported slip_ms=0 and the staleness check never engaged. Measured on the wall clock now. - The self-correction gate reseeded random.Random(0) every conversation, whose first draw below SELF_CORRECTION_RATE is #26 while a conversation runs ~7 turns, so no correction ever armed. Seeded per record id instead. --- src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/simulator.py | 88 ++++++---- .../user_simulator/cascade/test_simulator.py | 152 ++++++++++++++++-- 3 files changed, 202 insertions(+), 40 deletions(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index e94e6a08..a24fc46e 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.29" +simulation_version = "2.0.30" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py index 224b16aa..bc6a5740 100644 --- a/src/eva/user_simulator/cascade/simulator.py +++ b/src/eva/user_simulator/cascade/simulator.py @@ -4,6 +4,8 @@ import random import re +import time +import zlib from pathlib import Path import websockets @@ -93,9 +95,25 @@ def should_fire_self_correction(*, ticks_since_assistant_started: int, assistant return ticks_since_assistant_started >= ms_to_ticks(SELF_CORRECTION_DELAY_MS) -def interrupt_slip_ms(*, intended_tick: int, actual_tick: int) -> int: - """How far past its intended tick a barge-in actually landed.""" - return max(0, actual_tick - intended_tick) * TICK_DURATION_MS +def interrupt_slip_ms(*, elapsed_s: float) -> int: + """How far past its intended moment a barge-in actually landed. + + Measured on the wall clock, not the tick counter. `run_tick` is only pumped by + the `_run` loop, so while `_play_interruption` awaits its generation no tick can + advance — a tick-delta slip is structurally always zero and the staleness check + built on it never engages. + """ + return max(0, int(elapsed_s * 1000)) + + +def correction_rng(conversation_id: str) -> random.Random: + """Seed the self-correction gate per conversation, reproducibly but not identically. + + Seeding every conversation with 0 made the gate unreachable: `Random(0)` first + falls below SELF_CORRECTION_RATE on draw 26, while a conversation runs ~7 turns, + so no conversation ever armed a correction. + """ + return random.Random(zlib.crc32(conversation_id.encode())) def should_drop_interrupt(*, slip_ms: int, assistant_still_speaking: bool) -> bool: @@ -186,7 +204,7 @@ def __init__( self._decision_client = _DecisionClient(LiteLLMClient(model=simulator_config.decision_llm)) self._phrase_cache: PhraseCache | None = None self._decisions: ListenerDecisions | None = None - self._rng = random.Random(0) + self._rng = correction_rng(self._record_id or "cascade") self._armed_correction: bytes = b"" self._armed_correction_text = "" self._candidate_text = "" @@ -248,8 +266,8 @@ async def _run(self) -> None: ): self._fire_self_correction(scheduler) continue - if scheduler.is_check_tick(): - await self._run_checks(scheduler) + if scheduler.is_check_tick() and await self._run_checks(scheduler): + break continue self._ticks_since_assistant_started = 0 if self._assistant_is_inactive(scheduler, result): @@ -297,35 +315,41 @@ async def _prepare_listener_behaviors(self) -> None: backchannel_prompt=prompts.get_template("user_simulator.backchannel_decision"), ) - async def _run_checks(self, scheduler: TickScheduler) -> None: - """Run the listener-reaction checks and act on the verdict.""" + async def _run_checks(self, scheduler: TickScheduler) -> bool: + """Run the listener-reaction checks and act on the verdict. True means hang up.""" if self._decisions is None or self._phrase_cache is None: - return + return False verdict = await self._decisions.evaluate( self._stt.buffer.current_text(), allow_interrupt=self._config.enable_interruptions, allow_backchannel=self._config.enable_backchannel, ) if verdict.should_interrupt: - await self._play_interruption(scheduler) - return + return await self._play_interruption(scheduler) if verdict.should_backchannel: phrase = play_backchannel(scheduler, self._phrase_cache, BACKCHANNEL_PHRASES) # Recorded too, or the saved clean track diverges from what went on the wire. self._record_audio("user_clean", self._phrase_cache.get(phrase)) self.event_logger.log_event("backchannel", {"text": phrase, "tick_index": scheduler.tick}) + return False - async def _play_interruption(self, scheduler: TickScheduler) -> None: + async def _play_interruption(self, scheduler: TickScheduler) -> bool: """Voice a cached opener immediately, then stream the real content behind it. + Returns True when the caller decided to hang up. The generation is offered the + end_call tool, so discarding that verdict here strands a caller that wants to + say goodbye mid-assistant-turn: it emits nothing and the call runs on to the + inactivity timeout. + The opener buys the lead time the content generation costs. If the content still arrives too late to be a barge-in, it is dropped rather than emitted - stale — and both ticks are logged either way, because that gap is the - empirical risk this design carries. + stale — and the slip is logged either way, because that gap is the empirical + risk this design carries. """ if self._phrase_cache is None: - return + return False intended_tick = scheduler.tick + started_at = time.monotonic() opener = self._phrase_cache.choose(BARGE_IN_OPENERS) opener_audio = self._phrase_cache.get(opener) scheduler.enqueue_utterance(opener_audio) @@ -348,12 +372,12 @@ async def _play_interruption(self, scheduler: TickScheduler) -> None: "opener": opener, "intended_tick": intended_tick, "actual_tick": scheduler.tick, - "slip_ms": interrupt_slip_ms(intended_tick=intended_tick, actual_tick=scheduler.tick), + "slip_ms": interrupt_slip_ms(elapsed_s=time.monotonic() - started_at), "speculative": True, "dropped": False, }, ) - return + return False self.event_logger.log_event("interruption_candidate_rejected", {"text": candidate}) # Consumed, not peeked: leaving it in the buffer would re-append the same @@ -365,10 +389,14 @@ async def _play_interruption(self, scheduler: TickScheduler) -> None: self._history.append({"role": "assistant", "content": heard}) self._on_assistant_speaks(heard) message, _stats = await self._llm.complete(messages=self._messages(), tools=[END_CALL_TOOL]) - utterance, _end_call = extract_turn(message) + utterance, end_call = extract_turn(message) - slip = interrupt_slip_ms(intended_tick=intended_tick, actual_tick=scheduler.tick) - dropped = should_drop_interrupt(slip_ms=slip, assistant_still_speaking=scheduler.assistant_is_speaking) + slip = interrupt_slip_ms(elapsed_s=time.monotonic() - started_at) + # A hang-up is never stale: the caller has decided the call is over, and + # dropping it here is what left conversations looping until the timeout. + dropped = not end_call and should_drop_interrupt( + slip_ms=slip, assistant_still_speaking=scheduler.assistant_is_speaking + ) self.event_logger.log_event( "interruption", { @@ -378,16 +406,22 @@ async def _play_interruption(self, scheduler: TickScheduler) -> None: "actual_tick": scheduler.tick, "slip_ms": slip, "dropped": dropped, + "end_call": end_call, }, ) - if dropped or not utterance: - return + if dropped: + return False - self._history.append({"role": "user", "content": utterance}) - self._on_user_speaks(utterance) - async for chunk in self._tts.stream(utterance, voice_id=self._voice_id): - self._record_audio("user_clean", chunk) - scheduler.enqueue_utterance(chunk) + if utterance: + self._history.append({"role": "user", "content": utterance}) + self._on_user_speaks(utterance) + async for chunk in self._tts.stream(utterance, voice_id=self._voice_id): + self._record_audio("user_clean", chunk) + scheduler.enqueue_utterance(chunk) + + if end_call: + self._on_conversation_end("goodbye") + return end_call def _assistant_is_inactive(self, scheduler: TickScheduler, result: TickResult) -> bool: """Whether the assistant has produced no audio for INACTIVITY_TIMEOUT_MS. diff --git a/tests/unit/user_simulator/cascade/test_simulator.py b/tests/unit/user_simulator/cascade/test_simulator.py index dd3b138c..1e49631a 100644 --- a/tests/unit/user_simulator/cascade/test_simulator.py +++ b/tests/unit/user_simulator/cascade/test_simulator.py @@ -315,18 +315,6 @@ def test_verdict_with_no_action_queues_nothing(): assert verdict.should_backchannel is False -def test_slip_is_measured_in_ticks_converted_to_ms(): - from eva.user_simulator.cascade.simulator import interrupt_slip_ms - - assert interrupt_slip_ms(intended_tick=10, actual_tick=15) == 1000 - - -def test_no_slip_when_the_interrupt_lands_on_its_intended_tick(): - from eva.user_simulator.cascade.simulator import interrupt_slip_ms - - assert interrupt_slip_ms(intended_tick=10, actual_tick=10) == 0 - - def test_interrupt_kept_when_slip_is_within_budget(): from eva.user_simulator.cascade.simulator import should_drop_interrupt @@ -501,3 +489,143 @@ async def test_relevance_gate_fails_closed(): llm = FakeLLM(error=RuntimeError("down")) assert await candidate_is_relevant(llm, candidate="x", heard="y") is False + + +def test_slip_is_measured_on_the_wall_clock_not_the_tick_counter(): + # The tick counter cannot advance during _play_interruption's await: run_tick is + # only pumped by _run, so a tick-delta slip is structurally always zero. + from eva.user_simulator.cascade.simulator import interrupt_slip_ms + + assert interrupt_slip_ms(elapsed_s=1.0) == 1000 + assert interrupt_slip_ms(elapsed_s=0.0) == 0 + assert interrupt_slip_ms(elapsed_s=2.4) == 2400 + + +def test_slip_never_reports_negative_for_a_clock_hiccup(): + from eva.user_simulator.cascade.simulator import interrupt_slip_ms + + assert interrupt_slip_ms(elapsed_s=-0.5) == 0 + + +def test_self_correction_rng_differs_per_conversation(): + # Seeding every conversation with 0 made the 15% gate unreachable: Random(0) + # first drops below 0.15 on draw 26, and conversations run ~7 turns. + from eva.user_simulator.cascade.simulator import correction_rng + + a = correction_rng("record-1") + b = correction_rng("record-2") + + assert [a.random() for _ in range(5)] != [b.random() for _ in range(5)] + + +def test_self_correction_rng_is_reproducible_for_the_same_conversation(): + from eva.user_simulator.cascade.simulator import correction_rng + + first = [correction_rng("record-7").random() for _ in range(3)] + again = [correction_rng("record-7").random() for _ in range(3)] + + assert first == again + + +def test_self_correction_gate_actually_opens_within_a_normal_conversation(): + # Across a realistic spread of records, the 15% rate must be reachable. + from eva.user_simulator.cascade.constants import SELF_CORRECTION_RATE + from eva.user_simulator.cascade.simulator import correction_rng + + turns_per_conversation = 7 + fired = 0 + for index in range(60): + rng = correction_rng(f"record-{index}") + if any(rng.random() < SELF_CORRECTION_RATE for _ in range(turns_per_conversation)): + fired += 1 + + assert fired > 20, f"only {fired}/60 conversations could ever self-correct" + + +class _EndCallMessage: + """LLM reply that hangs up via the tool and says nothing.""" + + content = "" + + class _Fn: + name = "end_call" + + class _Call: + function = None + + def __init__(self) -> None: + call = self._Call() + call.function = self._Fn() + self.tool_calls = [call] + + +def _interrupting_simulator(message): + """Bare simulator wired for _play_interruption only.""" + from eva.models.config import CascadeSimulatorConfig + from eva.user_simulator.cascade.stt import TranscriptBuffer + + sim = CascadeUserSimulator.__new__(CascadeUserSimulator) + sim._config = CascadeSimulatorConfig(enable_interruptions=True) + sim._history = [] + sim._voice_id = "voice-f" + sim._build_prompt = lambda: "SYSTEM PROMPT" + sim.event_logger = _FakeEventLogger() + sim._phrase_cache = StubCache() + sim._record_audio = lambda *a, **k: None + sim._on_user_speaks = lambda *a, **k: None + sim._on_assistant_speaks = lambda *a, **k: None + sim.ended = [] + sim._on_conversation_end = sim.ended.append + buffer = TranscriptBuffer() + buffer.committed = "Your account is unlocked." + sim._stt = type("_Stt", (), {"buffer": buffer})() + + class _Llm: + async def complete(self, messages, tools=None): + return message, {} + + class _Tts: + async def stream(self, text, *, voice_id): + yield text.encode() + + sim._llm, sim._tts = _Llm(), _Tts() + return sim + + +class _InterruptScheduler: + tick = 40 + assistant_is_speaking = True + + def __init__(self) -> None: + self.queued: list[bytes] = [] + + def enqueue_utterance(self, audio: bytes) -> None: + self.queued.append(audio) + + +async def test_a_hangup_during_an_interruption_ends_the_call(): + # Discarding end_call here stranded the caller: it emitted nothing and the + # conversation ran on to the inactivity timeout, looping the assistant. + sim = _interrupting_simulator(_EndCallMessage()) + + hung_up = await sim._play_interruption(_InterruptScheduler()) + + assert hung_up is True + assert sim.ended == ["goodbye"] + + +async def test_a_hangup_is_never_dropped_as_stale(): + sim = _interrupting_simulator(_EndCallMessage()) + + class _StoppedScheduler(_InterruptScheduler): + assistant_is_speaking = False # would normally drop the interruption + + assert await sim._play_interruption(_StoppedScheduler()) is True + assert sim.ended == ["goodbye"] + + +async def test_an_ordinary_interruption_does_not_end_the_call(): + sim = _interrupting_simulator("It says my account is locked out.") + + assert await sim._play_interruption(_InterruptScheduler()) is False + assert sim.ended == [] From 6d466aeb029ff3cdd353cb3e4e08efa46332b16e Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Tue, 18 Aug 2026 20:33:22 -0700 Subject: [PATCH 44/65] record ablation defect root causes and the corrected diagnosis for defect 4 --- docs/changelog_cascade_out_of_turn.md | 148 +++++++++++++++++++++++--- 1 file changed, 136 insertions(+), 12 deletions(-) diff --git a/docs/changelog_cascade_out_of_turn.md b/docs/changelog_cascade_out_of_turn.md index d1316a5b..062a2f1b 100644 --- a/docs/changelog_cascade_out_of_turn.md +++ b/docs/changelog_cascade_out_of_turn.md @@ -39,16 +39,140 @@ The handoff forbids `git add -A` at the repo root (many unrelated untracked file ## Progress -- [ ] Task 1: behavior constants and vocabularies -- [ ] Task 2: behavior flags on the config -- [ ] Task 3: check-tick predicate -- [ ] Task 4: decision prompts (+ `current_text()`) -- [ ] Task 5: decision checks -- [ ] Task 6: phrase cache -- [ ] Task 7: backchannel behavior -- [ ] Task 8: streaming TTS -- [ ] Task 9: reactive interruption -- [ ] Task 10: self-correction -- [ ] Task 11: ambient noise mixing -- [ ] Task 12: speculative generation +- [x] Task 1: behavior constants and vocabularies +- [x] Task 2: behavior flags on the config +- [x] Task 3: check-tick predicate +- [x] Task 4: decision prompts (+ `current_text()`) +- [x] Task 5: decision checks +- [x] Task 6: phrase cache +- [x] Task 7: backchannel behavior +- [x] Task 8: streaming TTS +- [x] Task 9: reactive interruption +- [x] Task 10: self-correction +- [x] Task 11: ambient noise mixing +- [x] Task 12: speculative generation - [ ] Task 13: live ablation verification + +## Further deviations found during implementation + +### Task 10's ordering had to be rebuilt for the separate-call design + +With a separate follow-up call the turn call has already produced the *correct* +utterance, so asking for "a correction" would have flipped a right answer into a wrong +one — inverting the plan's invariant and putting `must_have_criteria` at risk. Instead +the correction prompt asks for a deliberately WRONG variant of the already-generated +line; the slip is spoken as the turn and the model's original goal-consistent line is +armed as the correction. Wrong-then-right ordering is preserved exactly, and the +goal-consistent line is still what lands last. Certainty: high. + +### Task 11's tests contradicted Plan 1's always-send-silence invariant + +The plan's `run_tick` edit made sending conditional (`if outgoing: await send(...)`) and +its test asserted `media == []` with no perturbator. `RealtimeWSAdapter` deliberately +sends a full tick of frames every tick — real audio or synthesized silence — because +"gaps with no frames at all are what caused turn detection to misfire". Implemented as +mixing noise *into* what is already sent, never as gating whether to send. Tests assert +the invariant instead. Certainty: high — this is documented in the adapter's own docstring. + +### `_warn_unsupported_perturbation` removed + +It warned that cascade drops `background_noise`/`snr_db`/`connection_degradation`. +Task 11 makes all three take effect via `AudioPerturbator.apply()`, so the warning became +a false statement rather than a stale one. Certainty: high. + +### Decision prompts are loaded with `get_template`, not a round-tripped placeholder + +Plan Task 7 Step 5 suggested calling `get_prompt(..., conversation_history="{conversation_history}")` +so the placeholder survives for `ListenerDecisions` to fill later, and flagged that it might +not round-trip. `PromptManager.get_template()` returns the raw unformatted template, which +removes the failure mode entirely. Certainty: high. + +### Interruption consumes the transcript rather than peeking at it + +The plan's `_play_interruption` appends `buffer.current_text()` to history but leaves it in +the buffer, so the same assistant prefix is appended again at the next ordinary turn. +Consumed instead, and logged once via `_on_assistant_speaks`. Certainty: high. + +### Plan 1's `cascade_` guard test rewritten + +`test_no_cascade_specific_prompts_remain_in_the_prompt_file` asserted the substring +`cascade_` never appears in `simulation.yaml`. The invariant it protected is that no +cascade-only contract is layered onto the *turn call's* system prompt. Plan 2's prompts are +used only in their own standalone calls, so the test now asserts the real invariant: +`_messages()[0]["content"] == _build_prompt()`. Certainty: high — strictly stronger guard. + +## Defects found by the Task 13 ablation runs + +Four defects. All four surfaced only against live services; the unit suite was green +throughout, which is the same pattern the Plan 1 handoff warned about. + +### 1. A backchannel consumed the caller's turn — FIXED (cb834b75) + +`TickScheduler.run_tick` set `_awaiting_reply = True` on *any* outgoing audio. A continuer +earns no reply, so `may_take_turn()` blocked permanently and the call died at the +inactivity timeout. Live: 7/8 conversations vs 4/9 at baseline. Fixed with +`enqueue_backchannel()`, which tracks continuer bytes at the head of the playout queue. + +**Loose end, deliberately not fixed:** post-fix the backchannel run still ends 6 +timeout / 3 goodbye against baseline's 4/5, and the backchannel count fell 20 -> 6 +between runs unexplained. Candidate second mechanism: a backchannel still resets +`_ticks_since_caller_speech`, delaying the caller's own next turn by the full +`WAIT_TO_RESPOND_SELF_MS`. Unproven — may be n=9 noise. Do not call defect 1 closed. + +### 2. Slip was structurally unmeasurable — FIXED (7dba2ba7) + +`interrupt_slip_ms` differenced tick counters, but `run_tick` is only pumped by the `_run` +loop and `_play_interruption` is awaited from inside it, so `scheduler.tick` provably cannot +advance during the generation. All 525 logged interruptions reported `slip_ms=0` with +`intended_tick == actual_tick`, and `should_drop_interrupt`'s slip branch never engaged. +Now measured with `time.monotonic()`. Certainty: high — deductive, not statistical. + +### 3. Self-correction was unreachable — FIXED (7dba2ba7) + +`random.Random(0)` was reseeded identically per conversation. Its first draw below +`SELF_CORRECTION_RATE` (0.15) is #26, while a conversation runs ~7 turns, so no conversation +ever armed a correction — zero events across both configs that enabled it. Verified the CLI +flag *did* propagate (`config.json` showed `enable_self_correction: true`) before blaming the +RNG, specifically to avoid fixing the wrong layer. Now seeded from the record id via +`crc32`, keeping runs reproducible while differing across conversations. + +### 4. Degraded interruption runs — root cause was NOT what it first looked like + +Initial reading of the aggregates (62-107 interruptions against ~2.6 caller turns) suggested +runaway barge-ins and a missing cooldown. **That diagnosis was wrong.** Reading a single +conversation timeline end to end showed two different causes: + +**4a. The interruption path swallowed the hang-up — FIXED (7dba2ba7).** `_play_interruption` +did `utterance, _end_call = extract_turn(message)`, discarding the flag, while `_take_turn` +used it. A caller deciding to hang up mid-assistant-turn therefore emitted nothing (the model +called the tool instead of speaking) and the call could never end. Observed directly: after +`'Thanks. Goodbye.'` at tick 640, six interruptions with empty text at ticks 660-800 while +the assistant looped "Confirmed... your account is unlocked" and then "Sure. What can I help +you with?". This — not barge-in spam — is what depressed `task_completion` to 0.500. + +**4b. The interrupt check preempts ordinary turn-taking — NOT FIXED, design question.** +The gaps between logged "interruptions" were 10s, 18s, 50s, and their content was ordinary +replies ("Employee ID is E M P zero four eight two seven one"). These are normal turns +routed through the interruption path, not barge-ins, which is why `caller_turn` appeared to +collapse — the turns were relabelled. Structurally: `_run` only reaches `_take_turn` when the +assistant is silent, but the check fires while it is speaking, so with interruptions enabled +the caller preferentially speaks via the check at 2s granularity instead of waiting out the +1s silence gate. Knock-on effects: turns taking the interrupt path bypass +`_maybe_arm_self_correction` and `_prerender_candidate`, and log as `interruption` rather +than `caller_turn`, which affects metric turn numbering. Deciding how the check and the turn +gate should interact is a design change to Plan 2, left for the author. + +### Method note + +Both times an initial diagnosis was wrong (the "service outage" that was really machine +sleep, and the "interruption storm" that was really a swallowed hang-up), the error came from +reading *aggregate counts* and inferring a mechanism. Both were settled immediately by +reading *one sequence end to end*. Prefer a single full timeline over a summary table. + +### Still outstanding + +- 4b (above), and defect 1's unexplained residual. +- `_run_checks` awaits two LLM calls inline in the tick loop with no timeout, so a slow + decision provider stalls the wire. Fixing it means moving the checks off the tick loop. +- Task 13 steps 2-6 need re-running: step 5 has never produced data, and the `interrupt` and + `all-on` rows from the last pass aggregate retry attempts (12 event files, not 9). From 97448885c0873b067a046f936cf33f77529bf9cb Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Tue, 18 Aug 2026 22:04:33 -0700 Subject: [PATCH 45/65] gate barge-ins to a fraction of assistant turns, at most one each The interrupt decision prompt's YES criteria ('has the user heard enough to have a response ready?') describe ordinary conversational readiness, so it answered YES at the end of every assistant turn. Combined with the tick loop only checking the interrupt path while the assistant speaks, every caller utterance became an interruption and normal turn-taking was bypassed: logged barge-ins were 10-50s apart and carried ordinary replies, each with a spurious 'Actually-'/'Hold on-' opener. INTERRUPT_RATE is rolled once when an assistant turn starts, and the eligibility is cleared on firing, so a turn carries at most one barge-in. Uses its own RNG stream so enabling interruptions cannot shift the self-correction draws. --- src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/constants.py | 9 ++++ src/eva/user_simulator/cascade/simulator.py | 23 ++++++++- .../user_simulator/cascade/test_constants.py | 6 +++ .../user_simulator/cascade/test_simulator.py | 47 +++++++++++++++++++ 5 files changed, 85 insertions(+), 2 deletions(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index a24fc46e..62b9d2a7 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.30" +simulation_version = "2.0.31" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/constants.py b/src/eva/user_simulator/cascade/constants.py index 4a531cdd..4685001a 100644 --- a/src/eva/user_simulator/cascade/constants.py +++ b/src/eva/user_simulator/cascade/constants.py @@ -47,6 +47,15 @@ SELF_CORRECTION_RATE = 0.15 """Fraction of caller turns generated with a self-correction attached.""" +INTERRUPT_RATE = 0.15 +"""Fraction of assistant turns eligible for one barge-in. + +Rolled once when the assistant starts speaking, not once per check: the interrupt prompt's +YES criteria ("has the user heard enough to have a response ready?") are satisfied at the end +of every assistant turn, so ungated every caller utterance became an interruption and normal +turn-taking was bypassed entirely. +""" + BACKCHANNEL_PHRASES = ["uh-huh", "mm-hmm"] """Fixed continuer vocabulary (tau: voice_config.py:126). Pre-rendered at init.""" diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py index bc6a5740..8f47c44c 100644 --- a/src/eva/user_simulator/cascade/simulator.py +++ b/src/eva/user_simulator/cascade/simulator.py @@ -19,6 +19,7 @@ BARGE_IN_OPENERS, CALLER_SAMPLE_RATE, INACTIVITY_TIMEOUT_MS, + INTERRUPT_RATE, MAX_INTERRUPT_SLIP_MS, SELF_CORRECTION_DELAY_MS, SELF_CORRECTION_RATE, @@ -106,6 +107,17 @@ def interrupt_slip_ms(*, elapsed_s: float) -> int: return max(0, int(elapsed_s * 1000)) +def interrupt_allowed_this_turn(*, enabled: bool, roll: float) -> bool: + """Whether this assistant turn is eligible for one barge-in. + + Rolled once per assistant turn rather than per check, and cleared after firing, so a + turn carries at most one interruption. Without this the caller barged in on every turn: + the decision prompt's YES criteria describe ordinary conversational readiness, so it + answers YES as soon as the caller has something to say — which is always. + """ + return enabled and roll < INTERRUPT_RATE + + def correction_rng(conversation_id: str) -> random.Random: """Seed the self-correction gate per conversation, reproducibly but not identically. @@ -167,6 +179,7 @@ class CascadeUserSimulator(AbstractUserSimulator): _ticks_awaiting_transcript = 0 _ticks_assistant_silent = 0 _ticks_since_assistant_started = 0 + _may_interrupt_this_turn = False def __init__( self, @@ -205,6 +218,8 @@ def __init__( self._phrase_cache: PhraseCache | None = None self._decisions: ListenerDecisions | None = None self._rng = correction_rng(self._record_id or "cascade") + # Its own stream, so enabling one behavior cannot shift the other's draws. + self._interrupt_rng = correction_rng(f"{self._record_id or 'cascade'}:interrupt") self._armed_correction: bytes = b"" self._armed_correction_text = "" self._candidate_text = "" @@ -259,6 +274,11 @@ async def _run(self) -> None: caller_was_speaking = scheduler.caller_spoke_this_tick assistant_was_speaking = result.has_assistant_speech if result.has_assistant_speech: + if self._ticks_since_assistant_started == 0: + # One roll per assistant turn, so a turn carries at most one barge-in. + self._may_interrupt_this_turn = interrupt_allowed_this_turn( + enabled=self._config.enable_interruptions, roll=self._interrupt_rng.random() + ) self._ticks_since_assistant_started += 1 if self._armed_correction and should_fire_self_correction( ticks_since_assistant_started=self._ticks_since_assistant_started, @@ -321,10 +341,11 @@ async def _run_checks(self, scheduler: TickScheduler) -> bool: return False verdict = await self._decisions.evaluate( self._stt.buffer.current_text(), - allow_interrupt=self._config.enable_interruptions, + allow_interrupt=self._may_interrupt_this_turn, allow_backchannel=self._config.enable_backchannel, ) if verdict.should_interrupt: + self._may_interrupt_this_turn = False return await self._play_interruption(scheduler) if verdict.should_backchannel: phrase = play_backchannel(scheduler, self._phrase_cache, BACKCHANNEL_PHRASES) diff --git a/tests/unit/user_simulator/cascade/test_constants.py b/tests/unit/user_simulator/cascade/test_constants.py index e34196f7..5f591ac4 100644 --- a/tests/unit/user_simulator/cascade/test_constants.py +++ b/tests/unit/user_simulator/cascade/test_constants.py @@ -46,3 +46,9 @@ def test_self_correction_delay_is_shorter_than_the_check_interval(): # The correction should land while the assistant is still on its first reply. assert SELF_CORRECTION_DELAY_MS < LISTENER_CHECK_INTERVAL_MS + + +def test_interrupt_rate_is_occasional_not_constant(): + from eva.user_simulator.cascade.constants import INTERRUPT_RATE + + assert 0.0 < INTERRUPT_RATE < 0.5 diff --git a/tests/unit/user_simulator/cascade/test_simulator.py b/tests/unit/user_simulator/cascade/test_simulator.py index 1e49631a..bdca140c 100644 --- a/tests/unit/user_simulator/cascade/test_simulator.py +++ b/tests/unit/user_simulator/cascade/test_simulator.py @@ -629,3 +629,50 @@ async def test_an_ordinary_interruption_does_not_end_the_call(): assert await sim._play_interruption(_InterruptScheduler()) is False assert sim.ended == [] + + +def test_an_assistant_turn_is_eligible_to_be_interrupted_only_sometimes(): + from eva.user_simulator.cascade.constants import INTERRUPT_RATE + from eva.user_simulator.cascade.simulator import interrupt_allowed_this_turn + + assert interrupt_allowed_this_turn(enabled=True, roll=INTERRUPT_RATE / 2) is True + assert interrupt_allowed_this_turn(enabled=True, roll=0.99) is False + + +def test_no_assistant_turn_is_eligible_when_interruptions_are_disabled(): + from eva.user_simulator.cascade.simulator import interrupt_allowed_this_turn + + assert interrupt_allowed_this_turn(enabled=False, roll=0.0) is False + + +async def test_only_one_interruption_fires_per_assistant_turn(): + # The eligibility flag is cleared on firing, so a second check in the same + # assistant turn is offered allow_interrupt=False and cannot barge in again. + from eva.models.config import CascadeSimulatorConfig + from eva.user_simulator.cascade.decisions import ListenerVerdict + + sim = CascadeUserSimulator.__new__(CascadeUserSimulator) + sim._config = CascadeSimulatorConfig(enable_interruptions=True) + sim._phrase_cache = StubCache() + sim._may_interrupt_this_turn = True + sim.plays = 0 + offered = [] + + class _Decisions: + async def evaluate(self, text, *, allow_interrupt, allow_backchannel): + offered.append(allow_interrupt) + return ListenerVerdict(should_interrupt=allow_interrupt, should_backchannel=False) + + async def _play(scheduler): + sim.plays += 1 + return False + + sim._decisions = _Decisions() + sim._play_interruption = _play + sim._stt = type("_Stt", (), {"buffer": type("_B", (), {"current_text": staticmethod(lambda: "hi")})()})() + + await sim._run_checks(_InterruptScheduler()) + await sim._run_checks(_InterruptScheduler()) + + assert offered == [True, False] + assert sim.plays == 1 From 1d139c075e39429a558c131e4ed89216706c2160 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Tue, 18 Aug 2026 22:25:00 -0700 Subject: [PATCH 46/65] give the interrupt decision the caller's goal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The decision only saw the transcript, so it could not tell whether the caller still had anything left to accomplish — it authorized barge-ins on turns where the caller's next move was to hang up, producing an opener with no speech behind it. It now receives the goal, must/nice-to-have criteria, option-evaluation steps, and the resolution, failure and escalation conditions, with the field meanings explained in the prompt template so the explanations stay static across calls. Placed ahead of the conversation history so the block is prompt-cacheable. edge_cases and information_required are omitted: they describe how to answer questions, not whether the goal is finished. --- configs/prompts/simulation.yaml | 19 +++++ src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/decisions.py | 14 +++- src/eva/user_simulator/cascade/simulator.py | 31 ++++++++ .../user_simulator/cascade/test_decisions.py | 76 +++++++++++++++++++ .../user_simulator/cascade/test_prompt.py | 16 +++- 6 files changed, 153 insertions(+), 5 deletions(-) diff --git a/configs/prompts/simulation.yaml b/configs/prompts/simulation.yaml index e06fb340..85965f2a 100644 --- a/configs/prompts/simulation.yaml +++ b/configs/prompts/simulation.yaml @@ -511,6 +511,23 @@ user_simulator: interruption_decision: | You are analyzing a conversation to decide if the user should interrupt the agent. + This is what the user called to accomplish, so you can judge what they still have left + to do. The fields mean: + + - GOAL: what the user is trying to achieve overall. + - MUST HAVE: non-negotiable requirements. The user will never accept an outcome that + fails any of these, and will not hang up until all of them are met. + - NICE TO HAVE: things the user wants but will give up if necessary. + - HOW THEY EVALUATE OPTIONS: the steps the user follows when the agent presents choices. + - RESOLVED WHEN: the success condition. Once this is met the user says a brief goodbye + and ends the call, so there is nothing left worth interrupting for. + - FAILED WHEN: the failure condition. This also ends the call. + - ESCALATION: how the user handles being transferred to a live agent. + + + {user_goal} + + Conversation history (most recent at bottom): @@ -526,6 +543,8 @@ user_simulator: - Has the user heard enough to have a response, question, or correction ready? - Did the agent just complete the sentence which has all the pertinent information the user was looking for? - Do NOT repeatedly interrupt the agent if it has spoken only a few words (say less than 5 words). + - Are the user's request(s) basically accomplished and the user is likely to hang up on the next turn (if so it should not interrupt)? + - Is this a logical point in the conversation to interrupt? Respond with ONLY "YES" if the user should interrupt now, or "NO" if they should keep listening. diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 62b9d2a7..1c4eafe5 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.31" +simulation_version = "2.0.32" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/decisions.py b/src/eva/user_simulator/cascade/decisions.py index 81e5418e..32165057 100644 --- a/src/eva/user_simulator/cascade/decisions.py +++ b/src/eva/user_simulator/cascade/decisions.py @@ -39,10 +39,13 @@ class ListenerDecisions: never inject a barge-in that the caller never actually decided to make. """ - def __init__(self, llm: DecisionLLM, *, interrupt_prompt: str, backchannel_prompt: str) -> None: + def __init__( + self, llm: DecisionLLM, *, interrupt_prompt: str, backchannel_prompt: str, user_goal: str = "" + ) -> None: self._llm = llm self._interrupt_prompt = interrupt_prompt self._backchannel_prompt = backchannel_prompt + self._user_goal = user_goal async def evaluate( self, conversation_history: str, *, allow_interrupt: bool, allow_backchannel: bool @@ -55,11 +58,16 @@ async def evaluate( return ListenerVerdict(should_interrupt=interrupt, should_backchannel=backchannel and not interrupt) async def _check(self, template: str, conversation_history: str, *, enabled: bool) -> bool: - """Ask the model one YES/NO question, returning False on anything unexpected.""" + """Ask the model one YES/NO question, returning False on anything unexpected. + + Both templates are filled with the same arguments; `str.format` ignores the ones a + given prompt does not use, so the backchannel prompt needs no goal slot. + """ if not enabled: return False try: - reply = await self._llm.decide(template.format(conversation_history=conversation_history)) + filled = template.format(conversation_history=conversation_history, user_goal=self._user_goal) + reply = await self._llm.decide(filled) except Exception as exc: logger.warning(f"Listener check failed, defaulting to no action: {exc}") return False diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py index 8f47c44c..5444da97 100644 --- a/src/eva/user_simulator/cascade/simulator.py +++ b/src/eva/user_simulator/cascade/simulator.py @@ -107,6 +107,36 @@ def interrupt_slip_ms(*, elapsed_s: float) -> int: return max(0, int(elapsed_s * 1000)) +def summarize_goal(goal: dict) -> str: + """State what the caller wants and what would end the call, for the listener checks. + + Field meanings live in the prompt template rather than here, so the explanations stay + static across the many calls this decision makes rather than being rebuilt per call. + `edge_cases` and `information_required` are deliberately omitted: they are long and + describe how to answer questions, not whether the goal is finished. + """ + tree = goal.get("decision_tree", {}) or {} + sections = [ + ("GOAL", goal.get("high_level_user_goal")), + ("MUST HAVE", tree.get("must_have_criteria")), + ("NICE TO HAVE", tree.get("nice_to_have_criteria")), + ("HOW THEY EVALUATE OPTIONS", tree.get("negotiation_behavior")), + ("RESOLVED WHEN", tree.get("resolution_condition")), + ("FAILED WHEN", tree.get("failure_condition")), + ("ESCALATION", tree.get("escalation_behavior")), + ] + lines: list[str] = [] + for label, value in sections: + if not value: + continue + if isinstance(value, list): + lines.append(f"{label}:") + lines += [f"- {item}" for item in value] + else: + lines.append(f"{label}: {value}") + return "\n".join(lines) + + def interrupt_allowed_this_turn(*, enabled: bool, roll: float) -> bool: """Whether this assistant turn is eligible for one barge-in. @@ -333,6 +363,7 @@ async def _prepare_listener_behaviors(self) -> None: self._decision_client, interrupt_prompt=prompts.get_template("user_simulator.interruption_decision"), backchannel_prompt=prompts.get_template("user_simulator.backchannel_decision"), + user_goal=summarize_goal(self.goal), ) async def _run_checks(self, scheduler: TickScheduler) -> bool: diff --git a/tests/unit/user_simulator/cascade/test_decisions.py b/tests/unit/user_simulator/cascade/test_decisions.py index 206c4437..7145c06a 100644 --- a/tests/unit/user_simulator/cascade/test_decisions.py +++ b/tests/unit/user_simulator/cascade/test_decisions.py @@ -77,3 +77,79 @@ async def test_a_failing_check_fails_closed(): assert verdict.should_interrupt is False assert verdict.should_backchannel is False + + +async def test_the_goal_is_substituted_into_the_interrupt_prompt(): + llm = FakeLLM(["NO", "NO"]) + seen: list[str] = [] + + class _Recorder(FakeLLM): + async def decide(self, prompt: str) -> str: + seen.append(prompt) + return await super().decide(prompt) + + decisions = ListenerDecisions( + _Recorder(["NO", "NO"]), + interrupt_prompt="goal={user_goal} history={conversation_history}", + backchannel_prompt="b {conversation_history}", + user_goal="Unlock my account.", + ) + + await decisions.evaluate("AGENT: hello", allow_interrupt=True, allow_backchannel=True) + + assert "goal=Unlock my account." in seen[0] + assert llm.calls == 0 + + +async def test_a_backchannel_prompt_without_a_goal_slot_still_works(): + # str.format ignores unused keyword arguments, so one signature serves both prompts. + decisions = ListenerDecisions( + FakeLLM(["NO", "YES"]), + interrupt_prompt="i {conversation_history}", + backchannel_prompt="b {conversation_history}", + user_goal="Unlock my account.", + ) + + verdict = await decisions.evaluate("AGENT: hello", allow_interrupt=True, allow_backchannel=True) + + assert verdict.should_backchannel is True + + +def test_goal_summary_is_compact_and_names_what_is_left(): + from eva.user_simulator.cascade.simulator import summarize_goal + + summary = summarize_goal( + { + "high_level_user_goal": "Unlock my AD account.", + "decision_tree": { + "must_have_criteria": ["account unlocked"], + "nice_to_have_criteria": ["a case number"], + "negotiation_behavior": "take the fastest fix offered", + "resolution_condition": "user can sign in", + "failure_condition": "agent cannot unlock it", + "escalation_behavior": "do not ask for a live agent", + "edge_cases": ["a very long irrelevant edge case " * 40], + }, + } + ) + + for expected in ( + "Unlock my AD account.", + "account unlocked", + "a case number", + "take the fastest fix offered", + "user can sign in", + "agent cannot unlock it", + "do not ask for a live agent", + ): + assert expected in summary, expected + # edge_cases is long and describes how to answer questions, not whether the goal is done. + assert "irrelevant" not in summary + + +def test_goal_summary_omits_absent_fields_without_blank_labels(): + from eva.user_simulator.cascade.simulator import summarize_goal + + summary = summarize_goal({"high_level_user_goal": "Unlock my account.", "decision_tree": {}}) + + assert summary == "GOAL: Unlock my account." diff --git a/tests/unit/user_simulator/cascade/test_prompt.py b/tests/unit/user_simulator/cascade/test_prompt.py index ea7c91e6..8f651c2c 100644 --- a/tests/unit/user_simulator/cascade/test_prompt.py +++ b/tests/unit/user_simulator/cascade/test_prompt.py @@ -24,7 +24,9 @@ def test_the_turn_call_carries_no_cascade_specific_contract(): def test_interruption_decision_prompt_has_a_history_slot_and_binary_contract(): - prompt = PromptManager().get_prompt("user_simulator.interruption_decision", conversation_history="AGENT: hello") + prompt = PromptManager().get_prompt( + "user_simulator.interruption_decision", conversation_history="AGENT: hello", user_goal="Unlock my account." + ) assert "AGENT: hello" in prompt assert "YES" in prompt @@ -51,3 +53,15 @@ def test_self_correction_prompt_never_mentions_ending_the_call(): prompt = PromptManager().get_template("user_simulator.cascade_self_correction") assert "end_call" not in prompt + + +def test_interruption_decision_prompt_carries_the_user_goal(): + prompt = PromptManager().get_prompt( + "user_simulator.interruption_decision", + conversation_history="AGENT: hello", + user_goal="Get my account unlocked.", + ) + + assert "Get my account unlocked." in prompt + # The goodbye case the caller kept barging in on. + assert "likely to hang up" in prompt From abd1a42a133cd33394c68856f21ceb452ef308b2 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Tue, 18 Aug 2026 22:47:48 -0700 Subject: [PATCH 47/65] fix cumulative-silence timeout, turn boundaries, and orphaned openers Three fixes, all confirmed against live conversation data. 1. inactivity_timeout measured cumulative, not contiguous, silence. The check was only reached on silent ticks because the speech branch continued first, making its reset line unreachable in production, so scattered quiet ticks accumulated over the whole call. Measured: one conversation was killed for '120s of assistant silence' after 73.7s of actual silence. Now checked on every tick. This has been ending healthy conversations since Plan 1, and inflated inactivity_timeout in every run including baseline. 2. An assistant turn boundary was any single 200ms quiet tick, so a pause between sentences re-armed the interruption cap mid-utterance and one turn could collect several barge-ins. Now uses the same 1s threshold the turn gate already applies. 3. The barge-in opener was emitted before the content existed, to hide ~1s of generation latency. A hang-up or a stale drop then left an orphaned 'Actually-' on the wire with nothing behind it. Content is now generated first, so dropping costs no audio at all. --- src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/simulator.py | 62 ++++++++++++------ .../user_simulator/cascade/test_simulator.py | 65 +++++++++++++++++++ 3 files changed, 107 insertions(+), 22 deletions(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 1c4eafe5..79950d74 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.32" +simulation_version = "2.0.33" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py index 5444da97..6927a7a2 100644 --- a/src/eva/user_simulator/cascade/simulator.py +++ b/src/eva/user_simulator/cascade/simulator.py @@ -25,6 +25,7 @@ SELF_CORRECTION_RATE, TICK_DURATION_MS, TRANSCRIPT_WAIT_MS, + WAIT_TO_RESPOND_OTHER_MS, ms_to_ticks, ) from eva.user_simulator.cascade.decisions import ListenerDecisions, parse_yes_no @@ -137,6 +138,17 @@ def summarize_goal(goal: dict) -> str: return "\n".join(lines) +def is_new_assistant_turn(*, ticks_silent_before: int) -> bool: + """Whether assistant audio arriving now starts a new turn or resumes the current one. + + A pause shorter than the turn-end threshold is a gap *inside* one turn — between + sentences, or a hole in the audio stream — not a new turn. Treating any single quiet + tick as a boundary re-armed the interruption cap mid-utterance, so one assistant turn + could collect several barge-ins. + """ + return ticks_silent_before >= ms_to_ticks(WAIT_TO_RESPOND_OTHER_MS) + + def interrupt_allowed_this_turn(*, enabled: bool, roll: float) -> bool: """Whether this assistant turn is eligible for one barge-in. @@ -303,8 +315,18 @@ async def _run(self) -> None: self._log_audio_boundaries(scheduler, result, assistant_was_speaking, caller_was_speaking) caller_was_speaking = scheduler.caller_spoke_this_tick assistant_was_speaking = result.has_assistant_speech + # Captured before the inactivity check, which clears it on a speech tick. + silent_before = self._ticks_assistant_silent + if self._assistant_is_inactive(scheduler, result): + logger.warning( + f"tick {scheduler.tick}: assistant silent for " + f"{INACTIVITY_TIMEOUT_MS // 1000}s; ending the conversation" + ) + self._on_conversation_end("inactivity_timeout") + break if result.has_assistant_speech: - if self._ticks_since_assistant_started == 0: + if is_new_assistant_turn(ticks_silent_before=silent_before): + self._ticks_since_assistant_started = 0 # One roll per assistant turn, so a turn carries at most one barge-in. self._may_interrupt_this_turn = interrupt_allowed_this_turn( enabled=self._config.enable_interruptions, roll=self._interrupt_rng.random() @@ -319,14 +341,6 @@ async def _run(self) -> None: if scheduler.is_check_tick() and await self._run_checks(scheduler): break continue - self._ticks_since_assistant_started = 0 - if self._assistant_is_inactive(scheduler, result): - logger.warning( - f"tick {scheduler.tick}: assistant silent for " - f"{INACTIVITY_TIMEOUT_MS // 1000}s; ending the conversation" - ) - self._on_conversation_end("inactivity_timeout") - break if scheduler.caller_is_speaking or not scheduler.may_take_turn(): continue heard, waiting = self._collect_heard_text(scheduler) @@ -386,17 +400,15 @@ async def _run_checks(self, scheduler: TickScheduler) -> bool: return False async def _play_interruption(self, scheduler: TickScheduler) -> bool: - """Voice a cached opener immediately, then stream the real content behind it. + """Decide what to say, then voice the opener and the content together. - Returns True when the caller decided to hang up. The generation is offered the - end_call tool, so discarding that verdict here strands a caller that wants to - say goodbye mid-assistant-turn: it emits nothing and the call runs on to the - inactivity timeout. + Returns True when the caller decided to hang up. - The opener buys the lead time the content generation costs. If the content - still arrives too late to be a barge-in, it is dropped rather than emitted - stale — and the slip is logged either way, because that gap is the empirical - risk this design carries. + The content is generated *before* anything reaches the wire. Emitting the opener + first hid its ~1s latency, but committed the caller to barging in before knowing + whether it had anything to say: a hang-up or a dropped-as-stale line then left an + orphaned "Actually—" hanging with nothing behind it, which is worse than either a + late line or silence. Generating first means "say nothing" is actually available. """ if self._phrase_cache is None: return False @@ -404,8 +416,6 @@ async def _play_interruption(self, scheduler: TickScheduler) -> bool: started_at = time.monotonic() opener = self._phrase_cache.choose(BARGE_IN_OPENERS) opener_audio = self._phrase_cache.get(opener) - scheduler.enqueue_utterance(opener_audio) - self._record_audio("user_clean", opener_audio) if self._config.speculative_generation and self._candidate_audio: candidate, audio = self._candidate_text, self._candidate_audio @@ -413,6 +423,8 @@ async def _play_interruption(self, scheduler: TickScheduler) -> bool: if await candidate_is_relevant( self._decision_client, candidate=candidate, heard=self._stt.buffer.current_text() ): + scheduler.enqueue_utterance(opener_audio) + self._record_audio("user_clean", opener_audio) scheduler.enqueue_utterance(audio) self._record_audio("user_clean", audio) self._history.append({"role": "user", "content": candidate}) @@ -461,10 +473,13 @@ async def _play_interruption(self, scheduler: TickScheduler) -> bool: "end_call": end_call, }, ) + # Nothing has reached the wire yet, so a stale line or a hang-up costs no audio. if dropped: return False if utterance: + scheduler.enqueue_utterance(opener_audio) + self._record_audio("user_clean", opener_audio) self._history.append({"role": "user", "content": utterance}) self._on_user_speaks(utterance) async for chunk in self._tts.stream(utterance, voice_id=self._voice_id): @@ -476,11 +491,16 @@ async def _play_interruption(self, scheduler: TickScheduler) -> bool: return end_call def _assistant_is_inactive(self, scheduler: TickScheduler, result: TickResult) -> bool: - """Whether the assistant has produced no audio for INACTIVITY_TIMEOUT_MS. + """Whether the assistant has produced no audio for INACTIVITY_TIMEOUT_MS *contiguously*. Mirrors ElevenLabsUserSimulator's keep-alive rule so both providers record the same terminal state: conversation_valid_end treats inactivity_timeout with the user speaking last as a definitive end, not a failure. + + Must be called on every tick, speech or silence. It was previously reached only on + silent ticks, which made the reset below dead code: the counter then measured + *cumulative* silence over the whole call and killed healthy conversations once their + quiet ticks happened to total two minutes. """ if result.has_assistant_speech: self._ticks_assistant_silent = 0 diff --git a/tests/unit/user_simulator/cascade/test_simulator.py b/tests/unit/user_simulator/cascade/test_simulator.py index bdca140c..add11e65 100644 --- a/tests/unit/user_simulator/cascade/test_simulator.py +++ b/tests/unit/user_simulator/cascade/test_simulator.py @@ -676,3 +676,68 @@ async def _play(scheduler): assert offered == [True, False] assert sim.plays == 1 + + +def test_a_short_pause_inside_one_assistant_turn_is_not_a_new_turn(): + # A single quiet tick used to re-arm the interruption cap mid-utterance. + from eva.user_simulator.cascade.simulator import is_new_assistant_turn + + assert is_new_assistant_turn(ticks_silent_before=1) is False + assert is_new_assistant_turn(ticks_silent_before=4) is False + + +def test_a_sustained_gap_starts_a_new_assistant_turn(): + from eva.user_simulator.cascade.constants import WAIT_TO_RESPOND_OTHER_MS, ms_to_ticks + from eva.user_simulator.cascade.simulator import is_new_assistant_turn + + assert is_new_assistant_turn(ticks_silent_before=ms_to_ticks(WAIT_TO_RESPOND_OTHER_MS)) is True + + +def test_inactivity_measures_contiguous_silence_not_cumulative(): + # The reset was unreachable in the live loop, so scattered quiet ticks accumulated + # and killed healthy calls once they happened to total two minutes. + from eva.user_simulator.cascade.constants import INACTIVITY_TIMEOUT_MS, ms_to_ticks + + sim = CascadeUserSimulator.__new__(CascadeUserSimulator) + sim._ticks_assistant_silent = 0 + limit = ms_to_ticks(INACTIVITY_TIMEOUT_MS) + + # Almost time out, then the assistant speaks once, then go quiet again. + for _ in range(limit): + assert sim._assistant_is_inactive(_SilenceScheduler(), _tick(False)) is False + assert sim._assistant_is_inactive(_SilenceScheduler(), _tick(True)) is False + for _ in range(limit): + assert sim._assistant_is_inactive(_SilenceScheduler(), _tick(False)) is False + + assert sim._assistant_is_inactive(_SilenceScheduler(), _tick(False)) is True + + +async def test_a_dropped_interruption_emits_no_audio_at_all(): + # Emitting the opener up front meant a stale drop left it orphaned on the wire. + sim = _interrupting_simulator("Active Directory.") + + class _StaleScheduler(_InterruptScheduler): + assistant_is_speaking = False # forces should_drop_interrupt + + scheduler = _StaleScheduler() + assert await sim._play_interruption(scheduler) is False + assert scheduler.queued == [] + + +async def test_a_hangup_during_an_interruption_emits_no_opener(): + sim = _interrupting_simulator(_EndCallMessage()) + scheduler = _InterruptScheduler() + + assert await sim._play_interruption(scheduler) is True + assert scheduler.queued == [] + assert sim.ended == ["goodbye"] + + +async def test_a_kept_interruption_speaks_the_opener_then_the_content(): + sim = _interrupting_simulator("Active Directory.") + scheduler = _InterruptScheduler() + + assert await sim._play_interruption(scheduler) is False + + assert scheduler.queued[0] == b"CACHED" + assert b"Active Directory." in b"".join(scheduler.queued[1:]) From 3cb17e379583311ab0b666ab79b5e4ba5ac2215d Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Tue, 18 Aug 2026 22:51:19 -0700 Subject: [PATCH 48/65] remove the interrupt rate gate and rely on the goal-aware decision The rate gate was added when the decision looked like it always said YES. It does not: it says YES about 14% of checks, once per assistant turn, at the point the caller is ready to reply. The defect was where it fired, not how often, so a random dice roll suppressed decisions that were correct while leaving the misjudgement in place. Giving the decision the caller's goal fixes the judgement at its source, so the gate became a second suppressor stacked on a fix. The per-turn eligibility flag stays, since that is the one-barge-in-per-assistant-turn cap. --- src/eva/__init__.py | 2 +- src/eva/user_simulator/cascade/constants.py | 9 --------- src/eva/user_simulator/cascade/simulator.py | 18 +----------------- .../user_simulator/cascade/test_constants.py | 6 ------ .../user_simulator/cascade/test_simulator.py | 14 -------------- 5 files changed, 2 insertions(+), 47 deletions(-) diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 79950d74..d428aa37 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -7,7 +7,7 @@ # Bump simulation_version when changes affect benchmark outputs (agent code, # user simulator, orchestrator, simulation prompts, agent configs, tool mocks). -simulation_version = "2.0.33" +simulation_version = "2.0.34" # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). diff --git a/src/eva/user_simulator/cascade/constants.py b/src/eva/user_simulator/cascade/constants.py index 4685001a..4a531cdd 100644 --- a/src/eva/user_simulator/cascade/constants.py +++ b/src/eva/user_simulator/cascade/constants.py @@ -47,15 +47,6 @@ SELF_CORRECTION_RATE = 0.15 """Fraction of caller turns generated with a self-correction attached.""" -INTERRUPT_RATE = 0.15 -"""Fraction of assistant turns eligible for one barge-in. - -Rolled once when the assistant starts speaking, not once per check: the interrupt prompt's -YES criteria ("has the user heard enough to have a response ready?") are satisfied at the end -of every assistant turn, so ungated every caller utterance became an interruption and normal -turn-taking was bypassed entirely. -""" - BACKCHANNEL_PHRASES = ["uh-huh", "mm-hmm"] """Fixed continuer vocabulary (tau: voice_config.py:126). Pre-rendered at init.""" diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py index 6927a7a2..5e3e8fec 100644 --- a/src/eva/user_simulator/cascade/simulator.py +++ b/src/eva/user_simulator/cascade/simulator.py @@ -19,7 +19,6 @@ BARGE_IN_OPENERS, CALLER_SAMPLE_RATE, INACTIVITY_TIMEOUT_MS, - INTERRUPT_RATE, MAX_INTERRUPT_SLIP_MS, SELF_CORRECTION_DELAY_MS, SELF_CORRECTION_RATE, @@ -149,17 +148,6 @@ def is_new_assistant_turn(*, ticks_silent_before: int) -> bool: return ticks_silent_before >= ms_to_ticks(WAIT_TO_RESPOND_OTHER_MS) -def interrupt_allowed_this_turn(*, enabled: bool, roll: float) -> bool: - """Whether this assistant turn is eligible for one barge-in. - - Rolled once per assistant turn rather than per check, and cleared after firing, so a - turn carries at most one interruption. Without this the caller barged in on every turn: - the decision prompt's YES criteria describe ordinary conversational readiness, so it - answers YES as soon as the caller has something to say — which is always. - """ - return enabled and roll < INTERRUPT_RATE - - def correction_rng(conversation_id: str) -> random.Random: """Seed the self-correction gate per conversation, reproducibly but not identically. @@ -260,8 +248,6 @@ def __init__( self._phrase_cache: PhraseCache | None = None self._decisions: ListenerDecisions | None = None self._rng = correction_rng(self._record_id or "cascade") - # Its own stream, so enabling one behavior cannot shift the other's draws. - self._interrupt_rng = correction_rng(f"{self._record_id or 'cascade'}:interrupt") self._armed_correction: bytes = b"" self._armed_correction_text = "" self._candidate_text = "" @@ -328,9 +314,7 @@ async def _run(self) -> None: if is_new_assistant_turn(ticks_silent_before=silent_before): self._ticks_since_assistant_started = 0 # One roll per assistant turn, so a turn carries at most one barge-in. - self._may_interrupt_this_turn = interrupt_allowed_this_turn( - enabled=self._config.enable_interruptions, roll=self._interrupt_rng.random() - ) + self._may_interrupt_this_turn = self._config.enable_interruptions self._ticks_since_assistant_started += 1 if self._armed_correction and should_fire_self_correction( ticks_since_assistant_started=self._ticks_since_assistant_started, diff --git a/tests/unit/user_simulator/cascade/test_constants.py b/tests/unit/user_simulator/cascade/test_constants.py index 5f591ac4..e34196f7 100644 --- a/tests/unit/user_simulator/cascade/test_constants.py +++ b/tests/unit/user_simulator/cascade/test_constants.py @@ -46,9 +46,3 @@ def test_self_correction_delay_is_shorter_than_the_check_interval(): # The correction should land while the assistant is still on its first reply. assert SELF_CORRECTION_DELAY_MS < LISTENER_CHECK_INTERVAL_MS - - -def test_interrupt_rate_is_occasional_not_constant(): - from eva.user_simulator.cascade.constants import INTERRUPT_RATE - - assert 0.0 < INTERRUPT_RATE < 0.5 diff --git a/tests/unit/user_simulator/cascade/test_simulator.py b/tests/unit/user_simulator/cascade/test_simulator.py index add11e65..74efc7c0 100644 --- a/tests/unit/user_simulator/cascade/test_simulator.py +++ b/tests/unit/user_simulator/cascade/test_simulator.py @@ -631,20 +631,6 @@ async def test_an_ordinary_interruption_does_not_end_the_call(): assert sim.ended == [] -def test_an_assistant_turn_is_eligible_to_be_interrupted_only_sometimes(): - from eva.user_simulator.cascade.constants import INTERRUPT_RATE - from eva.user_simulator.cascade.simulator import interrupt_allowed_this_turn - - assert interrupt_allowed_this_turn(enabled=True, roll=INTERRUPT_RATE / 2) is True - assert interrupt_allowed_this_turn(enabled=True, roll=0.99) is False - - -def test_no_assistant_turn_is_eligible_when_interruptions_are_disabled(): - from eva.user_simulator.cascade.simulator import interrupt_allowed_this_turn - - assert interrupt_allowed_this_turn(enabled=False, roll=0.0) is False - - async def test_only_one_interruption_fires_per_assistant_turn(): # The eligibility flag is cleared on firing, so a second check in the same # assistant turn is offered allow_interrupt=False and cannot barge in again. From 4d08eba8b6167a12f295336652605aa1e9b73cb5 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Tue, 18 Aug 2026 23:00:11 -0700 Subject: [PATCH 49/65] add paced_output flag to the assistant server base Co-Authored-By: Claude Opus 5 (1M context) --- src/eva/assistant/base_server.py | 6 ++++++ tests/unit/assistant/test_base_server_pacing.py | 10 ++++++++++ 2 files changed, 16 insertions(+) create mode 100644 tests/unit/assistant/test_base_server_pacing.py diff --git a/src/eva/assistant/base_server.py b/src/eva/assistant/base_server.py index ec4dc4a7..934b248b 100644 --- a/src/eva/assistant/base_server.py +++ b/src/eva/assistant/base_server.py @@ -52,6 +52,7 @@ def __init__( port: int, conversation_id: str, language: str = "en", + paced_output: bool = True, ): """Initialize the assistant server. @@ -65,10 +66,15 @@ def __init__( port: Port to listen on conversation_id: Unique ID for this conversation language: BCP 47 language tag for STT/TTS/S2S services (e.g. 'en', 'fr', 'es-MX') + paced_output: Whether to emit audio at real-time cadence. True for callers + that infer turn boundaries from silence timing (the ElevenLabs simulator). + False for a tick-driven caller, which buffers whatever arrives and + releases it one tick at a time, so pacing here would only add latency. """ self.current_date_time = current_date_time self.pipeline_config = pipeline_config self.language = language + self.paced_output = paced_output self.initial_message = get_initial_message(language) self.agent: AgentConfig = agent self.agent_config_path = agent_config_path diff --git a/tests/unit/assistant/test_base_server_pacing.py b/tests/unit/assistant/test_base_server_pacing.py new file mode 100644 index 00000000..6a793faa --- /dev/null +++ b/tests/unit/assistant/test_base_server_pacing.py @@ -0,0 +1,10 @@ +import inspect + +from eva.assistant.base_server import AbstractAssistantServer + + +def test_paced_output_defaults_to_true(): + # The existing ElevenLabs caller depends on real-time cadence for its + # silence heuristics, so the default must not change. + signature = inspect.signature(AbstractAssistantServer.__init__) + assert signature.parameters["paced_output"].default is True From fcd7b6828eeee09315c6c482ace411ccf247c983 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Tue, 18 Aug 2026 23:04:31 -0700 Subject: [PATCH 50/65] honor paced_output and count user activity in audio deltas Co-Authored-By: Claude Opus 5 (1M context) --- src/eva/assistant/openai_realtime_server.py | 51 ++++++++--- .../assistant/test_openai_realtime_pacing.py | 89 +++++++++++++++++++ 2 files changed, 128 insertions(+), 12 deletions(-) create mode 100644 tests/unit/assistant/test_openai_realtime_pacing.py diff --git a/src/eva/assistant/openai_realtime_server.py b/src/eva/assistant/openai_realtime_server.py index 0c29f16f..b01a9965 100644 --- a/src/eva/assistant/openai_realtime_server.py +++ b/src/eva/assistant/openai_realtime_server.py @@ -38,12 +38,11 @@ MULAW_CHUNK_SIZE = 160 # bytes per chunk (20ms at 8kHz, 1 byte per sample) MULAW_CHUNK_DURATION_S = 0.02 # 20ms per chunk # Don't pad the user track to align with the assistant when real user audio -# arrived within this window. The speaking-state flag can go stale under -# event-loop jitter, and padding then injects silence into an active user -# utterance (the choppy-audio bug). This guard only ever *skips* a pad, so it -# can never add silence or worsen alignment — during genuine user silence the -# timestamp is old and padding proceeds normally. -USER_ACTIVE_GUARD_S = 0.3 +# arrived recently. The speaking-state flag can go stale under event-loop +# jitter, and padding then injects silence into an active user utterance (the +# choppy-audio bug). This guard only ever *skips* a pad, so it can never add +# silence or worsen alignment — during genuine user silence the counter has run +# past the threshold and padding proceeds normally. def _wall_ms() -> str: @@ -84,13 +83,20 @@ class OpenAIRealtimeAssistantServer(AbstractAssistantServer): _service_name: str = "OpenAI Realtime" _metrics_processor_name: str = "openai_realtime" + USER_ACTIVE_GUARD_DELTAS = 25 + """Assistant audio deltas after the last user audio before the user counts as idle. + + Counted in deltas rather than wall-clock seconds so a tick-driven caller that + pauses to think is not mistaken for a caller who stopped talking. + """ + def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) self._audio_sample_rate = OPENAI_SAMPLE_RATE - # Monotonic time of the last real user-audio frame appended; used to - # guard against padding the user track mid-utterance (see USER_ACTIVE_GUARD_S). - self._last_user_audio_mono: float = 0.0 + # Assistant audio deltas seen since the last real user-audio frame; used to + # guard against padding the user track mid-utterance (see USER_ACTIVE_GUARD_DELTAS). + self._deltas_since_user_audio: int = self.USER_ACTIVE_GUARD_DELTAS self._system_prompt: str = self._build_system_prompt() @@ -308,6 +314,20 @@ async def _handle_session(self, websocket: WebSocket) -> None: finally: logger.info(f"Client disconnected from {self._service_name} server") + # ── User activity tracking (tick-safe, no wall clock) ──────────── + + def note_user_audio(self) -> None: + """Record that user audio just arrived.""" + self._deltas_since_user_audio = 0 + + def note_assistant_delta(self) -> None: + """Record one assistant audio delta since the last user audio.""" + self._deltas_since_user_audio += 1 + + def user_recently_active(self) -> bool: + """Whether user audio arrived recently enough to skip padding its track.""" + return self._deltas_since_user_audio < self.USER_ACTIVE_GUARD_DELTAS + # ── Audio output pacer (OpenAI -> Twilio WS at real-time rate) ─── async def _pace_audio_output(self, websocket: WebSocket, audio_output_queue: asyncio.Queue[bytes]) -> None: @@ -331,6 +351,12 @@ async def _pace_audio_output(self, websocket: WebSocket, audio_output_queue: asy logger.error(f"Error sending audio to Twilio WS: {e}") return + if not self.paced_output: + # Tick-driven caller: it buffers what arrives and releases one tick + # per tick, so pacing here would only add latency without changing + # what the caller hears. + continue + now = time.monotonic() if next_send_time <= now: next_send_time = now @@ -385,7 +411,7 @@ async def _forward_user_audio(self, websocket: WebSocket, conn: Any) -> None: sync_buffer_to_position(self.assistant_audio_buffer, sync_target) synced = len(self.assistant_audio_buffer) - asst_before self.user_audio_buffer.extend(pcm16_24k) - self._last_user_audio_mono = time.monotonic() + self.note_user_audio() self._user_frame_count += 1 if self._user_frame_count % 50 == 0: diff = len(self.user_audio_buffer) - len(self.assistant_audio_buffer) @@ -587,12 +613,13 @@ async def _on_audio_delta(self, event: Any, audio_output_queue: asyncio.Queue[by if 0 < latency_ms < 30_000: self._metrics_log.write_latency("model_response", latency_ms / 1000, self._model) + self.note_assistant_delta() + user_before = len(self.user_audio_buffer) synced = 0 # Skip the pad if the user track is actively receiving audio (flag may be # stale under jitter) — padding then would inject a mid-utterance chop. - user_recently_active = (time.monotonic() - self._last_user_audio_mono) <= USER_ACTIVE_GUARD_S - if not self._user_speaking and not user_recently_active: + if not self._user_speaking and not self.user_recently_active(): sync_buffer_to_position(self.user_audio_buffer, len(self.assistant_audio_buffer)) synced = len(self.user_audio_buffer) - user_before self.assistant_audio_buffer.extend(pcm16_bytes) diff --git a/tests/unit/assistant/test_openai_realtime_pacing.py b/tests/unit/assistant/test_openai_realtime_pacing.py new file mode 100644 index 00000000..ccc2f15c --- /dev/null +++ b/tests/unit/assistant/test_openai_realtime_pacing.py @@ -0,0 +1,89 @@ +import asyncio +import time + +import pytest + +from eva.assistant.openai_realtime_server import OpenAIRealtimeAssistantServer + + +class FakeWS: + def __init__(self) -> None: + self.sent: list[str] = [] + + async def send_text(self, message: str) -> None: + self.sent.append(message) + + +def _server(paced: bool, tmp_path) -> OpenAIRealtimeAssistantServer: + from eva.models.agents import AgentConfig + from eva.models.config import ModelConfig + + db_path = tmp_path / "db.json" + db_path.write_text("{}") + + return OpenAIRealtimeAssistantServer( + current_date_time="2026-01-01T00:00:00", + pipeline_config=ModelConfig(s2s="gpt-realtime", s2s_params={"model": "gpt-realtime", "api_key": "k"}), + agent=AgentConfig( + id="agent_itsm", + name="agent_itsm", + role="r", + description="d", + instructions="i", + tool_module_path="eva.assistant.tools.itsm_tools", + ), + agent_config_path="configs/agents/itsm_agent.yaml", + scenario_db_path=str(db_path), + output_dir=tmp_path, + port=9999, + conversation_id="c1", + paced_output=paced, + ) + + +@pytest.mark.asyncio +async def test_unpaced_output_drains_without_sleeping(tmp_path): + server = _server(paced=False, tmp_path=tmp_path) + server._running = True + ws = FakeWS() + queue: asyncio.Queue[bytes] = asyncio.Queue() + for _ in range(20): + queue.put_nowait(b"\xff" * 160) + + started = time.monotonic() + task = asyncio.create_task(server._pace_audio_output(ws, queue)) + await asyncio.sleep(0.05) + task.cancel() + + # 20 paced chunks would take ~400ms; unpaced must finish inside the 50ms window. + assert len(ws.sent) == 20 + assert time.monotonic() - started < 0.2 + + +@pytest.mark.asyncio +async def test_paced_output_still_holds_real_time_cadence(tmp_path): + server = _server(paced=True, tmp_path=tmp_path) + server._running = True + ws = FakeWS() + queue: asyncio.Queue[bytes] = asyncio.Queue() + for _ in range(20): + queue.put_nowait(b"\xff" * 160) + + task = asyncio.create_task(server._pace_audio_output(ws, queue)) + await asyncio.sleep(0.05) + task.cancel() + + # ~20ms per chunk means only a handful land in a 50ms window. + assert len(ws.sent) < 10 + + +def test_user_recently_active_counts_audio_deltas_not_wall_time(tmp_path): + server = _server(paced=False, tmp_path=tmp_path) + + server.note_user_audio() + assert server.user_recently_active() is True + + # A long stall must not make a just-spoken user look idle. + for _ in range(server.USER_ACTIVE_GUARD_DELTAS): + server.note_assistant_delta() + assert server.user_recently_active() is False From a0eedc319224e4a1361e91b2331fa1ec2e5154f7 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Tue, 18 Aug 2026 23:05:22 -0700 Subject: [PATCH 51/65] add playback-position truncation fields to TickResult Co-Authored-By: Claude Opus 5 (1M context) --- src/eva/user_simulator/cascade/tick_result.py | 18 ++++++++++- .../cascade/test_tick_result.py | 32 ++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/src/eva/user_simulator/cascade/tick_result.py b/src/eva/user_simulator/cascade/tick_result.py index 9260a992..d4e66603 100644 --- a/src/eva/user_simulator/cascade/tick_result.py +++ b/src/eva/user_simulator/cascade/tick_result.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from typing import NamedTuple -from eva.user_simulator.cascade.constants import BYTES_PER_TICK, SILENCE_BYTE +from eva.user_simulator.cascade.constants import BYTES_PER_TICK, SILENCE_BYTE, TICK_DURATION_MS @dataclass(frozen=True) @@ -24,12 +24,28 @@ class TickResult: wall_clock_ms: int """Unix ms at the tick's I/O boundary. For latency metrics only, never for ordering.""" + skip_item_id: str | None = None + """Provider item whose remaining audio must be discarded after a barge-in.""" + + interruption_audio_start_ms: int | None = None + """Played position where the caller cut in, in simulated ms.""" + @property def has_assistant_speech(self) -> bool: """Whether any real assistant audio arrived this tick.""" return self.assistant_audio_raw_bytes > 0 +def played_audio_ms(*, ticks_released: int) -> int: + """Assistant audio actually released into the conversation, in simulated ms. + + This is deliberately not "bytes received": a realtime provider generates + faster than real time, so the provider's idea of playback position runs ahead + of what the caller has heard. Truncation must use this number. + """ + return ticks_released * TICK_DURATION_MS + + class TickAudioSplit(NamedTuple): """Result of splitting audio at a tick boundary.""" diff --git a/tests/unit/user_simulator/cascade/test_tick_result.py b/tests/unit/user_simulator/cascade/test_tick_result.py index f1696f65..57a338ed 100644 --- a/tests/unit/user_simulator/cascade/test_tick_result.py +++ b/tests/unit/user_simulator/cascade/test_tick_result.py @@ -1,6 +1,6 @@ import pytest -from eva.user_simulator.cascade.tick_result import TickResult, split_tick_audio +from eva.user_simulator.cascade.tick_result import TickResult, played_audio_ms, split_tick_audio @pytest.mark.parametrize("input_length", [0, 1, 7, 8, 9, 17]) @@ -42,3 +42,33 @@ def test_has_assistant_speech_is_true_when_real_audio_arrived(): result = TickResult(tick_number=3, assistant_audio=b"\x01" * 8, assistant_audio_raw_bytes=8, wall_clock_ms=1) assert result.has_assistant_speech is True + + +def test_played_position_reflects_released_ticks_not_received_bytes(): + # 12 ticks released at 200ms each, regardless of how much arrived early. + assert played_audio_ms(ticks_released=12) == 2400 + + +def test_played_position_is_zero_before_anything_is_released(): + assert played_audio_ms(ticks_released=0) == 0 + + +def test_truncation_defaults_are_inert(): + result = TickResult(tick_number=1, assistant_audio=b"\x00" * 8, assistant_audio_raw_bytes=0, wall_clock_ms=0) + + assert result.skip_item_id is None + assert result.interruption_audio_start_ms is None + + +def test_truncation_fields_carry_the_played_position(): + result = TickResult( + tick_number=1, + assistant_audio=b"\x00" * 8, + assistant_audio_raw_bytes=0, + wall_clock_ms=0, + skip_item_id="item_42", + interruption_audio_start_ms=2400, + ) + + assert result.skip_item_id == "item_42" + assert result.interruption_audio_start_ms == 2400 From af64efaa60f5a392551bf678d01acb18297eb875 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Tue, 18 Aug 2026 23:07:27 -0700 Subject: [PATCH 52/65] add TickDrivenAdapter Co-Authored-By: Claude Opus 5 (1M context) --- .../cascade/adapter/tick_driven.py | 149 ++++++++++++++++++ .../cascade/test_tick_driven_adapter.py | 100 ++++++++++++ 2 files changed, 249 insertions(+) create mode 100644 src/eva/user_simulator/cascade/adapter/tick_driven.py create mode 100644 tests/unit/user_simulator/cascade/test_tick_driven_adapter.py diff --git a/src/eva/user_simulator/cascade/adapter/tick_driven.py b/src/eva/user_simulator/cascade/adapter/tick_driven.py new file mode 100644 index 00000000..534953bd --- /dev/null +++ b/src/eva/user_simulator/cascade/adapter/tick_driven.py @@ -0,0 +1,149 @@ +"""Tick-driven adapter: the caller owns the simulation clock. + +Talks to the same assistant server over the same Twilio WebSocket as the +real-time adapter, with two differences. Outbound audio is not paced, because +the server is not pacing its own output either (``paced_output=False``); and +nothing at all is emitted on a tick where the caller is silent. + +Because the provider's VAD advances on audio received rather than wall time, +not sending audio freezes the assistant. That is what makes caller compute time +invisible here — and why nothing may be emitted on a stalled tick. + +Inbound audio is released strictly one tick at a time however much arrives at +once, which the real-time adapter already does; the difference is that here the +release cadence *is* the simulation clock rather than an approximation of it. + +Implemented as a subclass of :class:`RealtimeWSAdapter` rather than a peer: the +handshake, the receive loop, the speech-event frames and — critically — the +mulaw/PCM resamplers all carry per-instance filter state that must not be +duplicated. Only the timing is overridden. +""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import time + +from eva.user_simulator.cascade.adapter.realtime_ws import FRAMES_PER_TICK, RealtimeWSAdapter +from eva.user_simulator.cascade.constants import BYTES_PER_TICK +from eva.user_simulator.cascade.tick_result import TickResult, played_audio_ms, split_tick_audio +from eva.utils.logging import get_logger + +logger = get_logger(__name__) + +MAX_INACTIVE_SECONDS = 40.0 +"""Fail loudly if the provider goes quiet this long (tau: DEFAULT_AUDIO_NATIVE_MAX_INACTIVE_SECONDS).""" + + +class TickDrivenAdapter(RealtimeWSAdapter): + """Exchanges one tick of audio with an unpaced assistant server.""" + + def __init__( + self, + *, + websocket, + conversation_id: str, + bytes_per_tick: int = BYTES_PER_TICK, + perturbator=None, + ) -> None: + super().__init__( + websocket=websocket, + conversation_id=conversation_id, + bytes_per_tick=bytes_per_tick, + perturbator=perturbator, + ) + self._ticks_released = 0 + self._last_inbound_monotonic = time.monotonic() + + @property + def played_ms(self) -> int: + """Assistant audio released into the conversation so far, in simulated ms.""" + return played_audio_ms(ticks_released=self._ticks_released) + + async def run_tick(self, tick_number: int, outgoing_audio: bytes | None, *, barge_in: bool = False) -> TickResult: + """Send this tick's caller audio unpaced and release exactly one tick of assistant audio. + + When ``barge_in`` is set, first tell the assistant to discard the audio it + generated past the position the caller has actually heard. + """ + if self._error is not None: + raise RuntimeError("TickDrivenAdapter receive loop failed") from self._error + + interruption_start: int | None = None + if barge_in: + interruption_start = self.played_ms + await self._ws.send( + json.dumps( + { + "event": "truncate", + "conversation_id": self._conversation_id, + "audio_end_ms": interruption_start, + } + ) + ) + # Everything already buffered is audio the caller never heard. + self._inbound.clear() + + is_speaking = bool(outgoing_audio) + if is_speaking and not self._caller_speaking: + await self._send_speech_event("user_speech_start") + elif not is_speaking and self._caller_speaking: + await self._send_speech_event("user_speech_stop") + self._caller_speaking = is_speaking + + if outgoing_audio: + # Perturbation only ever rides on real caller audio here: mixing ambient + # noise into a stalled tick would emit frames and unfreeze the assistant. + await self._send_unpaced(self._apply_perturbation(outgoing_audio) or outgoing_audio) + + await asyncio.sleep(0) + raw = bytes(self._inbound[: self._bytes_per_tick]) + del self._inbound[: len(raw)] + chunk, _ = split_tick_audio(raw, self._bytes_per_tick) + if raw: + self._ticks_released += 1 + + self._check_provider_alive(bool(raw)) + + if self._error is not None: + raise RuntimeError("TickDrivenAdapter receive loop failed") from self._error + + return TickResult( + tick_number=tick_number, + assistant_audio=chunk, + assistant_audio_raw_bytes=len(raw), + wall_clock_ms=int(time.time() * 1000), + interruption_audio_start_ms=interruption_start, + ) + + async def _send_unpaced(self, pcm: bytes) -> None: + """Split one tick of PCM16 into wire frames and send them with no sleeps.""" + mulaw = self._pcm16k_to_mulaw8k(pcm) + if not mulaw: + return + frame_size = len(mulaw) // FRAMES_PER_TICK or len(mulaw) + for index in range(0, len(mulaw), frame_size): + payload = base64.b64encode(mulaw[index : index + frame_size]).decode() + await self._ws.send( + json.dumps( + { + "event": "media", + "conversation_id": self._conversation_id, + "media": {"payload": payload}, + } + ) + ) + + def _check_provider_alive(self, received: bool) -> None: + """Raise if the provider has produced nothing for too long. + + Wall clock is the right measure here and only here: this is a liveness + check on a real network peer, not a measurement of conversation time. + """ + now = time.monotonic() + if received: + self._last_inbound_monotonic = now + elif now - self._last_inbound_monotonic > MAX_INACTIVE_SECONDS: + raise RuntimeError(f"No assistant audio for {MAX_INACTIVE_SECONDS}s; provider appears stalled") diff --git a/tests/unit/user_simulator/cascade/test_tick_driven_adapter.py b/tests/unit/user_simulator/cascade/test_tick_driven_adapter.py new file mode 100644 index 00000000..2f45da32 --- /dev/null +++ b/tests/unit/user_simulator/cascade/test_tick_driven_adapter.py @@ -0,0 +1,100 @@ +import asyncio +import json +import time + +from eva.user_simulator.cascade.adapter.tick_driven import TickDrivenAdapter +from tests.unit.user_simulator.cascade.test_realtime_ws_adapter import ( + BYTES_PER_TICK, + FakeWebSocket, + _media_frame, + _settle, +) + + +def _media(ws: FakeWebSocket) -> list[dict]: + return [json.loads(m) for m in ws.sent if json.loads(m).get("event") == "media"] + + +async def test_outbound_audio_is_sent_without_pacing_sleeps(): + ws = FakeWebSocket() + adapter = TickDrivenAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await adapter.start() + + started = time.monotonic() + await adapter.run_tick(0, b"\x00" * BYTES_PER_TICK) + + # The real-time adapter would spend ~200ms pacing this. + assert time.monotonic() - started < 0.05 + assert len(_media(ws)) == 10 + await adapter.stop() + + +async def test_burst_of_provider_audio_releases_one_tick_at_a_time(): + ws = FakeWebSocket() + adapter = TickDrivenAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await adapter.start() + # 1 second of assistant audio arrives at once: 8000 mulaw bytes -> 32000 PCM bytes. + await ws.inbound.put(_media_frame(b"\xff" * 8000)) + await _settle() + + results = [await adapter.run_tick(tick, None) for tick in range(5)] + + # The resampler's filter state leaves the last tick a couple of samples short; + # what matters is that no tick releases more than one tick's worth. + raw = [r.assistant_audio_raw_bytes for r in results] + assert raw[:4] == [BYTES_PER_TICK] * 4 + assert 0 < raw[4] <= BYTES_PER_TICK + # Short ticks are silence-padded, so every released chunk is exactly tick-sized. + assert all(len(r.assistant_audio) == BYTES_PER_TICK for r in results) + await adapter.stop() + + +async def test_nothing_is_emitted_on_a_silent_tick(): + ws = FakeWebSocket() + adapter = TickDrivenAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await adapter.start() + + await adapter.run_tick(0, None) + + # Audio sent during a stall would advance the provider's VAD and break the freeze. + assert _media(ws) == [] + await adapter.stop() + + +async def test_played_position_tracks_released_ticks(): + ws = FakeWebSocket() + adapter = TickDrivenAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await adapter.start() + await ws.inbound.put(_media_frame(b"\xff" * 8000)) + await _settle() + + for tick in range(3): + await adapter.run_tick(tick, None) + + assert adapter.played_ms == 600 + await adapter.stop() + + +async def test_a_tick_with_no_assistant_audio_does_not_advance_the_played_position(): + ws = FakeWebSocket() + adapter = TickDrivenAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await adapter.start() + + for tick in range(3): + await adapter.run_tick(tick, None) + + assert adapter.played_ms == 0 + await adapter.stop() + + +async def test_run_tick_does_not_wait_out_the_tick_duration(): + ws = FakeWebSocket() + adapter = TickDrivenAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await adapter.start() + + started = time.monotonic() + await asyncio.gather(*(adapter.run_tick(tick, None) for tick in range(3))) + + # The real-time adapter enforces a 200ms floor per tick; this one must not. + assert time.monotonic() - started < 0.05 + await adapter.stop() From 4f5f16ed7fa69e40944247fe9a3ecf014c81d68a Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Tue, 18 Aug 2026 23:11:12 -0700 Subject: [PATCH 53/65] select the cascade adapter by assistant framework and plumb the pacing mode Co-Authored-By: Claude Opus 5 (1M context) --- src/eva/orchestrator/worker.py | 19 +++++++++++- src/eva/user_simulator/cascade/simulator.py | 22 +++++++++++++- src/eva/user_simulator/factory.py | 10 +++++-- tests/unit/orchestrator/test_worker.py | 23 +++++++++++++++ .../user_simulator/cascade/test_simulator.py | 29 +++++++++++++++++++ 5 files changed, 99 insertions(+), 4 deletions(-) diff --git a/src/eva/orchestrator/worker.py b/src/eva/orchestrator/worker.py index ef5519c4..12a710bf 100644 --- a/src/eva/orchestrator/worker.py +++ b/src/eva/orchestrator/worker.py @@ -9,7 +9,7 @@ from eva.assistant.base_server import AbstractAssistantServer from eva.models.agents import AgentConfig -from eva.models.config import RunConfig +from eva.models.config import RunConfig, UserSimulatorConfig from eva.models.record import EvaluationRecord from eva.models.results import ConversationResult, ErrorDetails, LatencyStats from eva.user_simulator.factory import create_user_simulator @@ -23,6 +23,21 @@ USER_SIMULATOR_SHUTDOWN_GRACE_SECONDS = 20 +def should_pace_assistant_output(simulator_config: UserSimulatorConfig, *, framework: str) -> bool: + """Whether the assistant should emit audio at real-time cadence. + + Only a tick-driven cascade caller can consume unpaced output: it buffers what + arrives and releases one tick at a time. Every other caller infers turn + boundaries from cadence and would break. + """ + from eva.models.config import CascadeSimulatorConfig + from eva.user_simulator.cascade.simulator import TICK_DRIVEN_FRAMEWORKS + + if not isinstance(simulator_config, CascadeSimulatorConfig): + return True + return framework not in TICK_DRIVEN_FRAMEWORKS + + def _get_server_class(framework: str) -> type[AbstractAssistantServer]: """Return the server class for the given framework name. @@ -323,6 +338,7 @@ async def _start_assistant(self) -> None: port=self.port, conversation_id=self.record.id, language=self.config.language, + paced_output=should_pace_assistant_output(self.config.user_simulator, framework=self.config.framework), **server_kwargs, ) @@ -377,6 +393,7 @@ async def _start_user_simulator(self) -> None: timeout=self._conversation_guard_timeout_seconds(), perturbation_config=self.config.perturbation, language=language, + framework=self.config.framework, ) # Let the simulator tell the assistant the moment the call ends, rather than leaving it diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py index 5e3e8fec..d4909582 100644 --- a/src/eva/user_simulator/cascade/simulator.py +++ b/src/eva/user_simulator/cascade/simulator.py @@ -13,7 +13,9 @@ from eva.assistant.services.llm import LiteLLMClient from eva.models.config import CascadeSimulatorConfig, PerturbationConfig from eva.user_simulator.base import AbstractUserSimulator +from eva.user_simulator.cascade.adapter.base import Adapter from eva.user_simulator.cascade.adapter.realtime_ws import RealtimeWSAdapter +from eva.user_simulator.cascade.adapter.tick_driven import TickDrivenAdapter from eva.user_simulator.cascade.constants import ( BACKCHANNEL_PHRASES, BARGE_IN_OPENERS, @@ -203,6 +205,21 @@ async def decide(self, prompt: str) -> str: return getattr(message, "content", None) or "" +TICK_DRIVEN_FRAMEWORKS = frozenset({"openai_realtime"}) +"""Frameworks whose clock the caller can own. Others keep real-time streaming.""" + + +def adapter_class_for_framework(framework: str) -> type[Adapter]: + """Pick the adapter for a framework, defaulting to real-time streaming. + + Defaulting to real-time is deliberate: it works everywhere, whereas + tick-driving requires the assistant to have no wall-clock timers of its own. + """ + if framework in TICK_DRIVEN_FRAMEWORKS: + return TickDrivenAdapter + return RealtimeWSAdapter + + class CascadeUserSimulator(AbstractUserSimulator): """Simulated caller built from independently chosen STT, LLM, and TTS models.""" @@ -224,6 +241,7 @@ def __init__( language: str = "en", *, simulator_config: CascadeSimulatorConfig, + framework: str = "pipecat", ) -> None: super().__init__( current_date_time=current_date_time, @@ -238,6 +256,7 @@ def __init__( provider="cascade", ) self._config = simulator_config + self._framework = framework self._stt = LiveKitStreamingSTT(simulator_config.stt, simulator_config.stt_params, language=language) self._tts = CartesiaTTS(simulator_config.tts_params, language=language) self._llm = LiteLLMClient(model=simulator_config.llm) @@ -269,7 +288,8 @@ async def run_conversation(self) -> str: async def _run(self) -> None: """Drive the scheduler until end_call, timeout, or disconnect.""" websocket = await websockets.connect(self.server_url) - adapter = RealtimeWSAdapter( + adapter_cls = adapter_class_for_framework(self._framework) + adapter = adapter_cls( websocket=websocket, conversation_id=self._record_id or "cascade", perturbator=self._perturbator, diff --git a/src/eva/user_simulator/factory.py b/src/eva/user_simulator/factory.py index 0ad5da17..ca6c366f 100644 --- a/src/eva/user_simulator/factory.py +++ b/src/eva/user_simulator/factory.py @@ -17,7 +17,13 @@ def create_user_simulator( simulator_config: UserSimulatorConfig, **kwargs: Any, ) -> AbstractUserSimulator: - """Create the configured simulated caller without importing unused providers.""" + """Create the configured simulated caller without importing unused providers. + + ``framework`` names the assistant framework and is consumed here rather than + forwarded: only the cascade caller varies its transport by framework, and the + other two providers would raise on the unexpected keyword. + """ + framework = kwargs.pop("framework", "pipecat") if isinstance(simulator_config, ElevenLabsSimulatorConfig): from eva.user_simulator.elevenlabs import ElevenLabsUserSimulator @@ -29,5 +35,5 @@ def create_user_simulator( if isinstance(simulator_config, CascadeSimulatorConfig): from eva.user_simulator.cascade.simulator import CascadeUserSimulator - return CascadeUserSimulator(simulator_config=simulator_config, **kwargs) + return CascadeUserSimulator(simulator_config=simulator_config, framework=framework, **kwargs) raise ValueError(f"Unknown user simulator provider: {simulator_config.provider!r}") diff --git a/tests/unit/orchestrator/test_worker.py b/tests/unit/orchestrator/test_worker.py index e5a0366d..f7c2bee3 100644 --- a/tests/unit/orchestrator/test_worker.py +++ b/tests/unit/orchestrator/test_worker.py @@ -252,6 +252,7 @@ async def test_worker_uses_configured_factory_and_timeout(self, tmp_path, monkey timeout=worker._conversation_guard_timeout_seconds(), perturbation_config=None, language="en", + framework=worker.config.framework, ) def test_worker_timeout_reserves_provider_cleanup_window(self, tmp_path): @@ -342,3 +343,25 @@ async def test_stats_captured_on_time_limit_exceeded(self, tmp_path): assert result.num_turns == 3 assert result.num_tool_calls == 1 assert result.conversation_ended_reason == "time_limit_exceeded" + + +def test_pacing_disabled_for_tick_driven_cascade_runs(): + from eva.models.config import CascadeSimulatorConfig + from eva.orchestrator.worker import should_pace_assistant_output + + assert should_pace_assistant_output(CascadeSimulatorConfig(), framework="openai_realtime") is False + + +def test_pacing_kept_for_cascade_on_a_real_time_framework(): + from eva.models.config import CascadeSimulatorConfig + from eva.orchestrator.worker import should_pace_assistant_output + + assert should_pace_assistant_output(CascadeSimulatorConfig(), framework="pipecat") is True + + +def test_pacing_kept_for_the_elevenlabs_simulator(): + from eva.models.config import ElevenLabsSimulatorConfig + from eva.orchestrator.worker import should_pace_assistant_output + + # Its silence heuristics read cadence; unpacing it would break turn detection. + assert should_pace_assistant_output(ElevenLabsSimulatorConfig(), framework="openai_realtime") is True diff --git a/tests/unit/user_simulator/cascade/test_simulator.py b/tests/unit/user_simulator/cascade/test_simulator.py index 74efc7c0..b7038c68 100644 --- a/tests/unit/user_simulator/cascade/test_simulator.py +++ b/tests/unit/user_simulator/cascade/test_simulator.py @@ -727,3 +727,32 @@ async def test_a_kept_interruption_speaks_the_opener_then_the_content(): assert scheduler.queued[0] == b"CACHED" assert b"Active Directory." in b"".join(scheduler.queued[1:]) + + +def test_openai_realtime_uses_the_tick_driven_adapter(): + from eva.user_simulator.cascade.adapter.tick_driven import TickDrivenAdapter + from eva.user_simulator.cascade.simulator import adapter_class_for_framework + + assert adapter_class_for_framework("openai_realtime") is TickDrivenAdapter + + +def test_pipecat_stays_on_the_real_time_adapter(): + # Pipecat owns its own clock; freezing it is not possible. + from eva.user_simulator.cascade.adapter.realtime_ws import RealtimeWSAdapter + from eva.user_simulator.cascade.simulator import adapter_class_for_framework + + assert adapter_class_for_framework("pipecat") is RealtimeWSAdapter + + +def test_elevenlabs_stays_on_the_real_time_adapter(): + from eva.user_simulator.cascade.adapter.realtime_ws import RealtimeWSAdapter + from eva.user_simulator.cascade.simulator import adapter_class_for_framework + + assert adapter_class_for_framework("elevenlabs") is RealtimeWSAdapter + + +def test_unported_frameworks_default_to_the_real_time_adapter(): + from eva.user_simulator.cascade.adapter.realtime_ws import RealtimeWSAdapter + from eva.user_simulator.cascade.simulator import adapter_class_for_framework + + assert adapter_class_for_framework("gemini_live") is RealtimeWSAdapter From c1cf2e8b7f85ab3b315e8611ce48f382b34c0283 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Tue, 18 Aug 2026 23:17:05 -0700 Subject: [PATCH 54/65] truncate provider audio at the played position on barge-in Co-Authored-By: Claude Opus 5 (1M context) --- src/eva/assistant/openai_realtime_server.py | 28 ++++++++++ .../user_simulator/cascade/adapter/base.py | 8 ++- .../cascade/adapter/realtime_ws.py | 8 ++- src/eva/user_simulator/cascade/scheduler.py | 15 +++++- src/eva/user_simulator/cascade/simulator.py | 5 ++ .../assistant/test_openai_realtime_pacing.py | 54 +++++++++++++++++++ .../cascade/test_adapter_base.py | 4 +- .../user_simulator/cascade/test_scheduler.py | 31 ++++++++++- .../user_simulator/cascade/test_simulator.py | 6 +++ .../cascade/test_tick_driven_adapter.py | 33 ++++++++++++ 10 files changed, 184 insertions(+), 8 deletions(-) diff --git a/src/eva/assistant/openai_realtime_server.py b/src/eva/assistant/openai_realtime_server.py index b01a9965..c299a575 100644 --- a/src/eva/assistant/openai_realtime_server.py +++ b/src/eva/assistant/openai_realtime_server.py @@ -69,6 +69,8 @@ class _AssistantResponseState: first_audio_wall_ms: str | None = None responding: bool = False has_function_calls: bool = False + active_item_id: str | None = None + """Item currently producing audio; the truncation target on a caller barge-in.""" class OpenAIRealtimeAssistantServer(AbstractAssistantServer): @@ -387,6 +389,10 @@ async def _forward_user_audio(self, websocket: WebSocket, conn: Any) -> None: logger.debug("Twilio stream stopped") break + if event_type == "truncate": + await self._truncate_response(conn, int(data.get("audio_end_ms", 0))) + continue + if event_type == "user_speech_start": # Timestamp from audio_interface when user audio actually started self._audio_interface_speech_start_ts = data.get("timestamp_ms") @@ -512,6 +518,23 @@ async def _handle_openai_event( case _: logger.debug(f"Unhandled {self._service_name} event: {event_type}") + async def _truncate_response(self, conn: Any, audio_end_ms: int) -> None: + """Discard generated audio past the position the caller actually heard. + + Only a tick-driven caller sends this: with pacing off the provider runs + ahead of the caller's playout, so without truncation it would believe the + caller heard seconds of audio that were never released. + """ + item_id = self._assistant_state.active_item_id + if not item_id: + return + try: + await conn.conversation.item.truncate(item_id=item_id, content_index=0, audio_end_ms=audio_end_ms) + except Exception as exc: + logger.warning(f"Truncating item {item_id} at {audio_end_ms}ms failed: {exc}") + return + logger.info(f"Truncated item {item_id} at {audio_end_ms}ms on caller barge-in") + # ── Event handlers ──────────────────────────────────────────────── async def _on_speech_started(self, event: Any) -> None: @@ -598,6 +621,11 @@ async def _on_audio_delta(self, event: Any, audio_output_queue: asyncio.Queue[by return pcm16_bytes = base64.b64decode(delta_b64) + # Read off the delta rather than response.output_item.added: it names the same + # item and is already dispatched here, so no extra event handler is needed. + item_id = getattr(event, "item_id", None) + if item_id: + self._assistant_state.active_item_id = item_id if self._assistant_state.first_audio_wall_ms is None: self._assistant_state.first_audio_wall_ms = _wall_ms() diff --git a/src/eva/user_simulator/cascade/adapter/base.py b/src/eva/user_simulator/cascade/adapter/base.py index 3e90ae09..c81914c6 100644 --- a/src/eva/user_simulator/cascade/adapter/base.py +++ b/src/eva/user_simulator/cascade/adapter/base.py @@ -20,8 +20,12 @@ async def start(self) -> None: """Establish the connection. Must return once ready to exchange audio.""" @abstractmethod - async def run_tick(self, tick_number: int, outgoing_audio: bytes | None) -> TickResult: - """Send this tick's caller audio (None means silence) and collect what arrived.""" + async def run_tick(self, tick_number: int, outgoing_audio: bytes | None, *, barge_in: bool = False) -> TickResult: + """Send this tick's caller audio (None means silence) and collect what arrived. + + ``barge_in`` signals that this tick begins an interruption. Adapters that + cannot truncate provider-side audio ignore it. + """ @abstractmethod async def stop(self) -> None: diff --git a/src/eva/user_simulator/cascade/adapter/realtime_ws.py b/src/eva/user_simulator/cascade/adapter/realtime_ws.py index 2e1ca512..3ae154bf 100644 --- a/src/eva/user_simulator/cascade/adapter/realtime_ws.py +++ b/src/eva/user_simulator/cascade/adapter/realtime_ws.py @@ -69,8 +69,12 @@ async def start(self) -> None: await self._ws.send(json.dumps({"event": event, "conversation_id": self._conversation_id})) self._receive_task = asyncio.create_task(self._receive_loop()) - async def run_tick(self, tick_number: int, outgoing_audio: bytes | None) -> TickResult: - """Send one tick of caller audio at wire cadence and collect one tick of assistant audio.""" + async def run_tick(self, tick_number: int, outgoing_audio: bytes | None, *, barge_in: bool = False) -> TickResult: + """Send one tick of caller audio at wire cadence and collect one tick of assistant audio. + + ``barge_in`` is accepted and ignored: this assistant paces its own output, so + it has generated nothing past what the caller already heard to discard. + """ tick_start = asyncio.get_event_loop().time() if self._error is not None: raise RuntimeError("RealtimeWSAdapter receive loop failed") from self._error diff --git a/src/eva/user_simulator/cascade/scheduler.py b/src/eva/user_simulator/cascade/scheduler.py index b57cc51f..8d96d57f 100644 --- a/src/eva/user_simulator/cascade/scheduler.py +++ b/src/eva/user_simulator/cascade/scheduler.py @@ -36,6 +36,7 @@ def __init__(self, adapter: Adapter, *, bytes_per_tick: int = BYTES_PER_TICK) -> self._awaiting_reply = False self._caller_spoke_this_tick = False self._backchannel_bytes = 0 + self._barge_in_armed = False @property def tick(self) -> int: @@ -61,6 +62,15 @@ def enqueue_backchannel(self, audio: bytes) -> None: self._backchannel_bytes += len(audio) self._playout.extend(audio) + def arm_barge_in(self) -> None: + """Mark the next tick that puts caller audio on the wire as an interruption. + + Armed rather than passed directly because an interruption is enqueued as + audio and only reaches the wire on a later tick; the truncation must carry + the played position as of *that* tick, not as of the decision. + """ + self._barge_in_armed = True + @property def caller_is_speaking(self) -> bool: """Whether caller audio is still queued for playout.""" @@ -118,8 +128,11 @@ async def run_tick(self) -> TickResult: raised exception leaves the queue and tick count exactly as they were. """ outgoing, consumed = self._peek_chunk() - result = await self._adapter.run_tick(self._tick, outgoing) + barge_in = self._barge_in_armed and outgoing is not None + result = await self._adapter.run_tick(self._tick, outgoing, barge_in=barge_in) del self._playout[:consumed] + if barge_in: + self._barge_in_armed = False # Backchannel bytes sit at the head of the queue, so this tick is a continuer # only while they remain. Anything past them is real speech and takes the turn. diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py index d4909582..29189222 100644 --- a/src/eva/user_simulator/cascade/simulator.py +++ b/src/eva/user_simulator/cascade/simulator.py @@ -427,6 +427,10 @@ async def _play_interruption(self, scheduler: TickScheduler) -> bool: if await candidate_is_relevant( self._decision_client, candidate=candidate, heard=self._stt.buffer.current_text() ): + # Tell the adapter the next tick that reaches the wire cuts the + # assistant off, so a tick-driven transport can truncate the audio + # the caller never heard. Ignored on the real-time path. + scheduler.arm_barge_in() scheduler.enqueue_utterance(opener_audio) self._record_audio("user_clean", opener_audio) scheduler.enqueue_utterance(audio) @@ -482,6 +486,7 @@ async def _play_interruption(self, scheduler: TickScheduler) -> bool: return False if utterance: + scheduler.arm_barge_in() scheduler.enqueue_utterance(opener_audio) self._record_audio("user_clean", opener_audio) self._history.append({"role": "user", "content": utterance}) diff --git a/tests/unit/assistant/test_openai_realtime_pacing.py b/tests/unit/assistant/test_openai_realtime_pacing.py index ccc2f15c..1c29b046 100644 --- a/tests/unit/assistant/test_openai_realtime_pacing.py +++ b/tests/unit/assistant/test_openai_realtime_pacing.py @@ -87,3 +87,57 @@ def test_user_recently_active_counts_audio_deltas_not_wall_time(tmp_path): for _ in range(server.USER_ACTIVE_GUARD_DELTAS): server.note_assistant_delta() assert server.user_recently_active() is False + + +class FakeConn: + """Records conversation.item.truncate calls.""" + + def __init__(self) -> None: + self.truncated: list[dict] = [] + outer = self + + class _Item: + async def truncate(self, *, item_id, content_index, audio_end_ms): + outer.truncated.append( + {"item_id": item_id, "content_index": content_index, "audio_end_ms": audio_end_ms} + ) + + class _Conversation: + item = _Item() + + self.conversation = _Conversation() + + +@pytest.mark.asyncio +async def test_truncate_targets_the_item_currently_producing_audio(tmp_path): + server = _server(paced=False, tmp_path=tmp_path) + server._assistant_state.active_item_id = "item_42" + conn = FakeConn() + + await server._truncate_response(conn, 600) + + assert conn.truncated == [{"item_id": "item_42", "content_index": 0, "audio_end_ms": 600}] + + +@pytest.mark.asyncio +async def test_truncate_is_a_no_op_when_no_item_is_active(tmp_path): + server = _server(paced=False, tmp_path=tmp_path) + conn = FakeConn() + + await server._truncate_response(conn, 600) + + assert conn.truncated == [] + + +@pytest.mark.asyncio +async def test_audio_delta_records_the_active_item(tmp_path): + import base64 + from types import SimpleNamespace + + server = _server(paced=False, tmp_path=tmp_path) + queue: asyncio.Queue[bytes] = asyncio.Queue() + event = SimpleNamespace(delta=base64.b64encode(b"\x00" * 480).decode(), item_id="item_7") + + await server._on_audio_delta(event, queue) + + assert server._assistant_state.active_item_id == "item_7" diff --git a/tests/unit/user_simulator/cascade/test_adapter_base.py b/tests/unit/user_simulator/cascade/test_adapter_base.py index 1f6cfd83..bac37908 100644 --- a/tests/unit/user_simulator/cascade/test_adapter_base.py +++ b/tests/unit/user_simulator/cascade/test_adapter_base.py @@ -15,7 +15,9 @@ class StubAdapter(Adapter): async def start(self) -> None: pass - async def run_tick(self, tick_number: int, outgoing_audio: bytes | None) -> TickResult: + async def run_tick( + self, tick_number: int, outgoing_audio: bytes | None, *, barge_in: bool = False + ) -> TickResult: return TickResult( tick_number=tick_number, assistant_audio=b"\x00" * 4, diff --git a/tests/unit/user_simulator/cascade/test_scheduler.py b/tests/unit/user_simulator/cascade/test_scheduler.py index 1d236526..b59d509b 100644 --- a/tests/unit/user_simulator/cascade/test_scheduler.py +++ b/tests/unit/user_simulator/cascade/test_scheduler.py @@ -12,13 +12,16 @@ def __init__(self, speech_ticks: list[bool]) -> None: self.speech_ticks = speech_ticks self.sent: list[bytes | None] = [] self.received_ticks: list[int] = [] + self.barge_in_ticks: list[int] = [] async def start(self) -> None: pass - async def run_tick(self, tick_number: int, outgoing_audio: bytes | None) -> TickResult: + async def run_tick(self, tick_number: int, outgoing_audio: bytes | None, *, barge_in: bool = False) -> TickResult: self.sent.append(outgoing_audio) self.received_ticks.append(tick_number) + if barge_in: + self.barge_in_ticks.append(tick_number) speaking = self.speech_ticks[tick_number] if tick_number < len(self.speech_ticks) else False return TickResult( tick_number=tick_number, @@ -195,7 +198,7 @@ def __init__(self, fail_on_tick: int) -> None: async def start(self) -> None: pass - async def run_tick(self, tick_number: int, outgoing_audio: bytes | None) -> TickResult: + async def run_tick(self, tick_number: int, outgoing_audio: bytes | None, *, barge_in: bool = False) -> TickResult: if tick_number == self.fail_on_tick: raise RuntimeError("adapter failure") return TickResult( @@ -298,3 +301,27 @@ async def test_an_utterance_queued_after_a_backchannel_still_consumes_the_turn() await scheduler.run_tick() assert scheduler.may_take_turn() is False + + +async def test_armed_barge_in_fires_on_the_tick_audio_reaches_the_wire(): + scheduler = _scheduler([True, True, True]) + adapter = scheduler._adapter + + await scheduler.run_tick() # silent tick: nothing on the wire yet + scheduler.arm_barge_in() + scheduler.enqueue_utterance(b"\x01" * BYTES_PER_TICK * 2) + await scheduler.run_tick() + await scheduler.run_tick() + + # Exactly the first tick that carried caller audio, and only that one. + assert adapter.barge_in_ticks == [1] + + +async def test_arming_a_barge_in_with_nothing_queued_does_not_fire_on_silence(): + scheduler = _scheduler([True, True]) + adapter = scheduler._adapter + + scheduler.arm_barge_in() + await scheduler.run_tick() + + assert adapter.barge_in_ticks == [] diff --git a/tests/unit/user_simulator/cascade/test_simulator.py b/tests/unit/user_simulator/cascade/test_simulator.py index b7038c68..5e3ee303 100644 --- a/tests/unit/user_simulator/cascade/test_simulator.py +++ b/tests/unit/user_simulator/cascade/test_simulator.py @@ -598,6 +598,10 @@ class _InterruptScheduler: def __init__(self) -> None: self.queued: list[bytes] = [] + self.barge_in_armed = False + + def arm_barge_in(self) -> None: + self.barge_in_armed = True def enqueue_utterance(self, audio: bytes) -> None: self.queued.append(audio) @@ -727,6 +731,8 @@ async def test_a_kept_interruption_speaks_the_opener_then_the_content(): assert scheduler.queued[0] == b"CACHED" assert b"Active Directory." in b"".join(scheduler.queued[1:]) + # A tick-driven transport needs to know this audio cuts the assistant off. + assert scheduler.barge_in_armed is True def test_openai_realtime_uses_the_tick_driven_adapter(): diff --git a/tests/unit/user_simulator/cascade/test_tick_driven_adapter.py b/tests/unit/user_simulator/cascade/test_tick_driven_adapter.py index 2f45da32..3241064d 100644 --- a/tests/unit/user_simulator/cascade/test_tick_driven_adapter.py +++ b/tests/unit/user_simulator/cascade/test_tick_driven_adapter.py @@ -98,3 +98,36 @@ async def test_run_tick_does_not_wait_out_the_tick_duration(): # The real-time adapter enforces a 200ms floor per tick; this one must not. assert time.monotonic() - started < 0.05 await adapter.stop() + + +async def test_barge_in_reports_the_played_position_not_the_received_position(): + ws = FakeWebSocket() + adapter = TickDrivenAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await adapter.start() + # 1s of audio arrives at once but only 3 ticks (600ms) get released. + await ws.inbound.put(_media_frame(b"\xff" * 8000)) + await _settle() + for tick in range(3): + await adapter.run_tick(tick, None) + + result = await adapter.run_tick(3, b"\x00" * BYTES_PER_TICK, barge_in=True) + + assert result.interruption_audio_start_ms == 600 + truncate = [json.loads(m) for m in ws.sent if json.loads(m).get("event") == "truncate"] + assert truncate and truncate[0]["audio_end_ms"] == 600 + await adapter.stop() + + +async def test_barge_in_discards_audio_the_caller_never_heard(): + ws = FakeWebSocket() + adapter = TickDrivenAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await adapter.start() + await ws.inbound.put(_media_frame(b"\xff" * 8000)) + await _settle() + + result = await adapter.run_tick(0, b"\x00" * BYTES_PER_TICK, barge_in=True) + + # The buffered second of assistant audio is audio the caller cut off. + assert result.assistant_audio_raw_bytes == 0 + assert adapter.played_ms == 0 + await adapter.stop() From 526788b71de0d7feb017f3f57801bb1bd644da2b Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Tue, 18 Aug 2026 23:18:44 -0700 Subject: [PATCH 55/65] record the tick-driven adapter working log Co-Authored-By: Claude Opus 5 (1M context) --- docs/changelog_cascade_tick_driven.md | 115 ++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 docs/changelog_cascade_tick_driven.md diff --git a/docs/changelog_cascade_tick_driven.md b/docs/changelog_cascade_tick_driven.md new file mode 100644 index 00000000..8f5fcb37 --- /dev/null +++ b/docs/changelog_cascade_tick_driven.md @@ -0,0 +1,115 @@ +# Working log: cascade tick-driven adapter (Plan 3) + +Plan: `docs/superpowers/plans/2026-08-06-cascade-tick-driven-adapter.md` + +Implemented Tasks 1-8 on `worktree-user-sim-phase-3`, branched off +`feat/cascade-user-simulator` (Plan 1 + Plan 2 as merged there). Task 9 is live +end-to-end verification and has **not** been run — see the last section. + +## Deviations from the plan, and why + +### The worktree was branched off `main`, not off the plan's branch + +`.claude/worktrees/user-sim-phase-3` was created at `e0041e3d` (main), so none of +Plan 1's or Plan 2's code was present — `src/eva/user_simulator/cascade/` did not +exist. Reset the worktree branch to `feat/cascade-user-simulator` (`3cb17e37`) +before starting. Nothing was lost; the branch had no commits of its own. + +### Tests in this worktree import the *main* checkout's `src/` + +The editable install resolves `eva` to `/Users/tara.bogavelli/third_eva/EVA-Bench3/src` +under pytest, so a bare `pytest` in the worktree silently tests the other agent's +Plan 2 working tree instead of these changes. Every test run here used +`PYTHONPATH=/src`. Anyone re-verifying this branch must do the same or +the results are meaningless. + +### `TickDrivenAdapter` subclasses `RealtimeWSAdapter` instead of being a peer + +The plan sketched a standalone class calling `RealtimeWSAdapter._pcm16k_to_mulaw8k` +as though it were static. It is not: both resamplers carry per-instance +`audioop.ratecv` filter state, and calling them unbound would either crash or +produce discontinuous audio. Subclassing reuses the handshake, receive loop, +error propagation, speech-event frames and resamplers, and overrides only the +timing — which is the one thing this plan is actually about. + +Consequences: `perturbator` is accepted (and inherited) rather than rejected, so +the framework-selection site in `simulator.py` passes it unconditionally instead +of using the plan's `**({...} if adapter_cls is RealtimeWSAdapter else {})` +conditional. Perturbation is applied only to ticks that carry real caller audio — +mixing ambient noise into a stalled tick would emit frames and unfreeze the +assistant, defeating the whole mechanism. + +### Nothing armed a barge-in, so Task 8 would have been dead code + +Task 8 specified the adapter half and the server half but no caller. The +interruption is enqueued as *audio* by `_play_interruption` and only reaches the +wire on a later tick, so `run_tick(barge_in=True)` had no natural call site. +Added `TickScheduler.arm_barge_in()`: it marks the next tick that actually puts +caller audio on the wire, so the truncation carries the played position as of +*that* tick rather than as of the decision. `_play_interruption` arms it on both +of its enqueue paths (speculative and freshly generated). Without this the +truncate frame would never have been sent. + +### Truncation target read off the audio delta, not `response.output_item.added` + +The plan said to track `active_item_id` wherever `response.output_item.added` is +handled — that case does not exist in `openai_realtime_server.py`. The audio +delta event carries the same `item_id` and is already dispatched, so the item is +recorded there and no new event handler was added. + +Verified the SDK entry point rather than trusting the plan's spelling: +`openai.resources.realtime.realtime.AsyncRealtimeConversationItemResource.truncate(*, +audio_end_ms, content_index, item_id, event_id)`. The call matches. It is wrapped +in a `try/except` that logs and returns — a failed truncation should degrade +fidelity, not kill the conversation. + +### `USER_ACTIVE_GUARD_S` removal + +Removed the constant and `_last_user_audio_mono` as the plan specified, replaced +by `USER_ACTIVE_GUARD_DELTAS = 25` plus `note_user_audio()` / +`note_assistant_delta()` / `user_recently_active()`. The stale comment block that +documented the old constant was rewritten rather than left pointing at a name +that no longer exists. + +### Test-fixture adaptations + +- `AgentConfig` requires `name` and `role`, which the plan's fixture omitted, and + `tool_module_path` must resolve to a real module (`eva.assistant.tools.itsm_tools`); + `test_tools` does not exist. `ToolExecutor` also reads `scenario_db_path` at + construction, so the fixture writes an empty `db.json`. +- The plan's adapter tests queued inbound frames without calling `start()`, so no + receive loop existed to drain them. They now `start()` and `_settle()`. +- `1s` of mulaw resamples to 31998 PCM bytes, not 32000 — `ratecv` filter state + eats two samples. The plan's `[BYTES_PER_TICK] * 5` assertion is unreachable; + the test now asserts four full ticks plus a short-but-nonzero fifth, and that + every *released* chunk is exactly tick-sized (silence-padded). +- `Adapter.run_tick` gained the `barge_in` keyword, so the three test doubles + implementing the ABC were widened to match, and `_InterruptScheduler` gained + `arm_barge_in`. +- `test_worker_uses_configured_factory_and_timeout` pins the exact + `create_user_simulator` kwargs, so `framework=` was added to its expectation. + +## Verification actually performed + +`PYTHONPATH=/src uv run pytest tests/unit -q` → **2117 passed, 52 +skipped, 3 xfailed**. `pre-commit run --all-files` → all hooks pass. + +New tests: 1 base-server pacing, 6 openai-realtime (pacing both ways, delta-based +activity guard, truncation target/no-op/item tracking), 4 `TickResult`, 8 +tick-driven adapter (unpaced send, per-tick release, silent-tick emits nothing, +played position, no tick-duration floor, barge-in position, barge-in discard), +2 scheduler barge-in arming, 4 framework selection, 3 worker pacing. + +## Task 9 — NOT DONE + +Every step needs live paid runs against OpenAI Realtime plus the cascade caller's +STT/LLM/TTS providers, and step 5 additionally needs Plan 2's interruption path to +be stable — it is still being fixed in the other worktree, and that changelog +lists defect 4b (the interrupt check preempting ordinary turn-taking) as an open +design question. Running the fidelity comparison before 4b is settled would +measure Plan 2's turn-routing bug, not this plan's clock. + +Step 5 is the one that matters: it is the entire justification for this plan and +the evidence for or against porting `gemini_live` and `grok_voice`. Expected +result is `slip_ms == 0` and `dropped == false` on the tick-driven path against +non-zero slip and some drops on the real-time path. From c0ffdba8ba0555c622dc3f805e3ad6bc779ad385 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Tue, 18 Aug 2026 23:41:24 -0700 Subject: [PATCH 56/65] make the tick-driven path actually run against a live provider Two faults the unit tests could not see, both found by running it: - With all pacing removed nothing waited for the provider, so the tick loop exhausted its whole budget in 160ms and every call ended at tick zero. Ticks now wait up to one tick for a full tick of assistant audio, returning the moment it is available, so nothing already generated is held back. - Emitting nothing on a silent tick starves the provider's VAD of the trailing silence that ends the caller's turn, so the assistant never replies. The freeze comes from not ticking while the caller thinks, not from sending nothing inside a tick; every tick now sends a full tick, unpaced. Co-Authored-By: Claude Opus 5 (1M context) --- docs/changelog_cascade_tick_driven.md | 134 +++++++++++++++--- .../cascade/adapter/realtime_ws.py | 4 + .../cascade/adapter/tick_driven.py | 68 +++++++-- .../cascade/test_tick_driven_adapter.py | 55 ++++++- 4 files changed, 220 insertions(+), 41 deletions(-) diff --git a/docs/changelog_cascade_tick_driven.md b/docs/changelog_cascade_tick_driven.md index 8f5fcb37..14e57271 100644 --- a/docs/changelog_cascade_tick_driven.md +++ b/docs/changelog_cascade_tick_driven.md @@ -3,8 +3,10 @@ Plan: `docs/superpowers/plans/2026-08-06-cascade-tick-driven-adapter.md` Implemented Tasks 1-8 on `worktree-user-sim-phase-3`, branched off -`feat/cascade-user-simulator` (Plan 1 + Plan 2 as merged there). Task 9 is live -end-to-end verification and has **not** been run — see the last section. +`feat/cascade-user-simulator` (Plan 1 + Plan 2 as merged there). Task 9's steps +1-4 were run live; step 6 was blocked externally and step 5 was deliberately +deferred — see the last two sections, which also record two design errors in the +plan that only a live run exposed. ## Deviations from the plan, and why @@ -35,9 +37,9 @@ timing — which is the one thing this plan is actually about. Consequences: `perturbator` is accepted (and inherited) rather than rejected, so the framework-selection site in `simulator.py` passes it unconditionally instead of using the plan's `**({...} if adapter_cls is RealtimeWSAdapter else {})` -conditional. Perturbation is applied only to ticks that carry real caller audio — -mixing ambient noise into a stalled tick would emit frames and unfreeze the -assistant, defeating the whole mechanism. +conditional. Perturbation is applied exactly as on the real-time path (ambient +noise replaces the silence a quiet tick already sends) — see design error 2 below +for why the plan's "emit nothing when silent" rule had to go. ### Nothing armed a barge-in, so Task 8 would have been dead code @@ -91,25 +93,115 @@ that no longer exists. ## Verification actually performed -`PYTHONPATH=/src uv run pytest tests/unit -q` → **2117 passed, 52 +`PYTHONPATH=/src uv run pytest tests/unit -q` → **2119 passed, 52 skipped, 3 xfailed**. `pre-commit run --all-files` → all hooks pass. New tests: 1 base-server pacing, 6 openai-realtime (pacing both ways, delta-based activity guard, truncation target/no-op/item tracking), 4 `TickResult`, 8 -tick-driven adapter (unpaced send, per-tick release, silent-tick emits nothing, -played position, no tick-duration floor, barge-in position, barge-in discard), +tick-driven adapter (unpaced send, per-tick release, silent tick still sends a +full tick, played position, buffered ticks release without delay, quiet tick waits +out the grace, early return mid-grace, barge-in position, barge-in discard), 2 scheduler barge-in arming, 4 framework selection, 3 worker pacing. -## Task 9 — NOT DONE - -Every step needs live paid runs against OpenAI Realtime plus the cascade caller's -STT/LLM/TTS providers, and step 5 additionally needs Plan 2's interruption path to -be stable — it is still being fixed in the other worktree, and that changelog -lists defect 4b (the interrupt check preempting ordinary turn-taking) as an open -design question. Running the fidelity comparison before 4b is settled would -measure Plan 2's turn-routing bug, not this plan's clock. - -Step 5 is the one that matters: it is the entire justification for this plan and -the evidence for or against porting `gemini_live` and `grok_voice`. Expected -result is `slip_ms == 0` and `dropped == false` on the tick-driven path against -non-zero slip and some drops on the real-time path. +## Two design errors in the plan, found only by running it + +The unit tests all passed while the path was completely non-functional. Both +faults were invisible to them because both are about what happens across *many* +ticks against a real provider. + +### 1. Removing all pacing left nothing to wait for the provider + +First live run ended after 160ms with `reason: timeout` and an empty transcript. +`max_ticks = timeout * 1000 / TICK_DURATION_MS` ticks ran to exhaustion instantly: +the real-time adapter's 200ms per-tick floor was the only thing that had ever +given the assistant time to produce anything, and the plan removed it without +replacement. + +Added `QUIET_TICK_GRACE_S` (one tick, 200ms) and `_await_tick_of_audio()`: a tick +waits for a full tick of assistant audio, returning the moment it is available. +This is not pacing — it never delays audio that has already arrived, so a provider +generating faster than real time is still drained as fast as it produces and +caller compute still costs the conversation nothing. It only bounds how long "the +assistant has said nothing yet" takes to establish. The wait is driven by an +`asyncio.Event` set from a new `_on_inbound_audio()` hook on `RealtimeWSAdapter` +(a no-op there). + +### 2. "Nothing is emitted on a silent tick" was backwards + +Second live run reached a real turn — assistant greeted, caller replied at tick 23 +— and then died at the 40s liveness check with the assistant never answering. + +The plan asserts that emitting nothing freezes the assistant, and treats that as +the mechanism. Emitting nothing does freeze it, but it also starves the provider's +VAD of the trailing silence that *ends the caller's turn*, so no response is ever +generated. The freeze this plan is built on comes from the caller not calling +`run_tick` while it thinks; it does not require, and is actively broken by, a tick +that puts nothing on the wire. That is why the real-time adapter sends a full tick +every tick. + +`run_tick` now sends a full tick of audio — real or silence, perturbation applied +as on the real-time path — with no pacing sleeps and no minimum tick duration. +The plan's `test_nothing_is_emitted_on_a_silent_tick` was inverted accordingly, +and its two "no wait" tests were reframed: they now assert that ticks with audio +*already buffered* release without delay, plus new tests that a quiet tick waits +out the grace and that a tick returns early when audio arrives mid-grace. + +## Task 9 — steps 1-4 and 6 run; step 5 not run + +One ITSM record (record 1), `gpt-realtime-mini`, behaviors off, metrics enabled. +`--metrics=` was used only for the first diagnostic run. + +**Steps 1-2 (artifacts, tools, DB).** Tick-driven run completed, ended `goodbye`, +7 turns. Full artifact set produced and identical to the baseline's: +`audit_log.json`, `transcript.jsonl`, `audio_user.wav`, `audio_assistant.wav`, +`audio_user_clean.wav`, `audio_mixed.wav`, `framework_logs.jsonl`, +`pipecat_metrics.jsonl`, `initial_scenario_db.json`, `final_scenario_db.json`. +Audit log holds 2 `tool_call` / 2 `tool_response` entries +(`verify_employee_auth`, `attempt_account_unlock`), and the DB diff is exactly the +intended mutation: `active_directory.locked True -> False`, `lock_reason +too_many_attempts -> None`, plus the two session-auth fields. + +**Step 3 (latency).** Compared against the same record on the same framework with +`TICK_DRIVEN_FRAMEWORKS` temporarily emptied — a like-for-like real-time baseline, +which the ElevenLabs run could not provide: + +| | tick-driven | real-time | +|---|---|---| +| completed / end reason | yes / goodbye | yes / goodbye | +| `model_response` p50 | 765 ms | 780 ms | +| `model_response` mean (n) | 823 ms (4) | 1178 ms (2) | +| tools called | both | both | + +p50 is within 2%. The mean gap is small-n noise (4 samples vs 2), not distortion — +the measurement window (caller stops → first assistant byte) contains no caller +compute either way. + +**Step 4 (turn ordering).** Timestamps monotonic on both paths. The tick-driven +transcript alternates cleanly; the real-time baseline has two caller turns landing +before the assistant's reply block and answers "which system?" *before* the caller +says "Active Directory" — i.e. tick-driving visibly improved ordering, which is the +direction this plan predicts. The duplicated/concatenated assistant entries appear +**identically on the baseline**, so they are a pre-existing `openai_realtime` +transcript artifact, not something this plan introduced. + +**Step 6 (ElevenLabs regression).** Could not be completed: the ElevenLabs +simulator failed to connect to ElevenLabs' own service +(`EOFError: connection closed while reading HTTP status line`) on all three +attempts, before any of this plan's code ran. External credential/connectivity +issue, unrelated to these changes. The paced server branch was instead exercised +by the Step 3 baseline run, which used `paced_output=True` and completed normally +— so the pacing path is verified working, just not with the ElevenLabs caller. + +**Validation "failure" on every run, including the baseline and the ElevenLabs +attempt**, is `user_speech_fidelity` erroring with +`Unable to load vertex credentials from environment`. Environmental, identical +across all paths. `conversation_valid_end` and `user_behavioral_fidelity` both +scored 1.0 on the tick-driven run. + +**Step 5 (interruption fidelity) NOT run**, by decision. It needs Plan 2's +interruption path to be stable — still being fixed in the other worktree, whose +changelog lists defect 4b (the interrupt check preempting ordinary turn-taking) as +an open design question. Running it before 4b is settled would measure Plan 2's +turn-routing bug rather than this plan's clock. It remains the deciding evidence +for porting `gemini_live` and `grok_voice`; expected result is `slip_ms == 0` and +`dropped == false` tick-driven against non-zero slip and some drops real-time. diff --git a/src/eva/user_simulator/cascade/adapter/realtime_ws.py b/src/eva/user_simulator/cascade/adapter/realtime_ws.py index 3ae154bf..4dd3e817 100644 --- a/src/eva/user_simulator/cascade/adapter/realtime_ws.py +++ b/src/eva/user_simulator/cascade/adapter/realtime_ws.py @@ -207,6 +207,10 @@ def _ingest(self, raw: str) -> None: payload = message.get("media", {}).get("payload", "") if payload: self._inbound.extend(self._mulaw8k_to_pcm16k(base64.b64decode(payload))) + self._on_inbound_audio() + + def _on_inbound_audio(self) -> None: + """Hook fired after inbound audio lands in the buffer. No-op on this path.""" def _mulaw8k_to_pcm16k(self, mulaw: bytes) -> bytes: """Convert 8kHz mulaw from the wire to PCM16 at the caller sample rate.""" diff --git a/src/eva/user_simulator/cascade/adapter/tick_driven.py b/src/eva/user_simulator/cascade/adapter/tick_driven.py index 534953bd..d75c096b 100644 --- a/src/eva/user_simulator/cascade/adapter/tick_driven.py +++ b/src/eva/user_simulator/cascade/adapter/tick_driven.py @@ -1,13 +1,18 @@ """Tick-driven adapter: the caller owns the simulation clock. Talks to the same assistant server over the same Twilio WebSocket as the -real-time adapter, with two differences. Outbound audio is not paced, because -the server is not pacing its own output either (``paced_output=False``); and -nothing at all is emitted on a tick where the caller is silent. - -Because the provider's VAD advances on audio received rather than wall time, -not sending audio freezes the assistant. That is what makes caller compute time -invisible here — and why nothing may be emitted on a stalled tick. +real-time adapter, and sends a full tick of audio — real or silence — on every +tick just as it does. The difference is timing: outbound frames carry no pacing +sleeps, because the server is not pacing its own output either +(``paced_output=False``), and there is no minimum tick duration. + +Because the provider's VAD advances on audio received rather than on wall time, +the conversation advances only when the caller ticks it. While the caller is +generating a turn no tick runs, so no audio flows and the assistant is frozen — +that is what makes caller compute time invisible. The freeze comes from *not +ticking*, not from emitting nothing within a tick: a tick that sent nothing would +starve the VAD of the trailing silence that ends the caller's turn, and the +assistant would never reply at all. Inbound audio is released strictly one tick at a time however much arrives at once, which the real-time adapter already does; the difference is that here the @@ -27,7 +32,7 @@ import time from eva.user_simulator.cascade.adapter.realtime_ws import FRAMES_PER_TICK, RealtimeWSAdapter -from eva.user_simulator.cascade.constants import BYTES_PER_TICK +from eva.user_simulator.cascade.constants import BYTES_PER_TICK, SILENCE_BYTE, TICK_DURATION_MS from eva.user_simulator.cascade.tick_result import TickResult, played_audio_ms, split_tick_audio from eva.utils.logging import get_logger @@ -36,6 +41,16 @@ MAX_INACTIVE_SECONDS = 40.0 """Fail loudly if the provider goes quiet this long (tau: DEFAULT_AUDIO_NATIVE_MAX_INACTIVE_SECONDS).""" +QUIET_TICK_GRACE_S = TICK_DURATION_MS / 1000 +"""How long a tick waits for a full tick of assistant audio before calling it silence. + +Not pacing: it never *delays* audio that has already arrived, so a provider +generating faster than real time is still drained as fast as it produces, and +caller compute still costs the conversation nothing. It is only the bound on how +long "the assistant has said nothing yet" takes to establish. Without it the loop +spins through its whole tick budget in milliseconds and every conversation ends at +tick zero with nothing said.""" + class TickDrivenAdapter(RealtimeWSAdapter): """Exchanges one tick of audio with an unpaced assistant server.""" @@ -56,6 +71,7 @@ def __init__( ) self._ticks_released = 0 self._last_inbound_monotonic = time.monotonic() + self._audio_arrived = asyncio.Event() @property def played_ms(self) -> int: @@ -93,12 +109,15 @@ async def run_tick(self, tick_number: int, outgoing_audio: bytes | None, *, barg await self._send_speech_event("user_speech_stop") self._caller_speaking = is_speaking - if outgoing_audio: - # Perturbation only ever rides on real caller audio here: mixing ambient - # noise into a stalled tick would emit frames and unfreeze the assistant. - await self._send_unpaced(self._apply_perturbation(outgoing_audio) or outgoing_audio) + # Every tick puts a full tick on the wire, silence included, exactly as the + # real-time adapter does. The provider's VAD advances on audio *received*, so + # a tick that emits nothing never ends the caller's turn and the assistant + # never replies. The freeze this plan is built on comes from the caller not + # calling `run_tick` while it thinks — not from emitting nothing inside one. + outgoing = self._apply_perturbation(outgoing_audio) + await self._send_unpaced(outgoing or SILENCE_BYTE * self._bytes_per_tick) - await asyncio.sleep(0) + await self._await_tick_of_audio() raw = bytes(self._inbound[: self._bytes_per_tick]) del self._inbound[: len(raw)] chunk, _ = split_tick_audio(raw, self._bytes_per_tick) @@ -118,6 +137,29 @@ async def run_tick(self, tick_number: int, outgoing_audio: bytes | None, *, barg interruption_audio_start_ms=interruption_start, ) + def _on_inbound_audio(self) -> None: + """Wake a tick that is waiting on assistant audio.""" + self._audio_arrived.set() + + async def _await_tick_of_audio(self) -> None: + """Wait until a whole tick of assistant audio is buffered, or the grace expires. + + Returns as soon as the buffer is full, so nothing already generated is held + back. The grace only bounds the silent case. + """ + deadline = time.monotonic() + QUIET_TICK_GRACE_S + while len(self._inbound) < self._bytes_per_tick: + remaining = deadline - time.monotonic() + if remaining <= 0: + return + self._audio_arrived.clear() + if len(self._inbound) >= self._bytes_per_tick: + return + try: + await asyncio.wait_for(self._audio_arrived.wait(), timeout=remaining) + except TimeoutError: + return + async def _send_unpaced(self, pcm: bytes) -> None: """Split one tick of PCM16 into wire frames and send them with no sleeps.""" mulaw = self._pcm16k_to_mulaw8k(pcm) diff --git a/tests/unit/user_simulator/cascade/test_tick_driven_adapter.py b/tests/unit/user_simulator/cascade/test_tick_driven_adapter.py index 3241064d..96cb96c8 100644 --- a/tests/unit/user_simulator/cascade/test_tick_driven_adapter.py +++ b/tests/unit/user_simulator/cascade/test_tick_driven_adapter.py @@ -2,7 +2,7 @@ import json import time -from eva.user_simulator.cascade.adapter.tick_driven import TickDrivenAdapter +from eva.user_simulator.cascade.adapter.tick_driven import QUIET_TICK_GRACE_S, TickDrivenAdapter from tests.unit.user_simulator.cascade.test_realtime_ws_adapter import ( BYTES_PER_TICK, FakeWebSocket, @@ -19,11 +19,14 @@ async def test_outbound_audio_is_sent_without_pacing_sleeps(): ws = FakeWebSocket() adapter = TickDrivenAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) await adapter.start() + # Assistant audio already buffered, so the tick has no reason to wait for any. + await ws.inbound.put(_media_frame(b"\xff" * 8000)) + await _settle() started = time.monotonic() await adapter.run_tick(0, b"\x00" * BYTES_PER_TICK) - # The real-time adapter would spend ~200ms pacing this. + # The real-time adapter would spend ~200ms pacing the outbound frames. assert time.monotonic() - started < 0.05 assert len(_media(ws)) == 10 await adapter.stop() @@ -49,15 +52,16 @@ async def test_burst_of_provider_audio_releases_one_tick_at_a_time(): await adapter.stop() -async def test_nothing_is_emitted_on_a_silent_tick(): +async def test_a_silent_tick_still_puts_a_full_tick_of_silence_on_the_wire(): ws = FakeWebSocket() adapter = TickDrivenAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) await adapter.start() await adapter.run_tick(0, None) - # Audio sent during a stall would advance the provider's VAD and break the freeze. - assert _media(ws) == [] + # The provider's VAD ends the caller's turn on received silence. A tick that + # emitted nothing would starve it and the assistant would never reply. + assert len(_media(ws)) == 10 await adapter.stop() @@ -87,19 +91,56 @@ async def test_a_tick_with_no_assistant_audio_does_not_advance_the_played_positi await adapter.stop() -async def test_run_tick_does_not_wait_out_the_tick_duration(): +async def test_ticks_with_audio_already_buffered_do_not_wait_out_the_tick_duration(): ws = FakeWebSocket() adapter = TickDrivenAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) await adapter.start() + # 1s of audio arrives at once; releasing it must not take 1s of wall time. + await ws.inbound.put(_media_frame(b"\xff" * 8000)) + await _settle() started = time.monotonic() - await asyncio.gather(*(adapter.run_tick(tick, None) for tick in range(3))) + for tick in range(4): + await adapter.run_tick(tick, None) # The real-time adapter enforces a 200ms floor per tick; this one must not. assert time.monotonic() - started < 0.05 await adapter.stop() +async def test_a_tick_waits_out_the_grace_before_calling_the_assistant_silent(): + ws = FakeWebSocket() + adapter = TickDrivenAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await adapter.start() + + started = time.monotonic() + result = await adapter.run_tick(0, None) + + # Without this bound the tick loop spins through its whole budget instantly and + # the call ends at tick zero having heard nothing. + assert result.assistant_audio_raw_bytes == 0 + assert time.monotonic() - started >= QUIET_TICK_GRACE_S + await adapter.stop() + + +async def test_a_tick_returns_as_soon_as_audio_arrives_mid_grace(): + ws = FakeWebSocket() + adapter = TickDrivenAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await adapter.start() + + async def _deliver_late() -> None: + await asyncio.sleep(QUIET_TICK_GRACE_S / 4) + await ws.inbound.put(_media_frame(b"\xff" * 8000)) + + asyncio.create_task(_deliver_late()) + started = time.monotonic() + result = await adapter.run_tick(0, None) + + assert result.assistant_audio_raw_bytes == BYTES_PER_TICK + assert time.monotonic() - started < QUIET_TICK_GRACE_S + await adapter.stop() + + async def test_barge_in_reports_the_played_position_not_the_received_position(): ws = FakeWebSocket() adapter = TickDrivenAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) From b3525aca0c86b55019117f4e9f5807d7b9c28146 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Wed, 19 Aug 2026 22:07:15 -0700 Subject: [PATCH 57/65] record ablation findings: 89% barge-in drop rate on the real-time path Adds the session's defects (cumulative-silence inactivity timeout, 200ms turn boundaries, opener-before-content), the rate-gate add-then-remove reasoning, and the headline measurement: 35 barge-ins fired, 4 kept, 31 dropped for lateness, median slip 1902ms against a 1500ms budget. Corrects an earlier claim that the interrupt path affects metric turn numbering - it does not; nothing under src/eva/metrics reads caller_turn. --- docs/changelog_cascade_out_of_turn.md | 124 +++++++++++++++++++++++++- 1 file changed, 122 insertions(+), 2 deletions(-) diff --git a/docs/changelog_cascade_out_of_turn.md b/docs/changelog_cascade_out_of_turn.md index 062a2f1b..25737aea 100644 --- a/docs/changelog_cascade_out_of_turn.md +++ b/docs/changelog_cascade_out_of_turn.md @@ -158,8 +158,11 @@ collapse — the turns were relabelled. Structurally: `_run` only reaches `_take assistant is silent, but the check fires while it is speaking, so with interruptions enabled the caller preferentially speaks via the check at 2s granularity instead of waiting out the 1s silence gate. Knock-on effects: turns taking the interrupt path bypass -`_maybe_arm_self_correction` and `_prerender_candidate`, and log as `interruption` rather -than `caller_turn`, which affects metric turn numbering. Deciding how the check and the turn +`_maybe_arm_self_correction` and `_prerender_candidate`, so enabling interruptions silently +disables part of the other two behaviours. They also log as `interruption` rather than +`caller_turn` — which does **not** affect metrics (verified: nothing under `src/eva/metrics/` +reads `caller_turn`, and turns are numbered from `audio_start(simulated_user)`, which fires on +both paths), only the human-readable event log and the ablation analysis. Deciding how the check and the turn gate should interact is a design change to Plan 2, left for the author. ### Method note @@ -176,3 +179,120 @@ reading *one sequence end to end*. Prefer a single full timeline over a summary decision provider stalls the wire. Fixing it means moving the checks off the tick loop. - Task 13 steps 2-6 need re-running: step 5 has never produced data, and the `interrupt` and `all-on` rows from the last pass aggregate retry attempts (12 event files, not 9). + +--- + +## Session 2: further defects, and the measurement that matters + +### 5. inactivity_timeout measured cumulative, not contiguous, silence — FIXED (abd1a42a) + +**A Plan 1 bug, not a Plan 2 one, and it has distorted every run including baseline.** + +`_assistant_is_inactive` was only ever reached on ticks where the assistant was *silent*, +because the speech branch `continue`d before it. That made its own reset line unreachable in +production, so `_ticks_assistant_silent` accumulated quiet ticks across the whole call and +never reset. "Silent for 120s" therefore meant "quiet ticks have totalled 120s at some point", +which any long healthy conversation eventually satisfies. + +Measured live: one conversation was killed for `inactivity_timeout` after **73.7s** of actual +contiguous silence, in a 197.6s call. + +The unit test that appeared to cover this passes because it calls the method *directly* with a +speech tick — exercising a branch the real loop never reaches. Worth remembering as a shape: +a test can cover a line and still not cover the path. + +Now called on every tick. Certainty: high — mechanism read from source and confirmed against +event timestamps. + +### 6. Assistant turn boundaries were any 200ms gap — FIXED (abd1a42a) + +The interruption cap allows one barge-in per assistant turn, but "new turn" was implemented as +"a single tick with no assistant audio". A pause between sentences therefore re-armed the cap +mid-utterance. Now uses `is_new_assistant_turn`, requiring the same 1s (`WAIT_TO_RESPOND_OTHER_MS`) +the turn gate already applies. Note this also means earlier per-turn rate arithmetic used +*speech segments* as the denominator, which inflated it. + +### 7. The opener was emitted before the content existed — FIXED (abd1a42a) + +Plan 2's design plays a pre-rendered opener the instant the decision fires, to hide ~1s of +generation latency. But that commits the caller to barging in before knowing whether it has +anything to say. Both a hang-up and a stale drop then left an orphaned "Actually—" on the wire. + +Content is now generated *first*; the opener and content are queued together only if the line +is worth speaking. Dropping now costs zero audio. The cost is that the barge-in lands ~1s after +the decision rather than ~200ms — which is what makes the slip measurement below meaningful +rather than cosmetic. + +### The interrupt rate gate: added, then removed (97448885, then 3cb17e37) + +Recorded because the reasoning is the useful part. A rate gate was added on the belief that the +decision "always said YES". **It does not.** Measured: in one call the check was asked ~101 +times and answered YES 14 times (~14%) — correctly NO through most of each assistant turn, YES +once near the end. The defect was *where* it fired, not how often: "the caller has heard enough +to reply" and "the assistant is finishing" are nearly the same instant, so barge-ins displaced +ordinary turn-taking rather than being excessive. + +So the gate was a random suppressor discarding decisions that were correct, masking a judgement +problem. Passing the caller's goal to the decision (1d139c07) addresses the judgement at source, +which made the gate a second suppressor stacked on a fix. Removed. The per-turn eligibility flag +remains — that is the one-barge-in-per-turn cap, which is structurally right regardless. + +### Goal-aware interrupt decision (1d139c07) + +`summarize_goal` passes high-level goal, must/nice-to-have criteria, option-evaluation steps, +and the resolution/failure/escalation conditions. Field meanings live in the prompt template so +they stay static across calls; the block sits ahead of the conversation history so it is +prompt-cacheable. `edge_cases` and `information_required` are omitted — long, and about how to +answer questions rather than whether the goal is finished. Renders ~605 tokens on a real ITSM +record, of which `negotiation_behavior` is ~60%; drop that field first if cost bites. + +## The headline result: reactive interruption does not work on the real-time path + +Run: 3 ITSM records x 3 trials, concurrency 3 (deliberately low — slip is a wall-clock +measurement and contention would inflate it), no rate gate, generate-first. + +| | | +|---|---| +| barge-ins fired | 35 (across 75 assistant speech segments, ~47%) | +| kept | **4 (11%)** | +| dropped, slip > 1500ms | **31 (89%)** | +| dropped, assistant already stopped | 0 | +| slip_ms | min 1295, **median 1902**, max 2991 | + +Median generation latency is **1902ms against a 1500ms staleness budget**, and every single +drop was for lateness — not one because the assistant had finished. On `RealtimeWSAdapter`, +reactive interruption fires and then fails to land roughly nine times in ten. + +This is exactly the number Task 13 step 3 was designed to produce and never could while slip +was measured on a frozen tick counter. The plan states that a high drop rate is "the signal to +reconsider tick-driving more frameworks (Plan 3)" — that signal has now arrived quantified. On +a tick-driven adapter the assistant is frozen while the caller thinks, so slip vanishes and all +35 would land. + +**Before concluding the design is unworkable, try a fast `decision_llm`.** It currently defaults +to `user-llm`, a full-size model, and both the check and the content generation run on it. +A small fast model could plausibly halve slip. Untested. + +## Still outstanding + +- **Task 13 remains the only incomplete task.** Steps 2-6 all need re-running against the + current build: every earlier number describes code that no longer exists. +- Step 5 (self-correction validity) has **never** produced data, and it is the check the plan + calls the most important — it is also where this implementation departs furthest from the plan. +- 4b (the interrupt check preempting ordinary turn-taking) — still a design question. +- Defect 1's unexplained residual (backchannel timeouts still above baseline). +- `_run_checks` awaits two LLM calls inline in the tick loop with **no timeout**, so a slow + decision provider stalls the wire. +- The last interrupt run still showed 5 `inactivity_timeout` and 2 `unknown` of 10 *after* the + contiguous-silence fix. Cause unknown — deliberately not guessed at. +- `gpt-5.4` has two deployments in `.env`, one with a stale `sk-svcacct-` key, so it fails + preflight at random. Pre-existing and unrelated; ablations use `gpt-5.2` to avoid it. + +## Method notes worth keeping + +- Every diagnosis that turned out wrong came from reading **aggregate counts** and inferring a + mechanism. Every one was settled immediately by reading **one sequence end to end**. Prefer a + single full timeline over a summary table. +- Scripted text-surgery on source caused a fourth defect this session: `t.index("def test_...")` + matched inside `async def` and silently stripped the keyword, breaking test collection. The + handoff already recorded three from the same cause. Use targeted edits. From a0999423f421781fe97438bf5ac6dbd98dd08fc8 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 20 Aug 2026 17:09:17 -0700 Subject: [PATCH 58/65] Remove temp files --- docs/changelog_cascade_out_of_turn.md | 298 ---------------------- docs/changelog_cascade_stt_reliability.md | 163 ------------ docs/changelog_cascade_tick_driven.md | 207 --------------- 3 files changed, 668 deletions(-) delete mode 100644 docs/changelog_cascade_out_of_turn.md delete mode 100644 docs/changelog_cascade_stt_reliability.md delete mode 100644 docs/changelog_cascade_tick_driven.md diff --git a/docs/changelog_cascade_out_of_turn.md b/docs/changelog_cascade_out_of_turn.md deleted file mode 100644 index 25737aea..00000000 --- a/docs/changelog_cascade_out_of_turn.md +++ /dev/null @@ -1,298 +0,0 @@ -# Working log: cascade out-of-turn behaviors (Plan 2) - -Plan: `docs/superpowers/plans/2026-08-06-cascade-out-of-turn-behaviors.md` -Handoff: `docs/handoff_cascade_plan2.md` - -## Deviations from the plan, decided up front - -### Structured fields come from a separate follow-up call, not a JSON turn contract - -The plan's Tasks 10 and 12 read `self_correction` and `next_interruption` out of a JSON -turn response. That contract does not exist: Plan 1 removed it because demanding "a single -JSON object and nothing else" suppressed the `end_call` tool call and hung conversations -until timeout. - -Instead the turn call keeps its exact current shape (plain text + `END_CALL_TOOL`), and a -second call — made *after* the turn's audio is already queued — asks only for the extra -field. Reasoning: - -- It structurally cannot regress `end_call`, because the turn call is untouched. -- It costs no conversational latency. The correction is not needed until - `SELF_CORRECTION_DELAY_MS` (1200ms) after the assistant *starts replying*, which is - itself well after the caller's audio went out. -- The user additionally required that a self-correction never attach to a final turn, so - arming is gated on `end_call` being false. - -Certainty: high. This is strictly safer than the plan's version and the latency argument is -structural, not empirical. - -### Re-added `TranscriptBuffer.current_text()` - -Deleted as dead code at the end of Plan 1; Tasks 7, 9, and 12 all call it. Restored with the -`[CURRENTLY SPEAKING, INCOMPLETE]` marker the backchannel prompt's few-shot examples depend on. - -Certainty: high — the prompt examples are meaningless without the marker. - -### Task 13 stages explicitly rather than `git add -A` - -The handoff forbids `git add -A` at the repo root (many unrelated untracked files). - -## Progress - -- [x] Task 1: behavior constants and vocabularies -- [x] Task 2: behavior flags on the config -- [x] Task 3: check-tick predicate -- [x] Task 4: decision prompts (+ `current_text()`) -- [x] Task 5: decision checks -- [x] Task 6: phrase cache -- [x] Task 7: backchannel behavior -- [x] Task 8: streaming TTS -- [x] Task 9: reactive interruption -- [x] Task 10: self-correction -- [x] Task 11: ambient noise mixing -- [x] Task 12: speculative generation -- [ ] Task 13: live ablation verification - -## Further deviations found during implementation - -### Task 10's ordering had to be rebuilt for the separate-call design - -With a separate follow-up call the turn call has already produced the *correct* -utterance, so asking for "a correction" would have flipped a right answer into a wrong -one — inverting the plan's invariant and putting `must_have_criteria` at risk. Instead -the correction prompt asks for a deliberately WRONG variant of the already-generated -line; the slip is spoken as the turn and the model's original goal-consistent line is -armed as the correction. Wrong-then-right ordering is preserved exactly, and the -goal-consistent line is still what lands last. Certainty: high. - -### Task 11's tests contradicted Plan 1's always-send-silence invariant - -The plan's `run_tick` edit made sending conditional (`if outgoing: await send(...)`) and -its test asserted `media == []` with no perturbator. `RealtimeWSAdapter` deliberately -sends a full tick of frames every tick — real audio or synthesized silence — because -"gaps with no frames at all are what caused turn detection to misfire". Implemented as -mixing noise *into* what is already sent, never as gating whether to send. Tests assert -the invariant instead. Certainty: high — this is documented in the adapter's own docstring. - -### `_warn_unsupported_perturbation` removed - -It warned that cascade drops `background_noise`/`snr_db`/`connection_degradation`. -Task 11 makes all three take effect via `AudioPerturbator.apply()`, so the warning became -a false statement rather than a stale one. Certainty: high. - -### Decision prompts are loaded with `get_template`, not a round-tripped placeholder - -Plan Task 7 Step 5 suggested calling `get_prompt(..., conversation_history="{conversation_history}")` -so the placeholder survives for `ListenerDecisions` to fill later, and flagged that it might -not round-trip. `PromptManager.get_template()` returns the raw unformatted template, which -removes the failure mode entirely. Certainty: high. - -### Interruption consumes the transcript rather than peeking at it - -The plan's `_play_interruption` appends `buffer.current_text()` to history but leaves it in -the buffer, so the same assistant prefix is appended again at the next ordinary turn. -Consumed instead, and logged once via `_on_assistant_speaks`. Certainty: high. - -### Plan 1's `cascade_` guard test rewritten - -`test_no_cascade_specific_prompts_remain_in_the_prompt_file` asserted the substring -`cascade_` never appears in `simulation.yaml`. The invariant it protected is that no -cascade-only contract is layered onto the *turn call's* system prompt. Plan 2's prompts are -used only in their own standalone calls, so the test now asserts the real invariant: -`_messages()[0]["content"] == _build_prompt()`. Certainty: high — strictly stronger guard. - -## Defects found by the Task 13 ablation runs - -Four defects. All four surfaced only against live services; the unit suite was green -throughout, which is the same pattern the Plan 1 handoff warned about. - -### 1. A backchannel consumed the caller's turn — FIXED (cb834b75) - -`TickScheduler.run_tick` set `_awaiting_reply = True` on *any* outgoing audio. A continuer -earns no reply, so `may_take_turn()` blocked permanently and the call died at the -inactivity timeout. Live: 7/8 conversations vs 4/9 at baseline. Fixed with -`enqueue_backchannel()`, which tracks continuer bytes at the head of the playout queue. - -**Loose end, deliberately not fixed:** post-fix the backchannel run still ends 6 -timeout / 3 goodbye against baseline's 4/5, and the backchannel count fell 20 -> 6 -between runs unexplained. Candidate second mechanism: a backchannel still resets -`_ticks_since_caller_speech`, delaying the caller's own next turn by the full -`WAIT_TO_RESPOND_SELF_MS`. Unproven — may be n=9 noise. Do not call defect 1 closed. - -### 2. Slip was structurally unmeasurable — FIXED (7dba2ba7) - -`interrupt_slip_ms` differenced tick counters, but `run_tick` is only pumped by the `_run` -loop and `_play_interruption` is awaited from inside it, so `scheduler.tick` provably cannot -advance during the generation. All 525 logged interruptions reported `slip_ms=0` with -`intended_tick == actual_tick`, and `should_drop_interrupt`'s slip branch never engaged. -Now measured with `time.monotonic()`. Certainty: high — deductive, not statistical. - -### 3. Self-correction was unreachable — FIXED (7dba2ba7) - -`random.Random(0)` was reseeded identically per conversation. Its first draw below -`SELF_CORRECTION_RATE` (0.15) is #26, while a conversation runs ~7 turns, so no conversation -ever armed a correction — zero events across both configs that enabled it. Verified the CLI -flag *did* propagate (`config.json` showed `enable_self_correction: true`) before blaming the -RNG, specifically to avoid fixing the wrong layer. Now seeded from the record id via -`crc32`, keeping runs reproducible while differing across conversations. - -### 4. Degraded interruption runs — root cause was NOT what it first looked like - -Initial reading of the aggregates (62-107 interruptions against ~2.6 caller turns) suggested -runaway barge-ins and a missing cooldown. **That diagnosis was wrong.** Reading a single -conversation timeline end to end showed two different causes: - -**4a. The interruption path swallowed the hang-up — FIXED (7dba2ba7).** `_play_interruption` -did `utterance, _end_call = extract_turn(message)`, discarding the flag, while `_take_turn` -used it. A caller deciding to hang up mid-assistant-turn therefore emitted nothing (the model -called the tool instead of speaking) and the call could never end. Observed directly: after -`'Thanks. Goodbye.'` at tick 640, six interruptions with empty text at ticks 660-800 while -the assistant looped "Confirmed... your account is unlocked" and then "Sure. What can I help -you with?". This — not barge-in spam — is what depressed `task_completion` to 0.500. - -**4b. The interrupt check preempts ordinary turn-taking — NOT FIXED, design question.** -The gaps between logged "interruptions" were 10s, 18s, 50s, and their content was ordinary -replies ("Employee ID is E M P zero four eight two seven one"). These are normal turns -routed through the interruption path, not barge-ins, which is why `caller_turn` appeared to -collapse — the turns were relabelled. Structurally: `_run` only reaches `_take_turn` when the -assistant is silent, but the check fires while it is speaking, so with interruptions enabled -the caller preferentially speaks via the check at 2s granularity instead of waiting out the -1s silence gate. Knock-on effects: turns taking the interrupt path bypass -`_maybe_arm_self_correction` and `_prerender_candidate`, so enabling interruptions silently -disables part of the other two behaviours. They also log as `interruption` rather than -`caller_turn` — which does **not** affect metrics (verified: nothing under `src/eva/metrics/` -reads `caller_turn`, and turns are numbered from `audio_start(simulated_user)`, which fires on -both paths), only the human-readable event log and the ablation analysis. Deciding how the check and the turn -gate should interact is a design change to Plan 2, left for the author. - -### Method note - -Both times an initial diagnosis was wrong (the "service outage" that was really machine -sleep, and the "interruption storm" that was really a swallowed hang-up), the error came from -reading *aggregate counts* and inferring a mechanism. Both were settled immediately by -reading *one sequence end to end*. Prefer a single full timeline over a summary table. - -### Still outstanding - -- 4b (above), and defect 1's unexplained residual. -- `_run_checks` awaits two LLM calls inline in the tick loop with no timeout, so a slow - decision provider stalls the wire. Fixing it means moving the checks off the tick loop. -- Task 13 steps 2-6 need re-running: step 5 has never produced data, and the `interrupt` and - `all-on` rows from the last pass aggregate retry attempts (12 event files, not 9). - ---- - -## Session 2: further defects, and the measurement that matters - -### 5. inactivity_timeout measured cumulative, not contiguous, silence — FIXED (abd1a42a) - -**A Plan 1 bug, not a Plan 2 one, and it has distorted every run including baseline.** - -`_assistant_is_inactive` was only ever reached on ticks where the assistant was *silent*, -because the speech branch `continue`d before it. That made its own reset line unreachable in -production, so `_ticks_assistant_silent` accumulated quiet ticks across the whole call and -never reset. "Silent for 120s" therefore meant "quiet ticks have totalled 120s at some point", -which any long healthy conversation eventually satisfies. - -Measured live: one conversation was killed for `inactivity_timeout` after **73.7s** of actual -contiguous silence, in a 197.6s call. - -The unit test that appeared to cover this passes because it calls the method *directly* with a -speech tick — exercising a branch the real loop never reaches. Worth remembering as a shape: -a test can cover a line and still not cover the path. - -Now called on every tick. Certainty: high — mechanism read from source and confirmed against -event timestamps. - -### 6. Assistant turn boundaries were any 200ms gap — FIXED (abd1a42a) - -The interruption cap allows one barge-in per assistant turn, but "new turn" was implemented as -"a single tick with no assistant audio". A pause between sentences therefore re-armed the cap -mid-utterance. Now uses `is_new_assistant_turn`, requiring the same 1s (`WAIT_TO_RESPOND_OTHER_MS`) -the turn gate already applies. Note this also means earlier per-turn rate arithmetic used -*speech segments* as the denominator, which inflated it. - -### 7. The opener was emitted before the content existed — FIXED (abd1a42a) - -Plan 2's design plays a pre-rendered opener the instant the decision fires, to hide ~1s of -generation latency. But that commits the caller to barging in before knowing whether it has -anything to say. Both a hang-up and a stale drop then left an orphaned "Actually—" on the wire. - -Content is now generated *first*; the opener and content are queued together only if the line -is worth speaking. Dropping now costs zero audio. The cost is that the barge-in lands ~1s after -the decision rather than ~200ms — which is what makes the slip measurement below meaningful -rather than cosmetic. - -### The interrupt rate gate: added, then removed (97448885, then 3cb17e37) - -Recorded because the reasoning is the useful part. A rate gate was added on the belief that the -decision "always said YES". **It does not.** Measured: in one call the check was asked ~101 -times and answered YES 14 times (~14%) — correctly NO through most of each assistant turn, YES -once near the end. The defect was *where* it fired, not how often: "the caller has heard enough -to reply" and "the assistant is finishing" are nearly the same instant, so barge-ins displaced -ordinary turn-taking rather than being excessive. - -So the gate was a random suppressor discarding decisions that were correct, masking a judgement -problem. Passing the caller's goal to the decision (1d139c07) addresses the judgement at source, -which made the gate a second suppressor stacked on a fix. Removed. The per-turn eligibility flag -remains — that is the one-barge-in-per-turn cap, which is structurally right regardless. - -### Goal-aware interrupt decision (1d139c07) - -`summarize_goal` passes high-level goal, must/nice-to-have criteria, option-evaluation steps, -and the resolution/failure/escalation conditions. Field meanings live in the prompt template so -they stay static across calls; the block sits ahead of the conversation history so it is -prompt-cacheable. `edge_cases` and `information_required` are omitted — long, and about how to -answer questions rather than whether the goal is finished. Renders ~605 tokens on a real ITSM -record, of which `negotiation_behavior` is ~60%; drop that field first if cost bites. - -## The headline result: reactive interruption does not work on the real-time path - -Run: 3 ITSM records x 3 trials, concurrency 3 (deliberately low — slip is a wall-clock -measurement and contention would inflate it), no rate gate, generate-first. - -| | | -|---|---| -| barge-ins fired | 35 (across 75 assistant speech segments, ~47%) | -| kept | **4 (11%)** | -| dropped, slip > 1500ms | **31 (89%)** | -| dropped, assistant already stopped | 0 | -| slip_ms | min 1295, **median 1902**, max 2991 | - -Median generation latency is **1902ms against a 1500ms staleness budget**, and every single -drop was for lateness — not one because the assistant had finished. On `RealtimeWSAdapter`, -reactive interruption fires and then fails to land roughly nine times in ten. - -This is exactly the number Task 13 step 3 was designed to produce and never could while slip -was measured on a frozen tick counter. The plan states that a high drop rate is "the signal to -reconsider tick-driving more frameworks (Plan 3)" — that signal has now arrived quantified. On -a tick-driven adapter the assistant is frozen while the caller thinks, so slip vanishes and all -35 would land. - -**Before concluding the design is unworkable, try a fast `decision_llm`.** It currently defaults -to `user-llm`, a full-size model, and both the check and the content generation run on it. -A small fast model could plausibly halve slip. Untested. - -## Still outstanding - -- **Task 13 remains the only incomplete task.** Steps 2-6 all need re-running against the - current build: every earlier number describes code that no longer exists. -- Step 5 (self-correction validity) has **never** produced data, and it is the check the plan - calls the most important — it is also where this implementation departs furthest from the plan. -- 4b (the interrupt check preempting ordinary turn-taking) — still a design question. -- Defect 1's unexplained residual (backchannel timeouts still above baseline). -- `_run_checks` awaits two LLM calls inline in the tick loop with **no timeout**, so a slow - decision provider stalls the wire. -- The last interrupt run still showed 5 `inactivity_timeout` and 2 `unknown` of 10 *after* the - contiguous-silence fix. Cause unknown — deliberately not guessed at. -- `gpt-5.4` has two deployments in `.env`, one with a stale `sk-svcacct-` key, so it fails - preflight at random. Pre-existing and unrelated; ablations use `gpt-5.2` to avoid it. - -## Method notes worth keeping - -- Every diagnosis that turned out wrong came from reading **aggregate counts** and inferring a - mechanism. Every one was settled immediately by reading **one sequence end to end**. Prefer a - single full timeline over a summary table. -- Scripted text-surgery on source caused a fourth defect this session: `t.index("def test_...")` - matched inside `async def` and silently stripped the keyword, breaking test collection. The - handoff already recorded three from the same cause. Use targeted edits. diff --git a/docs/changelog_cascade_stt_reliability.md b/docs/changelog_cascade_stt_reliability.md deleted file mode 100644 index 3ba9f789..00000000 --- a/docs/changelog_cascade_stt_reliability.md +++ /dev/null @@ -1,163 +0,0 @@ -# Cascade STT reliability and provider abstraction - -Working log for the fix to Defect B (verbatim caller repeats) and the LiveKit STT port. -Reasoning and certainty are recorded per change. - -## Background: what actually broke - -Live run `cascade-repro-1` produced three verbatim repeats of the same caller utterance. -The assistant noticed: *"I'm hearing the same phrase repeated, so the line may be -transcribing you incorrectly."* - -Evidence chain: - -- `user_simulator_events.jsonl` shows three consecutive `caller_turn` events (ticks 312, - 390, 491) with **no `assistant_speech` between them**, while `transcript.jsonl` proves - the assistant spoke twice in that window, 14-20s before each repeat. -- The Scribe reconnect at 12:01:47 happened **after** all three repeats, so it is not the cause. -- The next successful commit contained **only** the 6th assistant utterance, with no trace - of the two missing ones. A merely slow commit would have landed in `committed` and - appeared prepended on the next `take_committed()`. It did not. So those utterances were - **never committed at all** - this is not a read-too-early race. - -Mechanism of the repeat itself is confirmed at `simulator.py:179`: `_take_turn` reads -`take_committed()`, and when it returns empty it still calls the caller LLM with unchanged -history. Same messages in, same utterance out. - -## The reframing that drives this work - -Provider-side turn detection was initially blamed. That was wrong, and the correction -matters for provider choice: - -- `TickScheduler.may_take_turn()` decides when the caller speaks, from counting silent - ticks of assistant **audio**. STT never participates. So provider endpointing cannot - corrupt who-speaks-when. -- We already run `commit_strategy=manual` (no provider VAD) and hit the failure anyway. - -The real root cause is upstream of any provider: **we read the transcript buffer -optimistically - no wait, no acknowledgement that the commit was processed, no fallback, -and no check that we heard anything at all.** - -Three failure shapes at read time: - -| State at read | Consequence | -|---|---| -| Nothing committed | Stale history -> verbatim repeat (Defect B) | -| Partially committed | Caller replies to half a sentence, silently. **Most insidious** | -| Fully committed | Correct | - -## Changes - -### 1. Bounded wait for the committed transcript - -**Certainty: high.** Implemented by *skipping* the turn and retrying on later ticks rather -than a blocking sleep - which fits the tick architecture and costs nothing, since each -retry just re-reads the buffer. Bounded by a tick counter so it cannot wait forever. - -Latency budget is ample: the caller's own LLM call takes ~10s observed, so absorbing a few -hundred ms of STT finalization is free. - -### 2. Fall back to the in-flight partial - -**Certainty: medium-high.** Approximate text beats stale text by a wide margin. Precedent: -tau-voice drives its interrupt/backchannel decisions off a linearly interpolated, -mid-word-truncating approximation (`get_proportional_text`, transcript_utils.py:7-25) and -that is good enough to have shipped. - -Open question: whether partials were actually flowing in the failing run. `commit()` clears -`in_flight`, and reconnect clears it too. Instrumentation now logs `in_flight` on this exact -path, but the defect has not recurred in the runs since, so this is **unmeasured**. - -### 3. Never generate a turn from unchanged history - -**Certainty: high on the rule, medium on the remedy.** The rule - never call the caller LLM -with history that did not change - is unambiguous. - -For the remedy, three options were considered: - -- *Stall until text arrives* - converts a corrupt-transcript bug into a dead-air bug. -- *Speak anyway* - the current behavior, i.e. the defect. -- **Chosen: tell the caller it did not hear.** Inject an explicit note so it says "Sorry, I - didn't catch that." This is what a real caller does, keeps the conversation alive, records - the failure honestly in the transcript, and can never emit a stale repeat. - -### 4-6. Provider-agnostic STT interface, LiveKit implementation, config + loss metric - -**Certainty: medium.** Motivation is that LiveKit's base `RecognizeStream` provides -`flush()`/`end_input()` (livekit-agents `stt.py:349-571`) - a flush sentinel *in the stream* -rather than our out-of-band `{"commit": true}` flag - plus typed events and a uniform -interface across 28 providers. Adopting it deletes our hand-rolled reconnect/idle-close -machinery. - -Caveats recorded before committing: - -- **Dependency cost is the real risk.** Any LiveKit STT plugin pulls `livekit-agents` -> - pinned `livekit==1.1.14`, a compiled native Rust/WebRTC wheel, plus `av` (FFmpeg - bindings), `sounddevice` and OpenTelemetry - for a codepath that never opens a room. - The `.venv.x86-broken/` directory in the tree makes this a concrete, not theoretical, risk. -- **Swappability is narrower than the catalogue suggests.** The uniform interface does not - expose whether provider turn detection can be disabled. That varies per plugin and is only - visible in each plugin's source. -- `ink-2` has **mandatory** turn detection (only `turn_start_threshold`, - `turn_eager_end_threshold`, `turn_end_threshold`, `turn_end_timeout_ms`; no off switch) - and is **English-only** per its docstring. `ink-whisper` has no interim results. - Neither Cartesia model satisfies both requirements today. - -The loss metric in change 6 exists so provider comparison is settled with data rather than -anecdote. - -## LiveKit spike results (measured, not inferred) - -Scratch venv, `livekit-agents` 1.6.8 + 4 plugins. One recorded 14s assistant utterance fed at -our 200ms tick cadence, then `flush()`. - -| Provider | interims (R2) | `flush()` honored | auto-final | verdict | -|---|---|---|---|---| -| `elevenlabs/scribe_v2_realtime` | 14 | **yes, 0.15s** | none (endpointing off) | full manual control | -| `cartesia/ink-2` | 42 | **no - explicitly ignored** | +1.20s after speech end | provider-driven only | -| `deepgram/flux-general-en` | - | - | - | blocked: WS handshake **HTTP 402** | -| `deepgram/nova-3` | - | - | - | same closure, same account | -| `assemblyai/universal-streaming-multilingual` | - | - | - | no key in `.env` | - -Cartesia logs it outright: `Cartesia STT stream.flush() was ignored.` - -**The decisive number: ink-2 auto-finalizes +1.20s after speech ends, while -`WAIT_TO_RESPOND_OTHER_MS` lets the caller take its turn at 1.0s.** So the final lands ~200ms -*after* we read the buffer - the late-final case, by a small margin. A bounded wait of ~500ms -covers it comfortably. This makes ink-2 viable *provided* changes 1-3 land first. - -**This also settles Defect B.** Scribe answers a commit in 150ms, so Defect B is an -intermittent dropout, not systematic slowness. That was the top falsification test - and it -says fix the race in place rather than redesign the transport. Changes 1-3 do exactly that -and are provider-independent. - -Integration details learned (needed by the real implementation): - -- Standalone use requires `async with livekit.agents.utils.http_context.open():` or an - explicitly passed `aiohttp.ClientSession`; plugins otherwise raise "http session outside of - a job context". -- Deepgram Flux is `deepgram.STTv2`, not `deepgram.STT` (which is the nova path). -- Install on arm64/py3.12: 74 packages, 268MB, all prebuilt wheels, no compilation. x86 CI - still unverified. - -## STT requirements these changes must preserve - -1. The caller owns the turn boundary; STT reports what was said. -2. In-flight partials for Plan 2's interrupt/backchannel checks (only when those are on). -3. Audio-only - we are a client on a Twilio WebSocket, with no access to the assistant's text. -4. A self-hostable option (NVIDIA Riva/NIM is first-party and runs offline). -5. No silent transcript loss. Violated by Defect B; the reason for changes 1-3. -6. Non-English support - the simulator takes a `language` param. - -## Related fixes landed alongside - -- **Deadlock (separate defect, 2/7 runs).** Caller said goodbye, assistant treated it as - terminal, `_awaiting_reply` never cleared, caller could never reach its `end_call` turn, - run stalled 5 minutes to pipecat's idle timeout. Fixed with `ASSISTANT_UNRESPONSIVE_MS` - (90s) in `may_take_turn()`, plus removal of contradictions in `END_CALL_DESCRIPTION`. - Threshold chosen from measurement: longest *legitimate* assistant gap observed live was - 220 ticks (44s), so 25s would have misfired mid-conversation in 2 of 5 runs. -- **Swallowed Scribe errors.** The receive loop dropped any message lacking a `text` field, - which includes error messages. Now logged explicitly. -- **Diagnostic instrumentation** for the commit boundary: per-commit audio fed vs non-silent - seconds, every Scribe message type, and a warning when a turn is taken having heard nothing. diff --git a/docs/changelog_cascade_tick_driven.md b/docs/changelog_cascade_tick_driven.md deleted file mode 100644 index 14e57271..00000000 --- a/docs/changelog_cascade_tick_driven.md +++ /dev/null @@ -1,207 +0,0 @@ -# Working log: cascade tick-driven adapter (Plan 3) - -Plan: `docs/superpowers/plans/2026-08-06-cascade-tick-driven-adapter.md` - -Implemented Tasks 1-8 on `worktree-user-sim-phase-3`, branched off -`feat/cascade-user-simulator` (Plan 1 + Plan 2 as merged there). Task 9's steps -1-4 were run live; step 6 was blocked externally and step 5 was deliberately -deferred — see the last two sections, which also record two design errors in the -plan that only a live run exposed. - -## Deviations from the plan, and why - -### The worktree was branched off `main`, not off the plan's branch - -`.claude/worktrees/user-sim-phase-3` was created at `e0041e3d` (main), so none of -Plan 1's or Plan 2's code was present — `src/eva/user_simulator/cascade/` did not -exist. Reset the worktree branch to `feat/cascade-user-simulator` (`3cb17e37`) -before starting. Nothing was lost; the branch had no commits of its own. - -### Tests in this worktree import the *main* checkout's `src/` - -The editable install resolves `eva` to `/Users/tara.bogavelli/third_eva/EVA-Bench3/src` -under pytest, so a bare `pytest` in the worktree silently tests the other agent's -Plan 2 working tree instead of these changes. Every test run here used -`PYTHONPATH=/src`. Anyone re-verifying this branch must do the same or -the results are meaningless. - -### `TickDrivenAdapter` subclasses `RealtimeWSAdapter` instead of being a peer - -The plan sketched a standalone class calling `RealtimeWSAdapter._pcm16k_to_mulaw8k` -as though it were static. It is not: both resamplers carry per-instance -`audioop.ratecv` filter state, and calling them unbound would either crash or -produce discontinuous audio. Subclassing reuses the handshake, receive loop, -error propagation, speech-event frames and resamplers, and overrides only the -timing — which is the one thing this plan is actually about. - -Consequences: `perturbator` is accepted (and inherited) rather than rejected, so -the framework-selection site in `simulator.py` passes it unconditionally instead -of using the plan's `**({...} if adapter_cls is RealtimeWSAdapter else {})` -conditional. Perturbation is applied exactly as on the real-time path (ambient -noise replaces the silence a quiet tick already sends) — see design error 2 below -for why the plan's "emit nothing when silent" rule had to go. - -### Nothing armed a barge-in, so Task 8 would have been dead code - -Task 8 specified the adapter half and the server half but no caller. The -interruption is enqueued as *audio* by `_play_interruption` and only reaches the -wire on a later tick, so `run_tick(barge_in=True)` had no natural call site. -Added `TickScheduler.arm_barge_in()`: it marks the next tick that actually puts -caller audio on the wire, so the truncation carries the played position as of -*that* tick rather than as of the decision. `_play_interruption` arms it on both -of its enqueue paths (speculative and freshly generated). Without this the -truncate frame would never have been sent. - -### Truncation target read off the audio delta, not `response.output_item.added` - -The plan said to track `active_item_id` wherever `response.output_item.added` is -handled — that case does not exist in `openai_realtime_server.py`. The audio -delta event carries the same `item_id` and is already dispatched, so the item is -recorded there and no new event handler was added. - -Verified the SDK entry point rather than trusting the plan's spelling: -`openai.resources.realtime.realtime.AsyncRealtimeConversationItemResource.truncate(*, -audio_end_ms, content_index, item_id, event_id)`. The call matches. It is wrapped -in a `try/except` that logs and returns — a failed truncation should degrade -fidelity, not kill the conversation. - -### `USER_ACTIVE_GUARD_S` removal - -Removed the constant and `_last_user_audio_mono` as the plan specified, replaced -by `USER_ACTIVE_GUARD_DELTAS = 25` plus `note_user_audio()` / -`note_assistant_delta()` / `user_recently_active()`. The stale comment block that -documented the old constant was rewritten rather than left pointing at a name -that no longer exists. - -### Test-fixture adaptations - -- `AgentConfig` requires `name` and `role`, which the plan's fixture omitted, and - `tool_module_path` must resolve to a real module (`eva.assistant.tools.itsm_tools`); - `test_tools` does not exist. `ToolExecutor` also reads `scenario_db_path` at - construction, so the fixture writes an empty `db.json`. -- The plan's adapter tests queued inbound frames without calling `start()`, so no - receive loop existed to drain them. They now `start()` and `_settle()`. -- `1s` of mulaw resamples to 31998 PCM bytes, not 32000 — `ratecv` filter state - eats two samples. The plan's `[BYTES_PER_TICK] * 5` assertion is unreachable; - the test now asserts four full ticks plus a short-but-nonzero fifth, and that - every *released* chunk is exactly tick-sized (silence-padded). -- `Adapter.run_tick` gained the `barge_in` keyword, so the three test doubles - implementing the ABC were widened to match, and `_InterruptScheduler` gained - `arm_barge_in`. -- `test_worker_uses_configured_factory_and_timeout` pins the exact - `create_user_simulator` kwargs, so `framework=` was added to its expectation. - -## Verification actually performed - -`PYTHONPATH=/src uv run pytest tests/unit -q` → **2119 passed, 52 -skipped, 3 xfailed**. `pre-commit run --all-files` → all hooks pass. - -New tests: 1 base-server pacing, 6 openai-realtime (pacing both ways, delta-based -activity guard, truncation target/no-op/item tracking), 4 `TickResult`, 8 -tick-driven adapter (unpaced send, per-tick release, silent tick still sends a -full tick, played position, buffered ticks release without delay, quiet tick waits -out the grace, early return mid-grace, barge-in position, barge-in discard), -2 scheduler barge-in arming, 4 framework selection, 3 worker pacing. - -## Two design errors in the plan, found only by running it - -The unit tests all passed while the path was completely non-functional. Both -faults were invisible to them because both are about what happens across *many* -ticks against a real provider. - -### 1. Removing all pacing left nothing to wait for the provider - -First live run ended after 160ms with `reason: timeout` and an empty transcript. -`max_ticks = timeout * 1000 / TICK_DURATION_MS` ticks ran to exhaustion instantly: -the real-time adapter's 200ms per-tick floor was the only thing that had ever -given the assistant time to produce anything, and the plan removed it without -replacement. - -Added `QUIET_TICK_GRACE_S` (one tick, 200ms) and `_await_tick_of_audio()`: a tick -waits for a full tick of assistant audio, returning the moment it is available. -This is not pacing — it never delays audio that has already arrived, so a provider -generating faster than real time is still drained as fast as it produces and -caller compute still costs the conversation nothing. It only bounds how long "the -assistant has said nothing yet" takes to establish. The wait is driven by an -`asyncio.Event` set from a new `_on_inbound_audio()` hook on `RealtimeWSAdapter` -(a no-op there). - -### 2. "Nothing is emitted on a silent tick" was backwards - -Second live run reached a real turn — assistant greeted, caller replied at tick 23 -— and then died at the 40s liveness check with the assistant never answering. - -The plan asserts that emitting nothing freezes the assistant, and treats that as -the mechanism. Emitting nothing does freeze it, but it also starves the provider's -VAD of the trailing silence that *ends the caller's turn*, so no response is ever -generated. The freeze this plan is built on comes from the caller not calling -`run_tick` while it thinks; it does not require, and is actively broken by, a tick -that puts nothing on the wire. That is why the real-time adapter sends a full tick -every tick. - -`run_tick` now sends a full tick of audio — real or silence, perturbation applied -as on the real-time path — with no pacing sleeps and no minimum tick duration. -The plan's `test_nothing_is_emitted_on_a_silent_tick` was inverted accordingly, -and its two "no wait" tests were reframed: they now assert that ticks with audio -*already buffered* release without delay, plus new tests that a quiet tick waits -out the grace and that a tick returns early when audio arrives mid-grace. - -## Task 9 — steps 1-4 and 6 run; step 5 not run - -One ITSM record (record 1), `gpt-realtime-mini`, behaviors off, metrics enabled. -`--metrics=` was used only for the first diagnostic run. - -**Steps 1-2 (artifacts, tools, DB).** Tick-driven run completed, ended `goodbye`, -7 turns. Full artifact set produced and identical to the baseline's: -`audit_log.json`, `transcript.jsonl`, `audio_user.wav`, `audio_assistant.wav`, -`audio_user_clean.wav`, `audio_mixed.wav`, `framework_logs.jsonl`, -`pipecat_metrics.jsonl`, `initial_scenario_db.json`, `final_scenario_db.json`. -Audit log holds 2 `tool_call` / 2 `tool_response` entries -(`verify_employee_auth`, `attempt_account_unlock`), and the DB diff is exactly the -intended mutation: `active_directory.locked True -> False`, `lock_reason -too_many_attempts -> None`, plus the two session-auth fields. - -**Step 3 (latency).** Compared against the same record on the same framework with -`TICK_DRIVEN_FRAMEWORKS` temporarily emptied — a like-for-like real-time baseline, -which the ElevenLabs run could not provide: - -| | tick-driven | real-time | -|---|---|---| -| completed / end reason | yes / goodbye | yes / goodbye | -| `model_response` p50 | 765 ms | 780 ms | -| `model_response` mean (n) | 823 ms (4) | 1178 ms (2) | -| tools called | both | both | - -p50 is within 2%. The mean gap is small-n noise (4 samples vs 2), not distortion — -the measurement window (caller stops → first assistant byte) contains no caller -compute either way. - -**Step 4 (turn ordering).** Timestamps monotonic on both paths. The tick-driven -transcript alternates cleanly; the real-time baseline has two caller turns landing -before the assistant's reply block and answers "which system?" *before* the caller -says "Active Directory" — i.e. tick-driving visibly improved ordering, which is the -direction this plan predicts. The duplicated/concatenated assistant entries appear -**identically on the baseline**, so they are a pre-existing `openai_realtime` -transcript artifact, not something this plan introduced. - -**Step 6 (ElevenLabs regression).** Could not be completed: the ElevenLabs -simulator failed to connect to ElevenLabs' own service -(`EOFError: connection closed while reading HTTP status line`) on all three -attempts, before any of this plan's code ran. External credential/connectivity -issue, unrelated to these changes. The paced server branch was instead exercised -by the Step 3 baseline run, which used `paced_output=True` and completed normally -— so the pacing path is verified working, just not with the ElevenLabs caller. - -**Validation "failure" on every run, including the baseline and the ElevenLabs -attempt**, is `user_speech_fidelity` erroring with -`Unable to load vertex credentials from environment`. Environmental, identical -across all paths. `conversation_valid_end` and `user_behavioral_fidelity` both -scored 1.0 on the tick-driven run. - -**Step 5 (interruption fidelity) NOT run**, by decision. It needs Plan 2's -interruption path to be stable — still being fixed in the other worktree, whose -changelog lists defect 4b (the interrupt check preempting ordinary turn-taking) as -an open design question. Running it before 4b is settled would measure Plan 2's -turn-routing bug rather than this plan's clock. It remains the deciding evidence -for porting `gemini_live` and `grok_voice`; expected result is `slip_ms == 0` and -`dropped == false` tick-driven against non-zero slip and some drops real-time. From dc5bf032416e39b1a51b2e1104fa063745d7652c Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 20 Aug 2026 20:54:25 -0700 Subject: [PATCH 59/65] Accept paced_output on every assistant server worker.py always passes paced_output, but only OpenAIRealtimeAssistantServer took it (via **kwargs). The four servers with explicit __init__ signatures raised TypeError on construction, which is why running --framework pipecat failed and the argument had to be commented out at the call site. Add the parameter to pipecat, gemini_live, elevenlabs and smallest_hydra, and forward it to the base class. Also guard the silent-ignore case. Accepting paced_output=False and then pacing anyway would leave a tick-driven caller believing the assistant was unpaced, so AbstractAssistantServer now declares supports_unpaced_output and rejects a request it cannot honor. Only a server whose outbound relay throttle we own can drop it: openai_realtime sets it True (grok_voice inherits), the rest keep the False default. No current framework is affected, since each receives the value it can honor. --- src/eva/assistant/base_server.py | 14 +++++++++ src/eva/assistant/elevenlabs_server.py | 2 ++ src/eva/assistant/gemini_live_server.py | 2 ++ src/eva/assistant/openai_realtime_server.py | 2 ++ src/eva/assistant/pipecat_server.py | 4 +++ src/eva/assistant/smallest_hydra_server.py | 2 ++ .../unit/assistant/test_base_server_pacing.py | 30 +++++++++++++++++++ 7 files changed, 56 insertions(+) diff --git a/src/eva/assistant/base_server.py b/src/eva/assistant/base_server.py index 934b248b..fc6df45c 100644 --- a/src/eva/assistant/base_server.py +++ b/src/eva/assistant/base_server.py @@ -41,6 +41,15 @@ class AbstractAssistantServer(ABC): 5. Populate the AuditLog with conversation events """ + supports_unpaced_output: bool = False + """Whether this server can honor ``paced_output=False``. + + Only a server whose outbound relay throttle we own can drop it. A server whose + pacing lives inside a third-party runtime cannot, and must reject the request + rather than accept it and keep pacing, which would leave a tick-driven caller + believing the assistant was unpaced. + """ + def __init__( self, current_date_time: str, @@ -74,6 +83,11 @@ def __init__( self.current_date_time = current_date_time self.pipeline_config = pipeline_config self.language = language + if not paced_output and not self.supports_unpaced_output: + raise ValueError( + f"{type(self).__name__} cannot honor paced_output=False: its output pacing is not ours to remove. " + "Tick-driving this framework requires leaving pacing on." + ) self.paced_output = paced_output self.initial_message = get_initial_message(language) self.agent: AgentConfig = agent diff --git a/src/eva/assistant/elevenlabs_server.py b/src/eva/assistant/elevenlabs_server.py index 8d37e276..0634f4c3 100644 --- a/src/eva/assistant/elevenlabs_server.py +++ b/src/eva/assistant/elevenlabs_server.py @@ -134,6 +134,7 @@ def __init__( port: int, conversation_id: str, language: str = "en", + paced_output: bool = True, ): super().__init__( current_date_time=current_date_time, @@ -145,6 +146,7 @@ def __init__( port=port, conversation_id=conversation_id, language=language, + paced_output=paced_output, ) # Recording sample rate (ElevenLabs operates at 16 kHz) diff --git a/src/eva/assistant/gemini_live_server.py b/src/eva/assistant/gemini_live_server.py index 377386a9..0dafaeb1 100644 --- a/src/eva/assistant/gemini_live_server.py +++ b/src/eva/assistant/gemini_live_server.py @@ -166,6 +166,7 @@ def __init__( port: int, conversation_id: str, language: str = "en", + paced_output: bool = True, ): super().__init__( current_date_time=current_date_time, @@ -177,6 +178,7 @@ def __init__( port=port, conversation_id=conversation_id, language=language, + paced_output=paced_output, ) # Recording sample rate (Gemini outputs 24 kHz) diff --git a/src/eva/assistant/openai_realtime_server.py b/src/eva/assistant/openai_realtime_server.py index c299a575..5582b257 100644 --- a/src/eva/assistant/openai_realtime_server.py +++ b/src/eva/assistant/openai_realtime_server.py @@ -92,6 +92,8 @@ class OpenAIRealtimeAssistantServer(AbstractAssistantServer): pauses to think is not mistaken for a caller who stopped talking. """ + supports_unpaced_output = True + def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) diff --git a/src/eva/assistant/pipecat_server.py b/src/eva/assistant/pipecat_server.py index 1b52be12..a7a50ca1 100644 --- a/src/eva/assistant/pipecat_server.py +++ b/src/eva/assistant/pipecat_server.py @@ -99,6 +99,7 @@ def __init__( port: int, conversation_id: str, language: str = "en", + paced_output: bool = True, turn_end_fallback_time: int | None = None, ): """Initialize the assistant server. @@ -113,6 +114,8 @@ def __init__( port: Port to listen on conversation_id: Unique ID for this conversation language: BCP 47 language tag for STT/TTS services (e.g. 'en', 'fr', 'es-MX') + paced_output: Accepted for interface parity; must be True, since Pipecat's output + pacing lives in its own transport and is not ours to remove. turn_end_fallback_time: Seconds of user-turn silence after the assistant stops speaking before nudging it to retry. ``None`` disables the fallback. """ @@ -126,6 +129,7 @@ def __init__( port=port, conversation_id=conversation_id, language=language, + paced_output=paced_output, ) self.agentic_system = None # Will be set in _handle_session diff --git a/src/eva/assistant/smallest_hydra_server.py b/src/eva/assistant/smallest_hydra_server.py index 64a79891..966fcfe3 100644 --- a/src/eva/assistant/smallest_hydra_server.py +++ b/src/eva/assistant/smallest_hydra_server.py @@ -129,6 +129,7 @@ def __init__( port: int, conversation_id: str, language: str = "en", + paced_output: bool = True, ): super().__init__( current_date_time=current_date_time, @@ -140,6 +141,7 @@ def __init__( port=port, conversation_id=conversation_id, language=language, + paced_output=paced_output, ) self._audio_sample_rate = _RECORDING_SAMPLE_RATE diff --git a/tests/unit/assistant/test_base_server_pacing.py b/tests/unit/assistant/test_base_server_pacing.py index 6a793faa..54a2e783 100644 --- a/tests/unit/assistant/test_base_server_pacing.py +++ b/tests/unit/assistant/test_base_server_pacing.py @@ -8,3 +8,33 @@ def test_paced_output_defaults_to_true(): # silence heuristics, so the default must not change. signature = inspect.signature(AbstractAssistantServer.__init__) assert signature.parameters["paced_output"].default is True + + +def test_every_server_accepts_paced_output(): + # worker.py always passes paced_output; a server that omits it from its signature + # raises TypeError at construction (this is what broke the pipecat run). + import inspect + + from eva.assistant.elevenlabs_server import ElevenLabsAssistantServer + from eva.assistant.gemini_live_server import GeminiLiveAssistantServer + from eva.assistant.pipecat_server import PipecatAssistantServer + from eva.assistant.smallest_hydra_server import SmallestHydraAssistantServer + + for server in ( + PipecatAssistantServer, + GeminiLiveAssistantServer, + ElevenLabsAssistantServer, + SmallestHydraAssistantServer, + ): + params = inspect.signature(server.__init__).parameters + assert "paced_output" in params, f"{server.__name__} rejects paced_output" + + +def test_only_servers_owning_their_throttle_claim_unpaced_support(): + from eva.assistant.elevenlabs_server import ElevenLabsAssistantServer + from eva.assistant.openai_realtime_server import OpenAIRealtimeAssistantServer + from eva.assistant.pipecat_server import PipecatAssistantServer + + assert OpenAIRealtimeAssistantServer.supports_unpaced_output is True + assert PipecatAssistantServer.supports_unpaced_output is False + assert ElevenLabsAssistantServer.supports_unpaced_output is False From 238748b1a3428c42dbf1c237dc2727e81ebef8a0 Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 20 Aug 2026 20:55:21 -0700 Subject: [PATCH 60/65] Add TranscriptBuffer.heard_text for transcript use MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit current_text() appends "[CURRENTLY SPEAKING, INCOMPLETE]" because the listener check prompts are tuned around that marker — their few-shot examples all end in it, and it tells the check to judge only the complete sentences. That makes it the wrong accessor for anything that records what was said. Add a marker-free heard_text() alongside it so prompt input and transcript content stop sharing one accessor. --- src/eva/user_simulator/cascade/stt.py | 4 ++++ tests/unit/user_simulator/cascade/test_stt.py | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/src/eva/user_simulator/cascade/stt.py b/src/eva/user_simulator/cascade/stt.py index 50fb5de4..750136fe 100644 --- a/src/eva/user_simulator/cascade/stt.py +++ b/src/eva/user_simulator/cascade/stt.py @@ -30,6 +30,10 @@ def current_text(self) -> str: return self.committed return f"{self.committed} {self.in_flight} [CURRENTLY SPEAKING, INCOMPLETE]".strip() + def heard_text(self) -> str: + """Return everything heard so far without the in-progress marker, for transcript use.""" + return f"{self.committed} {self.in_flight}".strip() + def take_committed(self) -> str: """Return and clear the committed text.""" text = self.committed diff --git a/tests/unit/user_simulator/cascade/test_stt.py b/tests/unit/user_simulator/cascade/test_stt.py index facbd2d8..357b0fa2 100644 --- a/tests/unit/user_simulator/cascade/test_stt.py +++ b/tests/unit/user_simulator/cascade/test_stt.py @@ -60,3 +60,11 @@ def test_current_text_does_not_consume_the_committed_text(): buffer.current_text() assert buffer.take_committed() == "hello" + + +def test_heard_text_never_carries_the_prompt_marker_into_the_transcript(): + buffer = TranscriptBuffer() + buffer.commit("I found your order.") + buffer.apply_partial("It includes a keyboa") + + assert buffer.heard_text() == "I found your order. It includes a keyboa" From 1f761dcbe994eb85c168d4ef4956a585da642e5a Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 20 Aug 2026 20:55:21 -0700 Subject: [PATCH 61/65] Report what each listener check did, not just its answer The interrupt and backchannel checks collapsed to a bool, so a check that ran and answered NO was indistinguishable from one that never ran or one whose call raised. That gap made a run with zero interruptions unexplainable from its artifacts. Have _check return a CheckTrace carrying whether it ran, the raw reply, its latency, and any error, and hang both traces off ListenerVerdict. The verdict fields are unchanged, so callers reading should_interrupt/should_backchannel behave exactly as before. --- src/eva/user_simulator/cascade/decisions.py | 37 ++++++++++++++++----- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/src/eva/user_simulator/cascade/decisions.py b/src/eva/user_simulator/cascade/decisions.py index 32165057..6afedf3d 100644 --- a/src/eva/user_simulator/cascade/decisions.py +++ b/src/eva/user_simulator/cascade/decisions.py @@ -3,7 +3,8 @@ from __future__ import annotations import asyncio -from dataclasses import dataclass +import time +from dataclasses import dataclass, field from typing import Protocol from eva.utils.logging import get_logger @@ -19,12 +20,24 @@ async def decide(self, prompt: str) -> str: ... +@dataclass(frozen=True) +class CheckTrace: + """What one YES/NO check actually did, for the diagnostic trace.""" + + ran: bool + raw: str = "" + latency_ms: int = 0 + error: str = "" + + @dataclass(frozen=True) class ListenerVerdict: """Outcome of one check tick.""" should_interrupt: bool should_backchannel: bool + interrupt_trace: CheckTrace = field(default_factory=lambda: CheckTrace(ran=False)) + backchannel_trace: CheckTrace = field(default_factory=lambda: CheckTrace(ran=False)) def parse_yes_no(raw: str) -> bool: @@ -51,24 +64,32 @@ async def evaluate( self, conversation_history: str, *, allow_interrupt: bool, allow_backchannel: bool ) -> ListenerVerdict: """Run whichever checks are enabled. Interrupt wins ties (tau: streaming.py:2549).""" - interrupt, backchannel = await asyncio.gather( + interrupt_trace, backchannel_trace = await asyncio.gather( self._check(self._interrupt_prompt, conversation_history, enabled=allow_interrupt), self._check(self._backchannel_prompt, conversation_history, enabled=allow_backchannel), ) - return ListenerVerdict(should_interrupt=interrupt, should_backchannel=backchannel and not interrupt) + interrupt = interrupt_trace.ran and parse_yes_no(interrupt_trace.raw) + backchannel = backchannel_trace.ran and parse_yes_no(backchannel_trace.raw) + return ListenerVerdict( + should_interrupt=interrupt, + should_backchannel=backchannel and not interrupt, + interrupt_trace=interrupt_trace, + backchannel_trace=backchannel_trace, + ) - async def _check(self, template: str, conversation_history: str, *, enabled: bool) -> bool: - """Ask the model one YES/NO question, returning False on anything unexpected. + async def _check(self, template: str, conversation_history: str, *, enabled: bool) -> CheckTrace: + """Ask the model one YES/NO question, reporting what happened rather than just the answer. Both templates are filled with the same arguments; `str.format` ignores the ones a given prompt does not use, so the backchannel prompt needs no goal slot. """ if not enabled: - return False + return CheckTrace(ran=False) + started = time.monotonic() try: filled = template.format(conversation_history=conversation_history, user_goal=self._user_goal) reply = await self._llm.decide(filled) except Exception as exc: logger.warning(f"Listener check failed, defaulting to no action: {exc}") - return False - return parse_yes_no(reply) + return CheckTrace(ran=True, latency_ms=int((time.monotonic() - started) * 1000), error=str(exc)) + return CheckTrace(ran=True, raw=reply, latency_ms=int((time.monotonic() - started) * 1000)) From d80618f1116467cc9b3d0fa15c82a9480e52dedd Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 20 Aug 2026 20:55:31 -0700 Subject: [PATCH 62/65] Add a decision trace for the cascade caller user_simulator_events.jsonl only records actions the caller took, so a listener check that declined leaves no trace at all. A run with no interruptions looks identical to one where the interrupt path was never reachable. DecisionLog writes user_simulator_decisions.jsonl, one JSON object per row. Rows are written and flushed as they happen rather than buffered until the end: a conversation that dies mid-run is exactly the one worth having a trace for, and the event log's save-at-exit is why failed attempts currently leave no diagnostics. The file is only created on first write, so a run that traces nothing leaves none behind. --- .../user_simulator/cascade/decision_log.py | 58 ++++++++++++++ .../cascade/test_decision_log.py | 78 +++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 src/eva/user_simulator/cascade/decision_log.py create mode 100644 tests/unit/user_simulator/cascade/test_decision_log.py diff --git a/src/eva/user_simulator/cascade/decision_log.py b/src/eva/user_simulator/cascade/decision_log.py new file mode 100644 index 00000000..f9cc29a7 --- /dev/null +++ b/src/eva/user_simulator/cascade/decision_log.py @@ -0,0 +1,58 @@ +"""Per-tick diagnostic trace for the cascade caller's out-of-turn decisions.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, TextIO + +from eva.utils.logging import get_logger + +logger = get_logger(__name__) + + +class DecisionLog: + """Records every listener check, including the ones that declined or never ran. + + `user_simulator_events.jsonl` only carries actions the caller took, so a check that + ran and said NO is indistinguishable there from a check that never fired. This trace + separates the two, which is the difference between "the model does not want to + interrupt" and "the interrupt path is unreachable". + + Rows are written and flushed as they happen rather than buffered to the end: a + conversation that dies mid-run is exactly the one worth having a trace for, and the + event log's save-at-exit is why failed attempts currently leave no diagnostics at all. + """ + + def __init__(self, output_path: Path) -> None: + self.output_path = output_path + self._handle: TextIO | None = None + self._counts: dict[str, int] = {} + + def log(self, kind: str, **fields: Any) -> None: + """Append one trace row of the given kind and flush it to disk.""" + self._counts[kind] = self._counts.get(kind, 0) + 1 + try: + handle = self._open() + handle.write(json.dumps({"kind": kind, **fields}, ensure_ascii=False) + "\n") + handle.flush() + except Exception as exc: + logger.warning(f"Could not write caller decision trace: {exc}") + + def _open(self) -> TextIO: + """Open the trace on first use, so a run that traces nothing leaves no file.""" + if self._handle is None: + self.output_path.parent.mkdir(parents=True, exist_ok=True) + self._handle = open(self.output_path, "w") + return self._handle + + def save(self) -> None: + """Close the trace. Safe to call more than once.""" + if self._handle is not None: + self._handle.close() + self._handle = None + logger.info(f"Caller decision trace written to {self.output_path}") + + def summary(self) -> dict[str, int]: + """Count rows by kind, for a one-line end-of-run report.""" + return dict(self._counts) diff --git a/tests/unit/user_simulator/cascade/test_decision_log.py b/tests/unit/user_simulator/cascade/test_decision_log.py new file mode 100644 index 00000000..7ca3311b --- /dev/null +++ b/tests/unit/user_simulator/cascade/test_decision_log.py @@ -0,0 +1,78 @@ +"""Tests for the caller's out-of-turn decision trace.""" + +import json + +from eva.user_simulator.cascade.decision_log import DecisionLog + + +def test_rows_are_written_one_json_object_per_line(tmp_path): + log = DecisionLog(tmp_path / "trace.jsonl") + log.log("tick", tick=1, has_assistant_speech=True) + log.log("listener_check", tick=10, should_interrupt=False) + + log.save() + + rows = [json.loads(line) for line in (tmp_path / "trace.jsonl").read_text().splitlines()] + assert [r["kind"] for r in rows] == ["tick", "listener_check"] + assert rows[0]["has_assistant_speech"] is True + + +def test_no_file_is_written_when_nothing_was_traced(tmp_path): + DecisionLog(tmp_path / "trace.jsonl").save() + + assert not (tmp_path / "trace.jsonl").exists() + + +def test_rows_are_readable_before_save_so_a_crashed_run_still_has_a_trace(tmp_path): + log = DecisionLog(tmp_path / "trace.jsonl") + log.log("tick", tick=1) + + # No save() call: this is the killed-mid-run case. + assert json.loads((tmp_path / "trace.jsonl").read_text().splitlines()[0])["tick"] == 1 + + +def test_save_is_idempotent(tmp_path): + log = DecisionLog(tmp_path / "trace.jsonl") + log.log("tick", tick=1) + + log.save() + log.save() + + +def test_summary_counts_rows_by_kind(tmp_path): + log = DecisionLog(tmp_path / "trace.jsonl") + for _ in range(3): + log.log("tick") + log.log("listener_check") + + assert log.summary() == {"tick": 3, "listener_check": 1} + + +async def test_a_declined_check_is_still_recorded_with_its_raw_reply(): + # The whole point: a NO must be distinguishable from a check that never ran. + from eva.user_simulator.cascade.decisions import ListenerDecisions + from tests.unit.user_simulator.cascade.test_decisions import FakeLLM + + decisions = ListenerDecisions( + FakeLLM(["NO", "NO"]), interrupt_prompt="{conversation_history}", backchannel_prompt="{conversation_history}" + ) + + verdict = await decisions.evaluate("agent talking", allow_interrupt=True, allow_backchannel=True) + + assert verdict.should_interrupt is False + assert verdict.interrupt_trace.ran is True + assert verdict.interrupt_trace.raw == "NO" + + +async def test_a_check_that_was_not_allowed_reports_that_it_never_ran(): + from eva.user_simulator.cascade.decisions import ListenerDecisions + from tests.unit.user_simulator.cascade.test_decisions import FakeLLM + + decisions = ListenerDecisions( + FakeLLM([]), interrupt_prompt="{conversation_history}", backchannel_prompt="{conversation_history}" + ) + + verdict = await decisions.evaluate("agent talking", allow_interrupt=False, allow_backchannel=False) + + assert verdict.interrupt_trace.ran is False + assert verdict.interrupt_trace.raw == "" From 1a6a2a6e0a9ec007cd953a520215145544a1d33e Mon Sep 17 00:00:00 2001 From: tara-servicenow Date: Thu, 20 Aug 2026 20:56:20 -0700 Subject: [PATCH 63/65] Remove self-correction, retime barge-in staleness, trace caller decisions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three changes to the cascade caller that cannot be separated into their own commits: they overlap in simulator.py, including the import block and the whole of _play_interruption, so splitting them at line level would leave commits that do not import. Remove self-correction entirely. Gone are the enable_self_correction config field, SELF_CORRECTION_DELAY_MS and _RATE, should_fire_self_correction, correction_rng, _maybe_arm_self_correction, _fire_self_correction, _drop_stale_correction, the armed-correction state, the tick-loop firing branch, and the cascade_self_correction prompt. --user-simulator.enable-self-correction is now a hard CLI error rather than a silently ignored flag. extract_correction survives as extract_optional_line: speculative generation uses it to parse its candidate, so deleting it would have broken that path. Its old name described only the removed feature. Replace the barge-in slip budget with a staleness test about the assistant. MAX_INTERRUPT_SLIP_MS and enforces_slip_budget are gone; should_drop_interrupt now asks whether the assistant is still speaking and still in the same turn. A line is a real interruption whenever the assistant is mid-utterance, however long generation took, and the 1500ms cap was discarding well-placed barge-ins on the real-time path — a live pipecat run had one die at slip_ms 1798 with the assistant still talking. "Still speaking" alone is not enough, though: over a couple of seconds one assistant turn can end and the next begin, landing the line as a non-sequitur against speech the caller never heard. A new _assistant_turn_index, captured at decision time, rules that out. slip_ms is still measured and traced, since it says whether speculative generation is earning its keep, but it no longer gates anything. Write the decision trace. Every tick records its speech state, including the audio RMS — has_assistant_speech is true for any non-zero bytes, silence included, so a transport that pads reads as continuous speech and this makes that measurable. Every listener check records its inputs and verdict, every speculative candidate records whether it was produced or declined and why, and every barge-in records its outcome with both turn indices. Also stop leaking the "[CURRENTLY SPEAKING, INCOMPLETE]" prompt marker into conversation history: _play_interruption now banks heard_text() instead of current_text(). The marker is prompt scaffolding for the checks, not something the assistant said. --- configs/prompts/simulation.yaml | 29 --- src/eva/models/config.py | 3 - src/eva/user_simulator/cascade/constants.py | 9 - src/eva/user_simulator/cascade/simulator.py | 212 +++++++-------- tests/unit/models/test_config_models.py | 1 - .../user_simulator/cascade/test_constants.py | 7 - .../user_simulator/cascade/test_prompt.py | 14 - .../user_simulator/cascade/test_simulator.py | 244 +++++++----------- 8 files changed, 200 insertions(+), 319 deletions(-) diff --git a/configs/prompts/simulation.yaml b/configs/prompts/simulation.yaml index 85965f2a..16a0ac50 100644 --- a/configs/prompts/simulation.yaml +++ b/configs/prompts/simulation.yaml @@ -604,35 +604,6 @@ user_simulator: Respond with ONLY "YES" or "NO". - cascade_self_correction: | - You are about to say this line to the agent: - - - {utterance} - - - On this turn only, you will misspeak first and then correct yourself, the way - people do on real phone calls. The line above is the CORRECT one — it is what - you will say as the correction. Your job here is to write the SLIP that comes - just before it. - - Write the version of that line that you say by mistake. - - Rules, all of which matter: - - It must state a decision that is WRONG but plausibly close to the intended - line. A different day, a different one of two options, a slightly wrong - number. Never something absurd, and never a different topic. - - Change exactly one detail. Everything else stays as it was. - - It must stand on its own as a natural thing to say. Do not hedge, do not - signal that it is wrong, and do not correct yourself inside it — the - self_correction is delivered separately, a moment later. - - Never contradict or drop a fact the agent needs; the correction that follows - is what satisfies your must_have_criteria, so the slip must be safely - reversible by it. - - Reply with ONLY the spoken line, nothing else. No quotes, no labels. - - If the intended line contains no detail that could plausibly be misspoken - (a greeting, a thank-you, a yes/no acknowledgement), reply with exactly NONE. - cascade_next_interruption: | You just said this line to the agent: diff --git a/src/eva/models/config.py b/src/eva/models/config.py index 592dc97a..e34bc8d7 100644 --- a/src/eva/models/config.py +++ b/src/eva/models/config.py @@ -513,9 +513,6 @@ class CascadeSimulatorConfig(BaseModel): enable_backchannel: bool = Field(False, description="Caller emits continuers while the assistant speaks.") enable_interruptions: bool = Field(False, description="Caller may barge in reacting to the assistant mid-turn.") - enable_self_correction: bool = Field( - False, description="Caller may reverse its own prior statement, pre-authored and fired on a timer." - ) speculative_generation: bool = Field( False, description="Pre-render a candidate interruption on the turn call, gated by a relevance check before firing.", diff --git a/src/eva/user_simulator/cascade/constants.py b/src/eva/user_simulator/cascade/constants.py index 4a531cdd..1186f815 100644 --- a/src/eva/user_simulator/cascade/constants.py +++ b/src/eva/user_simulator/cascade/constants.py @@ -38,15 +38,6 @@ LISTENER_CHECK_INTERVAL_MS = 2000 """How often the interrupt and backchannel checks run while the assistant speaks.""" -MAX_INTERRUPT_SLIP_MS = 1500 -"""Drop a reactive barge-in whose audio arrived this far past its intended tick.""" - -SELF_CORRECTION_DELAY_MS = 1200 -"""How long after the assistant starts replying to play a pre-authored correction.""" - -SELF_CORRECTION_RATE = 0.15 -"""Fraction of caller turns generated with a self-correction attached.""" - BACKCHANNEL_PHRASES = ["uh-huh", "mm-hmm"] """Fixed continuer vocabulary (tau: voice_config.py:126). Pre-rendered at init.""" diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py index 29189222..0f5c03cd 100644 --- a/src/eva/user_simulator/cascade/simulator.py +++ b/src/eva/user_simulator/cascade/simulator.py @@ -2,10 +2,9 @@ from __future__ import annotations -import random +import audioop import re import time -import zlib from pathlib import Path import websockets @@ -21,14 +20,12 @@ BARGE_IN_OPENERS, CALLER_SAMPLE_RATE, INACTIVITY_TIMEOUT_MS, - MAX_INTERRUPT_SLIP_MS, - SELF_CORRECTION_DELAY_MS, - SELF_CORRECTION_RATE, TICK_DURATION_MS, TRANSCRIPT_WAIT_MS, WAIT_TO_RESPOND_OTHER_MS, ms_to_ticks, ) +from eva.user_simulator.cascade.decision_log import DecisionLog from eva.user_simulator.cascade.decisions import ListenerDecisions, parse_yes_no from eva.user_simulator.cascade.phrase_cache import PhraseCache from eva.user_simulator.cascade.scheduler import TickScheduler @@ -79,25 +76,17 @@ def extract_turn(message: object) -> tuple[str, bool]: return parse_turn_response(content), end_call -def extract_correction(message: object) -> str: - """Read the self-correction line from its own dedicated call. +def extract_optional_line(message: object) -> str: + """Read a bare spoken line from a dedicated call, empty when the model declined. - It is a bare spoken line, not a JSON field: the caller's turn call keeps the - plain-text contract Plan 1 settled on, since demanding JSON there suppressed - the end_call tool call entirely. + A bare line rather than a JSON field: demanding JSON on a caller call suppressed + the end_call tool call entirely (Plan 1). """ content = message if isinstance(message, str) else (getattr(message, "content", None) or "") line = _FENCE.sub("", content).strip() return "" if line.upper() == "NONE" else line -def should_fire_self_correction(*, ticks_since_assistant_started: int, assistant_speaking: bool) -> bool: - """Whether an armed correction should play now.""" - if not assistant_speaking: - return False - return ticks_since_assistant_started >= ms_to_ticks(SELF_CORRECTION_DELAY_MS) - - def interrupt_slip_ms(*, elapsed_s: float) -> int: """How far past its intended moment a barge-in actually landed. @@ -150,19 +139,15 @@ def is_new_assistant_turn(*, ticks_silent_before: int) -> bool: return ticks_silent_before >= ms_to_ticks(WAIT_TO_RESPOND_OTHER_MS) -def correction_rng(conversation_id: str) -> random.Random: - """Seed the self-correction gate per conversation, reproducibly but not identically. +def should_drop_interrupt(*, assistant_still_speaking: bool, same_assistant_turn: bool) -> bool: + """Whether a barge-in has gone stale and should be abandoned. - Seeding every conversation with 0 made the gate unreachable: `Random(0)` first - falls below SELF_CORRECTION_RATE on draw 26, while a conversation runs ~7 turns, - so no conversation ever armed a correction. + Staleness is a fact about the assistant, not about how long generation took: a line is + still a real interruption whenever the assistant is mid-utterance, however many wall-clock + seconds elapsed. Both conditions are needed — "still speaking" alone is satisfied by a + *later* turn, which would land the line as a non-sequitur against speech it never heard. """ - return random.Random(zlib.crc32(conversation_id.encode())) - - -def should_drop_interrupt(*, slip_ms: int, assistant_still_speaking: bool) -> bool: - """Whether a barge-in has gone stale and should be abandoned.""" - return slip_ms > MAX_INTERRUPT_SLIP_MS or not assistant_still_speaking + return not (assistant_still_speaking and same_assistant_turn) async def candidate_is_relevant(llm, *, candidate: str, heard: str) -> bool: @@ -227,6 +212,7 @@ class CascadeUserSimulator(AbstractUserSimulator): _ticks_assistant_silent = 0 _ticks_since_assistant_started = 0 _may_interrupt_this_turn = False + _assistant_turn_index = 0 def __init__( self, @@ -266,11 +252,9 @@ def __init__( self._decision_client = _DecisionClient(LiteLLMClient(model=simulator_config.decision_llm)) self._phrase_cache: PhraseCache | None = None self._decisions: ListenerDecisions | None = None - self._rng = correction_rng(self._record_id or "cascade") - self._armed_correction: bytes = b"" - self._armed_correction_text = "" self._candidate_text = "" self._candidate_audio = b"" + self._decision_log = DecisionLog(self.output_dir / "user_simulator_decisions.jsonl") async def run_conversation(self) -> str: """Run the tick loop until the call ends, and return the end reason.""" @@ -283,6 +267,8 @@ async def run_conversation(self) -> str: finally: self._save_clean_user_audio(CALLER_SAMPLE_RATE) self.event_logger.save() + self._decision_log.save() + logger.info(f"Caller decision trace: {self._decision_log.summary()}") return self._end_reason async def _run(self) -> None: @@ -335,16 +321,21 @@ async def _run(self) -> None: self._ticks_since_assistant_started = 0 # One roll per assistant turn, so a turn carries at most one barge-in. self._may_interrupt_this_turn = self._config.enable_interruptions + self._assistant_turn_index += 1 + self._decision_log.log( + "assistant_turn_start", + tick=scheduler.tick, + turn_index=self._assistant_turn_index, + ticks_silent_before=silent_before, + armed_interrupt=self._may_interrupt_this_turn, + ) self._ticks_since_assistant_started += 1 - if self._armed_correction and should_fire_self_correction( - ticks_since_assistant_started=self._ticks_since_assistant_started, - assistant_speaking=True, - ): - self._fire_self_correction(scheduler) - continue - if scheduler.is_check_tick() and await self._run_checks(scheduler): + is_check = scheduler.is_check_tick() + self._log_tick_state(scheduler, result, is_check_tick=is_check) + if is_check and await self._run_checks(scheduler): break continue + self._log_tick_state(scheduler, result, is_check_tick=False) if scheduler.caller_is_speaking or not scheduler.may_take_turn(): continue heard, waiting = self._collect_heard_text(scheduler) @@ -388,11 +379,29 @@ async def _run_checks(self, scheduler: TickScheduler) -> bool: """Run the listener-reaction checks and act on the verdict. True means hang up.""" if self._decisions is None or self._phrase_cache is None: return False + history = self._stt.buffer.current_text() verdict = await self._decisions.evaluate( - self._stt.buffer.current_text(), + history, allow_interrupt=self._may_interrupt_this_turn, allow_backchannel=self._config.enable_backchannel, ) + self._decision_log.log( + "listener_check", + tick=scheduler.tick, + allow_interrupt=self._may_interrupt_this_turn, + allow_backchannel=self._config.enable_backchannel, + heard_chars=len(history), + heard=history, + interrupt_ran=verdict.interrupt_trace.ran, + interrupt_raw=verdict.interrupt_trace.raw, + interrupt_latency_ms=verdict.interrupt_trace.latency_ms, + interrupt_error=verdict.interrupt_trace.error, + backchannel_ran=verdict.backchannel_trace.ran, + backchannel_raw=verdict.backchannel_trace.raw, + backchannel_error=verdict.backchannel_trace.error, + should_interrupt=verdict.should_interrupt, + should_backchannel=verdict.should_backchannel, + ) if verdict.should_interrupt: self._may_interrupt_this_turn = False return await self._play_interruption(scheduler) @@ -417,6 +426,7 @@ async def _play_interruption(self, scheduler: TickScheduler) -> bool: if self._phrase_cache is None: return False intended_tick = scheduler.tick + intended_turn = self._assistant_turn_index started_at = time.monotonic() opener = self._phrase_cache.choose(BARGE_IN_OPENERS) opener_audio = self._phrase_cache.get(opener) @@ -424,9 +434,11 @@ async def _play_interruption(self, scheduler: TickScheduler) -> bool: if self._config.speculative_generation and self._candidate_audio: candidate, audio = self._candidate_text, self._candidate_audio self._candidate_text, self._candidate_audio = "", b"" - if await candidate_is_relevant( + relevant = await candidate_is_relevant( self._decision_client, candidate=candidate, heard=self._stt.buffer.current_text() - ): + ) + self._decision_log.log("relevance_gate", tick=intended_tick, candidate=candidate, relevant=relevant) + if relevant: # Tell the adapter the next tick that reaches the wire cuts the # assistant off, so a tick-driven transport can truncate the audio # the caller never heard. Ignored on the real-time path. @@ -449,12 +461,16 @@ async def _play_interruption(self, scheduler: TickScheduler) -> bool: "dropped": False, }, ) + self._decision_log.log( + "interruption", tick=intended_tick, outcome="spoken", speculative=True, text=candidate + ) return False self.event_logger.log_event("interruption_candidate_rejected", {"text": candidate}) + self._decision_log.log("interruption", tick=intended_tick, outcome="candidate_rejected", text=candidate) # Consumed, not peeked: leaving it in the buffer would re-append the same # assistant prefix at the next ordinary turn and duplicate it in the history. - heard = self._stt.buffer.current_text() + heard = self._stt.buffer.heard_text() self._stt.buffer.take_committed() self._stt.buffer.in_flight = "" if heard: @@ -467,7 +483,8 @@ async def _play_interruption(self, scheduler: TickScheduler) -> bool: # A hang-up is never stale: the caller has decided the call is over, and # dropping it here is what left conversations looping until the timeout. dropped = not end_call and should_drop_interrupt( - slip_ms=slip, assistant_still_speaking=scheduler.assistant_is_speaking + assistant_still_speaking=scheduler.assistant_is_speaking, + same_assistant_turn=self._assistant_turn_index == intended_turn, ) self.event_logger.log_event( "interruption", @@ -481,6 +498,16 @@ async def _play_interruption(self, scheduler: TickScheduler) -> bool: "end_call": end_call, }, ) + self._decision_log.log( + "interruption", + tick=intended_tick, + outcome="dropped" if dropped else ("end_call" if end_call else "spoken"), + text=utterance, + slip_ms=slip, + assistant_still_speaking=scheduler.assistant_is_speaking, + intended_turn=intended_turn, + actual_turn=self._assistant_turn_index, + ) # Nothing has reached the wire yet, so a stale line or a hang-up costs no audio. if dropped: return False @@ -542,6 +569,29 @@ def _log_audio_boundaries( elif not result.has_assistant_speech and assistant_was_speaking: self.event_logger.log_audio_end("assistant", seconds) + def _log_tick_state(self, scheduler: TickScheduler, result: TickResult, *, is_check_tick: bool) -> None: + """Trace one tick's speech state, so a check that never ran can be traced to its gate. + + `rms` is why this exists: `has_assistant_speech` is true for any non-zero bytes, + digital silence included, so a transport that pads with silence reads as continuous + speech. Recording both lets that be measured rather than inferred. + """ + raw = result.assistant_audio[: result.assistant_audio_raw_bytes] + self._decision_log.log( + "tick", + tick=scheduler.tick, + has_assistant_speech=result.has_assistant_speech, + raw_bytes=result.assistant_audio_raw_bytes, + rms=audioop.rms(raw, 2) if len(raw) >= 2 else 0, + caller_is_speaking=scheduler.caller_is_speaking, + caller_spoke_this_tick=scheduler.caller_spoke_this_tick, + ticks_assistant_silent=self._ticks_assistant_silent, + ticks_since_assistant_started=self._ticks_since_assistant_started, + may_interrupt_this_turn=self._may_interrupt_this_turn, + is_check_tick=is_check_tick, + has_candidate=bool(self._candidate_audio), + ) + def _collect_heard_text(self, scheduler: TickScheduler) -> tuple[str, bool]: """Return what the assistant said and whether to keep waiting for it. @@ -579,13 +629,9 @@ async def _take_turn(self, scheduler: TickScheduler, heard: str) -> bool: self._history.append({"role": "assistant", "content": heard}) self._on_assistant_speaks(heard) - self._drop_stale_correction() message, _stats = await self._llm.complete(messages=self._messages(), tools=[END_CALL_TOOL]) utterance, end_call = extract_turn(message) - if utterance and not end_call: - utterance = await self._maybe_arm_self_correction(utterance) - if utterance: self._history.append({"role": "user", "content": utterance}) self._on_user_speaks(utterance) @@ -604,19 +650,6 @@ async def _take_turn(self, scheduler: TickScheduler, heard: str) -> bool: await self._prerender_candidate(utterance) return False - def _fire_self_correction(self, scheduler: TickScheduler) -> None: - """Play the armed correction over the assistant's reply and clear the arming.""" - scheduler.enqueue_utterance(self._armed_correction) - self._record_audio("user_clean", self._armed_correction) - self._history.append({"role": "user", "content": self._armed_correction_text}) - self._on_user_speaks(self._armed_correction_text) - self.event_logger.log_event( - "self_correction", - {"text": self._armed_correction_text, "tick_index": scheduler.tick}, - ) - self._armed_correction = b"" - self._armed_correction_text = "" - async def _prerender_candidate(self, utterance: str) -> None: """Pre-generate and pre-render the line the caller would barge in with. @@ -628,6 +661,7 @@ async def _prerender_candidate(self, utterance: str) -> None: self._candidate_text, self._candidate_audio = "", b"" if not self._config.speculative_generation: return + started = time.monotonic() prompt = PromptManager().get_prompt("user_simulator.cascade_next_interruption", utterance=utterance) try: message, _stats = await self._llm.complete( @@ -635,57 +669,23 @@ async def _prerender_candidate(self, utterance: str) -> None: ) except Exception as exc: logger.warning(f"Speculative interruption generation failed: {exc}") + self._decision_log.log("candidate_generation", ok=False, error=str(exc)) return - candidate = extract_correction(message) + candidate = extract_optional_line(message) if not candidate: + raw = message if isinstance(message, str) else (getattr(message, "content", None) or "") + self._decision_log.log("candidate_generation", ok=False, declined=True, after=utterance, raw=raw[:400]) return self._candidate_text = candidate self._candidate_audio = await self._tts.synthesize(candidate, voice_id=self._voice_id) - - def _drop_stale_correction(self) -> None: - """Abandon an armed correction whose assistant turn never arrived. - - Carrying it into a later turn would read as a non-sequitur, since it refers - to an utterance that is now several exchanges back. - """ - if not self._armed_correction: - return - self.event_logger.log_event("self_correction_dropped", {"text": self._armed_correction_text}) - self._armed_correction = b"" - self._armed_correction_text = "" - - async def _maybe_arm_self_correction(self, utterance: str) -> str: - """Maybe misspeak: return a wrong variant to say now, arming `utterance` as the fix. - - The wrong-then-right ordering is the design, not a detail. The generated slip - is spoken first and the model's own goal-consistent line lands as the - correction, so the conversation's end state still satisfies must_have_criteria - by construction and this behavior cannot make a record unachievable. - - Asked as its own call rather than as an extra JSON field on the turn call: a - JSON contract there suppressed the end_call tool entirely (Plan 1), and this - way a failure degrades to an ordinary turn instead of a broken one. - """ - if not self._config.enable_self_correction or self._rng.random() >= SELF_CORRECTION_RATE: - return utterance - - prompt = PromptManager().get_prompt("user_simulator.cascade_self_correction", utterance=utterance) - try: - message, _stats = await self._llm.complete( - messages=[*self._messages(), {"role": "user", "content": prompt}] - ) - except Exception as exc: - logger.warning(f"Self-correction generation failed, speaking the turn unchanged: {exc}") - return utterance - - slip = extract_correction(message) - if not slip or slip == utterance: - return utterance - - self._armed_correction = await self._tts.synthesize(utterance, voice_id=self._voice_id) - self._armed_correction_text = utterance - self.event_logger.log_event("self_correction_armed", {"slip": slip, "correction": utterance}) - return slip + self._decision_log.log( + "candidate_generation", + ok=True, + after=utterance, + candidate=candidate, + audio_bytes=len(self._candidate_audio), + latency_ms=int((time.monotonic() - started) * 1000), + ) def _messages(self) -> list[dict[str, str]]: """Build the message list: the shared per-domain caller prompt plus flipped history. diff --git a/tests/unit/models/test_config_models.py b/tests/unit/models/test_config_models.py index 6c6df440..548fc309 100644 --- a/tests/unit/models/test_config_models.py +++ b/tests/unit/models/test_config_models.py @@ -1279,7 +1279,6 @@ def test_cascade_behaviors_default_off(): assert config.enable_backchannel is False assert config.enable_interruptions is False - assert config.enable_self_correction is False assert config.speculative_generation is False diff --git a/tests/unit/user_simulator/cascade/test_constants.py b/tests/unit/user_simulator/cascade/test_constants.py index e34196f7..5d7fb0dc 100644 --- a/tests/unit/user_simulator/cascade/test_constants.py +++ b/tests/unit/user_simulator/cascade/test_constants.py @@ -39,10 +39,3 @@ def test_fixed_vocabularies_are_non_empty(): assert BACKCHANNEL_PHRASES == ["uh-huh", "mm-hmm"] assert len(BARGE_IN_OPENERS) >= 2 - - -def test_self_correction_delay_is_shorter_than_the_check_interval(): - from eva.user_simulator.cascade.constants import LISTENER_CHECK_INTERVAL_MS, SELF_CORRECTION_DELAY_MS - - # The correction should land while the assistant is still on its first reply. - assert SELF_CORRECTION_DELAY_MS < LISTENER_CHECK_INTERVAL_MS diff --git a/tests/unit/user_simulator/cascade/test_prompt.py b/tests/unit/user_simulator/cascade/test_prompt.py index 8f651c2c..f3ad979b 100644 --- a/tests/unit/user_simulator/cascade/test_prompt.py +++ b/tests/unit/user_simulator/cascade/test_prompt.py @@ -41,20 +41,6 @@ def test_backchannel_decision_prompt_has_a_history_slot_and_frequency_guidance() assert "When in doubt, say NO" in prompt -def test_self_correction_prompt_states_the_wrong_then_right_ordering(): - prompt = PromptManager().get_template("user_simulator.cascade_self_correction") - - assert "self_correction" in prompt - assert "must_have_criteria" in prompt - - -def test_self_correction_prompt_never_mentions_ending_the_call(): - # It runs as its own call; if it could elicit a hang-up it would race the turn call. - prompt = PromptManager().get_template("user_simulator.cascade_self_correction") - - assert "end_call" not in prompt - - def test_interruption_decision_prompt_carries_the_user_goal(): prompt = PromptManager().get_prompt( "user_simulator.interruption_decision", diff --git a/tests/unit/user_simulator/cascade/test_simulator.py b/tests/unit/user_simulator/cascade/test_simulator.py index 5e3ee303..42ded4ba 100644 --- a/tests/unit/user_simulator/cascade/test_simulator.py +++ b/tests/unit/user_simulator/cascade/test_simulator.py @@ -1,6 +1,15 @@ from eva.user_simulator.cascade.simulator import CascadeUserSimulator, extract_turn, parse_turn_response +def _trace_sink(): + """DecisionLog that accumulates in memory and never writes.""" + from pathlib import Path + + from eva.user_simulator.cascade.decision_log import DecisionLog + + return DecisionLog(Path("unused-decision-trace.jsonl")) + + def test_parse_turn_response_returns_the_spoken_line_unchanged(): assert parse_turn_response("I need to reset my password.") == "I need to reset my password." @@ -52,6 +61,7 @@ def test_outbound_perturbation_reaches_the_adapter(): def _make_bare_simulator() -> CascadeUserSimulator: """Build a CascadeUserSimulator without running __init__, for pure _messages() testing.""" sim = object.__new__(CascadeUserSimulator) + sim._decision_log = _trace_sink() sim._build_prompt = lambda: "SYSTEM PROMPT" sim._history = [] return sim @@ -88,6 +98,7 @@ def _simulator_with_buffer(committed: str = "", in_flight: str = ""): from eva.user_simulator.cascade.stt import TranscriptBuffer sim = CascadeUserSimulator.__new__(CascadeUserSimulator) + sim._decision_log = _trace_sink() buffer = TranscriptBuffer() buffer.committed, buffer.in_flight = committed, in_flight sim._stt = type("_Stt", (), {"buffer": buffer})() @@ -135,6 +146,7 @@ def test_the_wait_counter_resets_after_a_successful_read(): def _boundary_simulator(): """Bare simulator exposing only what _log_audio_boundaries touches.""" sim = CascadeUserSimulator.__new__(CascadeUserSimulator) + sim._decision_log = _trace_sink() sim.event_logger = _FakeAudioEventLogger() return sim @@ -230,6 +242,7 @@ def test_inactivity_ends_the_call_after_the_shared_two_minute_threshold(): from eva.user_simulator.cascade.constants import INACTIVITY_TIMEOUT_MS, ms_to_ticks sim = CascadeUserSimulator.__new__(CascadeUserSimulator) + sim._decision_log = _trace_sink() sim._ticks_assistant_silent = 0 silent = _tick(False) for _ in range(ms_to_ticks(INACTIVITY_TIMEOUT_MS)): @@ -240,6 +253,7 @@ def test_inactivity_ends_the_call_after_the_shared_two_minute_threshold(): def test_assistant_speech_resets_the_inactivity_counter(): sim = CascadeUserSimulator.__new__(CascadeUserSimulator) + sim._decision_log = _trace_sink() sim._ticks_assistant_silent = 500 assert sim._assistant_is_inactive(_SilenceScheduler(), _tick(True)) is False @@ -255,6 +269,7 @@ class _NeverSpoke: assistant_has_spoken = False sim = CascadeUserSimulator.__new__(CascadeUserSimulator) + sim._decision_log = _trace_sink() sim._ticks_assistant_silent = 0 for _ in range(ms_to_ticks(INACTIVITY_TIMEOUT_MS) + 5): assert sim._assistant_is_inactive(_NeverSpoke(), _tick(False)) is False @@ -264,6 +279,7 @@ def test_tick_counters_exist_without_manual_setup(): # The unit tests build bare instances, so a counter initialised only inside __init__ # would still pass them and then AttributeError on the first live tick. sim = CascadeUserSimulator.__new__(CascadeUserSimulator) + sim._decision_log = _trace_sink() assert sim._ticks_assistant_silent == 0 assert sim._ticks_awaiting_transcript == 0 @@ -315,153 +331,66 @@ def test_verdict_with_no_action_queues_nothing(): assert verdict.should_backchannel is False -def test_interrupt_kept_when_slip_is_within_budget(): +def test_a_barge_in_is_kept_while_the_assistant_is_mid_utterance(): from eva.user_simulator.cascade.simulator import should_drop_interrupt - assert should_drop_interrupt(slip_ms=800, assistant_still_speaking=True) is False + assert should_drop_interrupt(assistant_still_speaking=True, same_assistant_turn=True) is False -def test_interrupt_dropped_when_slip_exceeds_budget(): +def test_a_slow_barge_in_is_no_longer_dropped_for_being_slow(): + # Wall-clock slip is not staleness: the assistant is still talking, so the line still lands. from eva.user_simulator.cascade.simulator import should_drop_interrupt - assert should_drop_interrupt(slip_ms=2000, assistant_still_speaking=True) is True + assert should_drop_interrupt(assistant_still_speaking=True, same_assistant_turn=True) is False -def test_interrupt_dropped_when_the_assistant_already_stopped(): +def test_a_barge_in_is_dropped_once_the_assistant_stopped(): # No longer an interruption — it would land as an ordinary reply. from eva.user_simulator.cascade.simulator import should_drop_interrupt - assert should_drop_interrupt(slip_ms=100, assistant_still_speaking=False) is True - - -def test_correction_fires_once_the_delay_has_elapsed(): - from eva.user_simulator.cascade.simulator import should_fire_self_correction + assert should_drop_interrupt(assistant_still_speaking=False, same_assistant_turn=True) is True - assert should_fire_self_correction(ticks_since_assistant_started=6, assistant_speaking=True) is True - - -def test_correction_does_not_fire_before_the_delay(): - from eva.user_simulator.cascade.simulator import should_fire_self_correction - - assert should_fire_self_correction(ticks_since_assistant_started=2, assistant_speaking=True) is False +def test_a_barge_in_is_dropped_when_the_assistant_moved_to_a_later_turn(): + # "Still speaking" is satisfied by a *different* turn, which would land the line as a + # non-sequitur against speech the caller never reacted to. + from eva.user_simulator.cascade.simulator import should_drop_interrupt -def test_correction_is_abandoned_if_the_assistant_never_replied(): - from eva.user_simulator.cascade.simulator import should_fire_self_correction + assert should_drop_interrupt(assistant_still_speaking=True, same_assistant_turn=False) is True - assert should_fire_self_correction(ticks_since_assistant_started=6, assistant_speaking=False) is False +def test_extract_optional_line_reads_a_plain_line(): + from eva.user_simulator.cascade.simulator import extract_optional_line -def test_extract_correction_reads_a_plain_line(): - from eva.user_simulator.cascade.simulator import extract_correction + assert extract_optional_line("I wanted Friday, not Thursday.") == "I wanted Friday, not Thursday." - assert extract_correction("Actually, wait — I said Thursday, I meant Friday.") == ( - "Actually, wait — I said Thursday, I meant Friday." - ) +def test_extract_optional_line_strips_a_code_fence(): + from eva.user_simulator.cascade.simulator import extract_optional_line -def test_extract_correction_strips_a_code_fence(): - from eva.user_simulator.cascade.simulator import extract_correction + assert extract_optional_line("```\nI meant Friday.\n```") == "I meant Friday." - assert extract_correction("```\nI meant Friday.\n```") == "I meant Friday." +def test_extract_optional_line_is_empty_for_an_empty_reply(): + from eva.user_simulator.cascade.simulator import extract_optional_line -def test_extract_correction_is_empty_for_an_empty_reply(): - from eva.user_simulator.cascade.simulator import extract_correction + assert extract_optional_line("") == "" - assert extract_correction("") == "" - -def test_extract_correction_reads_through_a_message_object(): - from eva.user_simulator.cascade.simulator import extract_correction +def test_extract_optional_line_reads_through_a_message_object(): + from eva.user_simulator.cascade.simulator import extract_optional_line class _Message: content = "I meant Friday." - tool_calls: list = [] - assert extract_correction(_Message()) == "I meant Friday." + assert extract_optional_line(_Message()) == "I meant Friday." -def test_extract_correction_rejects_a_refusal_style_non_answer(): +def test_extract_optional_line_rejects_a_refusal_style_non_answer(): # The prompt allows the model to decline by returning NONE. - from eva.user_simulator.cascade.simulator import extract_correction - - assert extract_correction("NONE") == "" - - -def _correcting_simulator(reply: str, *, rate_roll: float = 0.0): - """Bare simulator wired for _maybe_arm_self_correction only.""" - from eva.models.config import CascadeSimulatorConfig - - sim = CascadeUserSimulator.__new__(CascadeUserSimulator) - sim._config = CascadeSimulatorConfig(enable_self_correction=True) - sim._rng = type("_Rng", (), {"random": staticmethod(lambda: rate_roll)})() - sim._build_prompt = lambda: "SYSTEM PROMPT" - sim._history = [] - sim._voice_id = "voice-f" - sim.event_logger = _FakeEventLogger() - sim._armed_correction = b"" - sim._armed_correction_text = "" - - class _Llm: - async def complete(self, messages, tools=None): - return reply, {} - - class _Tts: - async def synthesize(self, text, *, voice_id): - return text.encode() - - sim._llm, sim._tts = _Llm(), _Tts() - return sim - - -async def test_the_slip_is_spoken_and_the_original_line_is_armed_as_the_correction(): - # Wrong-then-right: the goal-consistent line must be what lands last. - sim = _correcting_simulator("Book me Thursday.") - - spoken = await sim._maybe_arm_self_correction("Book me Friday.") - - assert spoken == "Book me Thursday." - assert sim._armed_correction_text == "Book me Friday." - - -async def test_no_correction_is_armed_when_the_rate_gate_declines(): - sim = _correcting_simulator("Book me Thursday.", rate_roll=0.99) - - spoken = await sim._maybe_arm_self_correction("Book me Friday.") - - assert spoken == "Book me Friday." - assert sim._armed_correction == b"" - - -async def test_a_none_reply_leaves_the_turn_unchanged(): - sim = _correcting_simulator("NONE") - - spoken = await sim._maybe_arm_self_correction("Thanks, goodbye.") - - assert spoken == "Thanks, goodbye." - assert sim._armed_correction == b"" + from eva.user_simulator.cascade.simulator import extract_optional_line - -async def test_a_failed_correction_call_degrades_to_an_ordinary_turn(): - sim = _correcting_simulator("unused") - - class _Failing: - async def complete(self, messages, tools=None): - raise RuntimeError("provider down") - - sim._llm = _Failing() - - assert await sim._maybe_arm_self_correction("Book me Friday.") == "Book me Friday." - - -async def test_self_correction_is_skipped_when_the_behavior_is_disabled(): - from eva.models.config import CascadeSimulatorConfig - - sim = _correcting_simulator("Book me Thursday.") - sim._config = CascadeSimulatorConfig() - - assert await sim._maybe_arm_self_correction("Book me Friday.") == "Book me Friday." + assert extract_optional_line("NONE") == "" async def test_relevance_gate_allows_a_still_relevant_candidate(): @@ -507,41 +436,6 @@ def test_slip_never_reports_negative_for_a_clock_hiccup(): assert interrupt_slip_ms(elapsed_s=-0.5) == 0 -def test_self_correction_rng_differs_per_conversation(): - # Seeding every conversation with 0 made the 15% gate unreachable: Random(0) - # first drops below 0.15 on draw 26, and conversations run ~7 turns. - from eva.user_simulator.cascade.simulator import correction_rng - - a = correction_rng("record-1") - b = correction_rng("record-2") - - assert [a.random() for _ in range(5)] != [b.random() for _ in range(5)] - - -def test_self_correction_rng_is_reproducible_for_the_same_conversation(): - from eva.user_simulator.cascade.simulator import correction_rng - - first = [correction_rng("record-7").random() for _ in range(3)] - again = [correction_rng("record-7").random() for _ in range(3)] - - assert first == again - - -def test_self_correction_gate_actually_opens_within_a_normal_conversation(): - # Across a realistic spread of records, the 15% rate must be reachable. - from eva.user_simulator.cascade.constants import SELF_CORRECTION_RATE - from eva.user_simulator.cascade.simulator import correction_rng - - turns_per_conversation = 7 - fired = 0 - for index in range(60): - rng = correction_rng(f"record-{index}") - if any(rng.random() < SELF_CORRECTION_RATE for _ in range(turns_per_conversation)): - fired += 1 - - assert fired > 20, f"only {fired}/60 conversations could ever self-correct" - - class _EndCallMessage: """LLM reply that hangs up via the tool and says nothing.""" @@ -559,12 +453,14 @@ def __init__(self) -> None: self.tool_calls = [call] -def _interrupting_simulator(message): +def _interrupting_simulator(message, framework="elevenlabs"): """Bare simulator wired for _play_interruption only.""" from eva.models.config import CascadeSimulatorConfig from eva.user_simulator.cascade.stt import TranscriptBuffer sim = CascadeUserSimulator.__new__(CascadeUserSimulator) + sim._decision_log = _trace_sink() + sim._framework = framework sim._config = CascadeSimulatorConfig(enable_interruptions=True) sim._history = [] sim._voice_id = "voice-f" @@ -642,6 +538,7 @@ async def test_only_one_interruption_fires_per_assistant_turn(): from eva.user_simulator.cascade.decisions import ListenerVerdict sim = CascadeUserSimulator.__new__(CascadeUserSimulator) + sim._decision_log = _trace_sink() sim._config = CascadeSimulatorConfig(enable_interruptions=True) sim._phrase_cache = StubCache() sim._may_interrupt_this_turn = True @@ -689,6 +586,7 @@ def test_inactivity_measures_contiguous_silence_not_cumulative(): from eva.user_simulator.cascade.constants import INACTIVITY_TIMEOUT_MS, ms_to_ticks sim = CascadeUserSimulator.__new__(CascadeUserSimulator) + sim._decision_log = _trace_sink() sim._ticks_assistant_silent = 0 limit = ms_to_ticks(INACTIVITY_TIMEOUT_MS) @@ -762,3 +660,49 @@ def test_unported_frameworks_default_to_the_real_time_adapter(): from eva.user_simulator.cascade.simulator import adapter_class_for_framework assert adapter_class_for_framework("gemini_live") is RealtimeWSAdapter + + +async def test_the_incomplete_marker_never_reaches_the_conversation_history(): + sim = _interrupting_simulator("Active Directory.") + sim._stt.buffer.apply_partial("and I still nee") + + await sim._play_interruption(_InterruptScheduler()) + + heard = [m["content"] for m in sim._history if m["role"] == "assistant"] + assert heard == ["Your account is unlocked. and I still nee"] + + +async def test_a_slow_barge_in_survives_on_every_transport(): + # Slip is now reported, not enforced: the assistant is mid-utterance, so the line lands. + from eva.user_simulator.cascade import simulator as module + + for framework in ("openai_realtime", "pipecat"): + sim = _interrupting_simulator("Active Directory.", framework=framework) + scheduler = _InterruptScheduler() + + original = module.interrupt_slip_ms + module.interrupt_slip_ms = lambda *, elapsed_s: 2400 + try: + assert await sim._play_interruption(scheduler) is False + finally: + module.interrupt_slip_ms = original + + assert scheduler.queued != [], framework + + +async def test_a_barge_in_is_abandoned_when_a_new_assistant_turn_started_meanwhile(): + # The assistant finished the utterance we reacted to and began another one while the + # line was being generated; firing now would answer speech the caller never heard. + sim = _interrupting_simulator("Active Directory.", framework="pipecat") + scheduler = _InterruptScheduler() + + original_complete = sim._llm.complete + + async def _complete_then_new_turn(messages, tools=None): + sim._assistant_turn_index += 1 + return await original_complete(messages, tools) + + sim._llm.complete = _complete_then_new_turn + + assert await sim._play_interruption(scheduler) is False + assert scheduler.queued == [] From f5dea15f168768ff56bb673b90164ebb0e9b31ce Mon Sep 17 00:00:00 2001 From: Gabrielle Gauthier-Melancon Date: Fri, 21 Aug 2026 10:00:11 -0400 Subject: [PATCH 64/65] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ae86d0b6..462c6411 100644 --- a/README.md +++ b/README.md @@ -343,7 +343,7 @@ output// | **🎯 EVA-A · Accuracy** | **✨ EVA-X · Experience** | |---|---| | *Did the agent complete the task correctly?* | *Was the conversational experience high quality?* | -| **Task Completion** · Deterministic | **Turn Taking** · LLM Judge `BETA` | +| **Task Completion** · Deterministic | **Turn Taking** · Deterministic | | **Agent Speech Fidelity** · Audio LLM Judge `BETA` | **Conciseness** · LLM Judge | | **Faithfulness** · LLM Judge | **Conversation Progression** · LLM Judge | From 8467d0a47042e59cbe775f4f4d85c059e8cb230c Mon Sep 17 00:00:00 2001 From: "raghav.mehndiratta" Date: Tue, 25 Aug 2026 16:44:02 -0700 Subject: [PATCH 65/65] add multilingual barge in, interruption gate interruption calls on STT update to reduce usage save stalls as failed records instead of exceptions make phrase cache globally shared, reduce generation cost --- configs/caller_phrases.yaml | 69 +++++++ configs/prompts/simulation.yaml | 21 +- scripts/add_culture_data.py | 68 +++++++ .../cascade/adapter/tick_driven.py | 16 +- src/eva/user_simulator/cascade/constants.py | 6 - .../user_simulator/cascade/phrase_cache.py | 56 ++++-- src/eva/user_simulator/cascade/phrases.py | 102 ++++++++++ src/eva/user_simulator/cascade/simulator.py | 64 +++++- src/eva/user_simulator/cascade/tick_result.py | 9 + src/eva/user_simulator/cascade/tts.py | 2 +- tests/unit/user_simulator/cascade/conftest.py | 17 ++ .../user_simulator/cascade/test_constants.py | 10 +- .../cascade/test_phrase_cache.py | 48 +++++ .../user_simulator/cascade/test_phrases.py | 88 +++++++++ .../user_simulator/cascade/test_simulator.py | 187 +++++++++++++++++- .../cascade/test_tick_driven_adapter.py | 35 +++- tests/unit/user_simulator/cascade/test_tts.py | 17 +- 17 files changed, 770 insertions(+), 45 deletions(-) create mode 100644 configs/caller_phrases.yaml create mode 100644 src/eva/user_simulator/cascade/phrases.py create mode 100644 tests/unit/user_simulator/cascade/conftest.py create mode 100644 tests/unit/user_simulator/cascade/test_phrases.py diff --git a/configs/caller_phrases.yaml b/configs/caller_phrases.yaml new file mode 100644 index 00000000..aaef84ba --- /dev/null +++ b/configs/caller_phrases.yaml @@ -0,0 +1,69 @@ +de: + backchannels: + - mhm + - aha + - ja + barge_in_openers: + - Moment— + - Entschuldigung— + - Warte— + - Also— +en: + backchannels: + - uh-huh + - mm-hmm + barge_in_openers: + - Wait— + - Sorry— + - Hold on— + - Actually— +es: + backchannels: + - ajá + - mmm + - ya + barge_in_openers: + - Espera + - Perdona + - Un momento + - Bueno +fr: + backchannels: + - hum hum + - oui + - d’accord + barge_in_openers: + - Attendez + - Pardon + - Juste + - En fait +fr-CA: + backchannels: + - hum hum + - ouais + - OK + barge_in_openers: + - Attends + - Excuse + - Juste + - En fait +hi: + backchannels: + - हूँ + - अच्छा + - हाँ + barge_in_openers: + - रुकिए + - सुनिए + - माफ़ कीजिए + - असल में +ko: + backchannels: + - 네 + - 음 + - 아 + barge_in_openers: + - 잠깐만요 + - 죄송한데 + - 아니요 + - 근데 diff --git a/configs/prompts/simulation.yaml b/configs/prompts/simulation.yaml index 16a0ac50..12e3f2d9 100644 --- a/configs/prompts/simulation.yaml +++ b/configs/prompts/simulation.yaml @@ -542,14 +542,19 @@ user_simulator: - Has the user heard enough to understand what the agent is asking or saying? - Has the user heard enough to have a response, question, or correction ready? - Did the agent just complete the sentence which has all the pertinent information the user was looking for? - - Do NOT repeatedly interrupt the agent if it has spoken only a few words (say less than 5 words). + - Do NOT repeatedly interrupt the agent if it has spoken only a few words (roughly fewer than 5 words in English, or the equivalent short fragment in whatever language the conversation is in — word counts are not comparable across languages). - Are the user's request(s) basically accomplished and the user is likely to hang up on the next turn (if so it should not interrupt)? - Is this a logical point in the conversation to interrupt? Respond with ONLY "YES" if the user should interrupt now, or "NO" if they should keep listening. backchannel_decision: | - You simulate a natural listener who occasionally says "uh-huh" or "mm-hmm" to show they're following along. + You simulate a natural listener who occasionally makes a brief continuer sound to show they're following along. + + The conversation may be in any language. Judge the FUNCTION of what is said, not the + presence of any particular word: every language has these sounds, and the caller draws + its own from a vocabulary configured for the conversation's language. The English + examples below illustrate the judgement, not the words to look for. {conversation_history} @@ -557,7 +562,8 @@ user_simulator: The agent is still speaking [CURRENTLY SPEAKING, INCOMPLETE]. Ignore the trailing incomplete word/phrase — focus only on the COMPLETE sentences delivered so far in the agent's current turn. - Continuers ("uh-huh", "mm-hmm", "yeah") are brief sounds that mean "I'm listening, keep going." They: + Continuers (English: "uh-huh", "mm-hmm", "yeah"; every language has equivalents) are brief + sounds that mean "I'm listening, keep going." They: - Happen naturally during extended speech - Show engagement without interrupting - Are NOT responses to specific content — just signals of attention @@ -565,7 +571,8 @@ user_simulator: Say YES if: - The agent has completed at least 2 full, substantive sentences in their current turn (Short phrases like "Thanks for your patience" or "Let me check on that" don't count as substantive) - - The user hasn't spoken or backchanneled recently (check the last 3 exchanges for ANY user sound including "mm-hmm", "uh-huh", "okay") + - The user hasn't spoken or backchanneled recently (check the last 3 exchanges for ANY brief + acknowledgement sound from the user, in whatever language the conversation is in) - It would feel natural to briefly signal "I'm still here" Say NO if: @@ -581,7 +588,7 @@ user_simulator: - When in doubt, say NO — silence is also natural - Too few continuers is better than too many - Examples: + Examples (English, for illustration — apply the same judgement in any language): AGENT: "Hi there! How can I hel [CURRENTLY SPEAKING, INCOMPLETE]" → NO (just started) @@ -598,7 +605,7 @@ user_simulator: AGENT: "I found your order. It includes a keyboard, thermostat, and headphones. The order was delivered last Tuesday. Now for the exchange, we have a few opti [CURRENTLY SPEAKING, INCOMPLETE]" → YES (extended explanation with specific details) - [If user said "mm-hmm" 2 exchanges ago] + [If the user made a brief acknowledgement sound 2 exchanges ago] AGENT: "...and those are the available options. Now I'll need your input on which [CURRENTLY SPEAKING, INCOMPLETE]" → NO (user backchanneled recently, don't do it again so soon) @@ -618,6 +625,8 @@ user_simulator: - Write it as a natural interruption, not a full turn. Short. - It must follow from your own goal, not from any specific thing the agent might say — you have not heard the reply yet. + - Write it in the same language as the line above; the caller does not switch + languages mid-call. - Reply with ONLY the spoken line, nothing else. No quotes, no labels. - If you would have no reason to interrupt whatever the agent says next, reply with exactly NONE. diff --git a/scripts/add_culture_data.py b/scripts/add_culture_data.py index 4a0b8211..e86f7c9c 100644 --- a/scripts/add_culture_data.py +++ b/scripts/add_culture_data.py @@ -103,6 +103,7 @@ REPO_ROOT = Path(__file__).resolve().parent.parent DATA_DIR = REPO_ROOT / "data" INITIAL_MESSAGES_PATH = REPO_ROOT / "configs" / "agents" / "initial_messages.yaml" +CALLER_PHRASES_PATH = REPO_ROOT / "configs" / "caller_phrases.yaml" WER_CONFIGS_DIR = REPO_ROOT / "src" / "eva" / "utils" / "wer_normalization" / "configs" DEFAULT_MODEL = "gpt-5.5-2026-04-23" @@ -353,6 +354,63 @@ def _update_initial_messages(language: str, message: str) -> None: INITIAL_MESSAGES_PATH.write_text(yaml.safe_dump(existing, allow_unicode=True, sort_keys=True), encoding="utf-8") +async def _translate_caller_phrases( + language: str, language_name: str, llm: LLMClient, overwrite: bool = False +) -> dict[str, list[str]] | None: + """Generate the simulated caller's out-of-turn vocabulary for ``language``. + + These are the continuers and barge-in openers the caller speaks *as audio*, so + they must be things a native speaker actually says, not translations of the + English ones — "uh-huh" has no word-for-word equivalent in most languages. + + Returns None when the language already has phrases and ``overwrite`` is unset. + """ + existing: dict[str, Any] = {} + if CALLER_PHRASES_PATH.exists(): + existing = yaml.safe_load(CALLER_PHRASES_PATH.read_text(encoding="utf-8")) or {} + if language in existing and not overwrite: + return None + + english = existing.get("en", {}) + prompt = ( + f"You are localising a simulated phone caller for {language_name}.\n\n" + "Produce two short lists of things the caller says out loud:\n\n" + "1. backchannels — brief continuer sounds meaning 'I am listening, keep going', " + "said while the other person is still talking. Give the sounds a native speaker " + "actually makes, not translations of English ones.\n" + "2. barge_in_openers — the very first word or two of an interruption, said just " + "before the interrupting sentence. Keep them to one or two words so they sound " + "like a real cut-in.\n\n" + "These are spoken aloud by a text-to-speech voice, so use ordinary spelling with " + "no stage directions, no parentheses and no transliteration hints. Give 2-3 " + "backchannels and 3-4 openers.\n\n" + f"For reference, the English set is: {json.dumps(english, ensure_ascii=False)}\n\n" + 'Return JSON: {"backchannels": ["..."], "barge_in_openers": ["..."]}' + ) + text, _ = await llm.generate_text( + [{"role": "user", "content": prompt}], + response_format={"type": "json_object"}, + ) + data = extract_and_load_json(text) + backchannels = [str(p).strip() for p in (data.get("backchannels") or []) if str(p).strip()] + openers = [str(p).strip() for p in (data.get("barge_in_openers") or []) if str(p).strip()] + if not backchannels or not openers: + raise ValueError(f"Caller phrase generation returned an incomplete result: {data!r}") + return {"backchannels": backchannels, "barge_in_openers": openers} + + +def _update_caller_phrases(language: str, phrases: dict[str, list[str]]) -> None: + """Merge one language's phrase set into configs/caller_phrases.yaml.""" + existing: dict[str, Any] = {} + if CALLER_PHRASES_PATH.exists(): + existing = yaml.safe_load(CALLER_PHRASES_PATH.read_text(encoding="utf-8")) or {} + if existing.get(language) == phrases: + return + existing[language] = phrases + CALLER_PHRASES_PATH.parent.mkdir(parents=True, exist_ok=True) + CALLER_PHRASES_PATH.write_text(yaml.safe_dump(existing, allow_unicode=True, sort_keys=True), encoding="utf-8") + + async def _translate_aliases( name_to_base: dict[str, list[str]], language_name: str, @@ -680,6 +738,16 @@ async def amain(args: argparse.Namespace) -> int: _update_initial_messages(args.language, initial_message) logger.info(f"Updated {INITIAL_MESSAGES_PATH}") + logger.info(f"Generating caller out-of-turn phrases for {args.language_name}") + caller_phrases = await _translate_caller_phrases(args.language, args.language_name, llm, args.overwrite_all) + if caller_phrases is None: + logger.info(f"Caller phrases for {args.language} already present — skipping") + elif args.dry_run: + logger.info(f"[dry-run] would write caller phrases for {args.language}: {caller_phrases}") + else: + _update_caller_phrases(args.language, caller_phrases) + logger.info(f"Updated {CALLER_PHRASES_PATH}: {caller_phrases}") + for domain in domains: logger.info(f"=== Domain: {domain} ===") if domain == "airline": diff --git a/src/eva/user_simulator/cascade/adapter/tick_driven.py b/src/eva/user_simulator/cascade/adapter/tick_driven.py index d75c096b..7c0944de 100644 --- a/src/eva/user_simulator/cascade/adapter/tick_driven.py +++ b/src/eva/user_simulator/cascade/adapter/tick_driven.py @@ -124,7 +124,7 @@ async def run_tick(self, tick_number: int, outgoing_audio: bytes | None, *, barg if raw: self._ticks_released += 1 - self._check_provider_alive(bool(raw)) + stalled = self._provider_has_stalled(bool(raw)) if self._error is not None: raise RuntimeError("TickDrivenAdapter receive loop failed") from self._error @@ -135,6 +135,7 @@ async def run_tick(self, tick_number: int, outgoing_audio: bytes | None, *, barg assistant_audio_raw_bytes=len(raw), wall_clock_ms=int(time.time() * 1000), interruption_audio_start_ms=interruption_start, + provider_stalled=stalled, ) def _on_inbound_audio(self) -> None: @@ -178,14 +179,19 @@ async def _send_unpaced(self, pcm: bytes) -> None: ) ) - def _check_provider_alive(self, received: bool) -> None: - """Raise if the provider has produced nothing for too long. + def _provider_has_stalled(self, received: bool) -> bool: + """Whether the provider has produced nothing for too long. Wall clock is the right measure here and only here: this is a liveness check on a real network peer, not a measurement of conversation time. + + Reported, not raised. Raising aborted the tick loop from underneath the + simulator, so the record ended on the generic "error" reason with no way to + tell a stalled provider from a bug in the caller, and the terminal state the + runner reads was never written. The caller ends the conversation instead. """ now = time.monotonic() if received: self._last_inbound_monotonic = now - elif now - self._last_inbound_monotonic > MAX_INACTIVE_SECONDS: - raise RuntimeError(f"No assistant audio for {MAX_INACTIVE_SECONDS}s; provider appears stalled") + return False + return now - self._last_inbound_monotonic > MAX_INACTIVE_SECONDS diff --git a/src/eva/user_simulator/cascade/constants.py b/src/eva/user_simulator/cascade/constants.py index 1186f815..94e5c930 100644 --- a/src/eva/user_simulator/cascade/constants.py +++ b/src/eva/user_simulator/cascade/constants.py @@ -38,12 +38,6 @@ LISTENER_CHECK_INTERVAL_MS = 2000 """How often the interrupt and backchannel checks run while the assistant speaks.""" -BACKCHANNEL_PHRASES = ["uh-huh", "mm-hmm"] -"""Fixed continuer vocabulary (tau: voice_config.py:126). Pre-rendered at init.""" - -BARGE_IN_OPENERS = ["Wait—", "Sorry—", "Hold on—", "Actually—"] -"""Fixed barge-in openers. Pre-rendered so a decision can be voiced at zero latency.""" - def ms_to_ticks(milliseconds: int) -> int: """Convert milliseconds to whole ticks, flooring.""" diff --git a/src/eva/user_simulator/cascade/phrase_cache.py b/src/eva/user_simulator/cascade/phrase_cache.py index 7621a0fa..69d7e966 100644 --- a/src/eva/user_simulator/cascade/phrase_cache.py +++ b/src/eva/user_simulator/cascade/phrase_cache.py @@ -4,7 +4,7 @@ import asyncio import random -from typing import Protocol +from typing import ClassVar, Protocol from eva.utils.logging import get_logger @@ -20,30 +20,64 @@ async def synthesize(self, text: str, *, voice_id: str) -> bytes: class PhraseCache: - """Renders a fixed vocabulary once at init so it can be voiced at zero latency. + """Renders a fixed vocabulary once so it can be voiced at zero latency. Backchannels and barge-in openers are short, fixed, and voice-stable, which makes them cacheable — and caching is the only way a 300ms "mm-hmm" reliably lands on the tick the decision chose. + + The audio is held **per process, keyed by (voice_id, phrase)**, not per + simulator. The vocabulary depends only on the voice, so a run of N records + against two voices needs two renders of each phrase rather than 2N: every + conversation after the first finds the audio already there and starts without + waiting on TTS at all. Only the RNG is per-instance, so each conversation + still chooses its phrases independently and reproducibly. """ + _audio: ClassVar[dict[tuple[str, str], bytes]] = {} + _render_lock: ClassVar[asyncio.Lock | None] = None + def __init__(self, tts: SpeechSynthesizer, *, voice_id: str, seed: int = 0) -> None: self._tts = tts self._voice_id = voice_id self._rng = random.Random(seed) - self._audio: dict[str, bytes] = {} + + @classmethod + def _lock(cls) -> asyncio.Lock: + """Lazily create the shared render lock, on whichever loop is running.""" + if cls._render_lock is None: + cls._render_lock = asyncio.Lock() + return cls._render_lock + + @classmethod + def clear(cls) -> None: + """Drop all cached audio. For tests, and for a voice set changing mid-process.""" + cls._audio.clear() async def prerender(self, phrases: list[str]) -> None: - """Synthesize every phrase concurrently and hold the audio in memory.""" - rendered = await asyncio.gather(*(self._tts.synthesize(p, voice_id=self._voice_id) for p in phrases)) - self._audio.update(dict(zip(phrases, rendered, strict=True))) - logger.info(f"Pre-rendered {len(phrases)} caller phrases") + """Synthesize any phrase this voice has not rendered yet. + + Serialized across conversations: concurrent records share one voice, so + without the lock each would discover the same empty cache and render the + same phrases. The lock is held only for genuine misses. + """ + async with self._lock(): + missing = [p for p in phrases if (self._voice_id, p) not in self._audio] + if not missing: + logger.debug(f"All {len(phrases)} caller phrases already rendered for {self._voice_id}") + return + rendered = await asyncio.gather(*(self._tts.synthesize(p, voice_id=self._voice_id) for p in missing)) + self._audio.update( + {(self._voice_id, phrase): audio for phrase, audio in zip(missing, rendered, strict=True)} + ) + logger.info(f"Pre-rendered {len(missing)} caller phrases for voice {self._voice_id}") def get(self, phrase: str) -> bytes: - """Return cached audio for a phrase.""" - if phrase not in self._audio: - raise KeyError(f"Phrase not pre-rendered: {phrase!r}") - return self._audio[phrase] + """Return cached audio for a phrase in this cache's voice.""" + key = (self._voice_id, phrase) + if key not in self._audio: + raise KeyError(f"Phrase not pre-rendered for voice {self._voice_id}: {phrase!r}") + return self._audio[key] def choose(self, phrases: list[str]) -> str: """Pick a phrase using the cache's seeded RNG, so runs stay reproducible.""" diff --git a/src/eva/user_simulator/cascade/phrases.py b/src/eva/user_simulator/cascade/phrases.py new file mode 100644 index 00000000..6031d92d --- /dev/null +++ b/src/eva/user_simulator/cascade/phrases.py @@ -0,0 +1,102 @@ +"""Per-language phrase vocabularies for the caller's out-of-turn behavior. + +Data rather than constants because these are *words*, and words are language +specific. Timing thresholds stay in `constants.py` — varying those across runs +makes metrics incomparable, whereas speaking English continuers into a French +conversation is simply wrong. + +Generated per language by `scripts/add_culture_data.py`, alongside the initial +message it already translates, so a run only ever reads this file. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +from eva.utils.logging import get_logger + +logger = get_logger(__name__) + +PHRASES_PATH = Path(__file__).resolve().parents[4] / "configs" / "caller_phrases.yaml" + +FALLBACK_LANGUAGE = "en" + +_cache: dict[str, CallerPhrases] = {} + + +@dataclass(frozen=True) +class CallerPhrases: + """The fixed things the caller can say without taking a turn.""" + + backchannels: list[str] + """Continuers: "I'm listening, keep going".""" + + barge_in_openers: list[str] + """Opening fragment of an interruption, voiced ahead of the content.""" + + @property + def vocabulary(self) -> list[str]: + """Every phrase that needs pre-rendering, in a stable order.""" + return [*self.backchannels, *self.barge_in_openers] + + +def _read_file() -> dict[str, Any]: + """Load the phrase file, or return empty when it is missing or unreadable.""" + try: + data = yaml.safe_load(PHRASES_PATH.read_text(encoding="utf-8")) + except OSError as exc: + logger.warning(f"Caller phrase file unreadable ({exc}); out-of-turn behavior has no vocabulary") + return {} + return data or {} + + +def candidate_languages(language: str) -> list[str]: + """Language tags to try, most specific first: 'fr-CA' -> 'fr-CA', 'fr', 'en'. + + A regional variant nearly always shares its continuers with the base language, + so falling back to it is far better than falling back to English. + """ + candidates = [language] + if "-" in language: + candidates.append(language.split("-", 1)[0]) + if FALLBACK_LANGUAGE not in candidates: + candidates.append(FALLBACK_LANGUAGE) + return candidates + + +def load_phrases(language: str) -> CallerPhrases: + """Return the caller's phrase vocabulary for a language. + + Falls back to the base language and then to English rather than raising: a + missing translation should degrade the realism of the behavior, not abort the + run. The fallback is logged because English continuers in a non-English call + are a defect in the data, not an acceptable outcome. + """ + if language in _cache: + return _cache[language] + + data = _read_file() + for candidate in candidate_languages(language): + entry = data.get(candidate) + if not entry: + continue + if candidate != language: + logger.warning( + f"No caller phrases for {language!r}; falling back to {candidate!r}. " + f"Run scripts/add_culture_data.py --language {language} to generate them." + ) + phrases = CallerPhrases( + backchannels=list(entry.get("backchannels") or []), + barge_in_openers=list(entry.get("barge_in_openers") or []), + ) + _cache[language] = phrases + return phrases + + logger.error(f"No caller phrases for {language!r} and no {FALLBACK_LANGUAGE!r} fallback in {PHRASES_PATH}") + empty = CallerPhrases(backchannels=[], barge_in_openers=[]) + _cache[language] = empty + return empty diff --git a/src/eva/user_simulator/cascade/simulator.py b/src/eva/user_simulator/cascade/simulator.py index 0f5c03cd..b650f8ef 100644 --- a/src/eva/user_simulator/cascade/simulator.py +++ b/src/eva/user_simulator/cascade/simulator.py @@ -14,10 +14,8 @@ from eva.user_simulator.base import AbstractUserSimulator from eva.user_simulator.cascade.adapter.base import Adapter from eva.user_simulator.cascade.adapter.realtime_ws import RealtimeWSAdapter -from eva.user_simulator.cascade.adapter.tick_driven import TickDrivenAdapter +from eva.user_simulator.cascade.adapter.tick_driven import MAX_INACTIVE_SECONDS, TickDrivenAdapter from eva.user_simulator.cascade.constants import ( - BACKCHANNEL_PHRASES, - BARGE_IN_OPENERS, CALLER_SAMPLE_RATE, INACTIVITY_TIMEOUT_MS, TICK_DURATION_MS, @@ -28,6 +26,7 @@ from eva.user_simulator.cascade.decision_log import DecisionLog from eva.user_simulator.cascade.decisions import ListenerDecisions, parse_yes_no from eva.user_simulator.cascade.phrase_cache import PhraseCache +from eva.user_simulator.cascade.phrases import load_phrases from eva.user_simulator.cascade.scheduler import TickScheduler from eva.user_simulator.cascade.stt_livekit import LiveKitStreamingSTT from eva.user_simulator.cascade.tick_result import TickResult @@ -213,6 +212,7 @@ class CascadeUserSimulator(AbstractUserSimulator): _ticks_since_assistant_started = 0 _may_interrupt_this_turn = False _assistant_turn_index = 0 + _last_checked_text = "" def __init__( self, @@ -247,6 +247,8 @@ def __init__( self._tts = CartesiaTTS(simulator_config.tts_params, language=language) self._llm = LiteLLMClient(model=simulator_config.llm) self._voice_id = self._tts.voice_for_persona(persona_config) + # The caller's out-of-turn vocabulary is language data, not code. + self._phrases = load_phrases(language) self._history: list[dict[str, str]] = [] # Shared by the listener checks and the relevance gate so both cost one client. self._decision_client = _DecisionClient(LiteLLMClient(model=simulator_config.decision_llm)) @@ -307,6 +309,17 @@ async def _run(self) -> None: self._log_audio_boundaries(scheduler, result, assistant_was_speaking, caller_was_speaking) caller_was_speaking = scheduler.caller_spoke_this_tick assistant_was_speaking = result.has_assistant_speech + if result.provider_stalled: + # Distinct from inactivity_timeout, which is a *legitimate* end the + # metrics treat as definitive when the user spoke last. A stall is a + # dead peer: the record is invalid and the runner should retry it, + # which is what the reason not being "goodbye" already means to it. + logger.error( + f"tick {scheduler.tick}: no assistant audio for " + f"{MAX_INACTIVE_SECONDS}s; abandoning the conversation as unusable" + ) + self._on_conversation_end("provider_stalled") + break # Captured before the inactivity check, which clears it on a speech tick. silent_before = self._ticks_assistant_silent if self._assistant_is_inactive(scheduler, result): @@ -322,6 +335,11 @@ async def _run(self) -> None: # One roll per assistant turn, so a turn carries at most one barge-in. self._may_interrupt_this_turn = self._config.enable_interruptions self._assistant_turn_index += 1 + # The transcript is consumed between turns, so it can shrink back + # toward the in-flight partial and collide with a value already + # judged. Clearing here keeps "unchanged" meaning "unchanged + # within this turn", which is the only span it is asked about. + self._last_checked_text = "" self._decision_log.log( "assistant_turn_start", tick=scheduler.tick, @@ -359,9 +377,9 @@ async def _prepare_listener_behaviors(self) -> None: """ vocabulary: list[str] = [] if self._config.enable_backchannel: - vocabulary += BACKCHANNEL_PHRASES + vocabulary += self._phrases.backchannels if self._config.enable_interruptions: - vocabulary += BARGE_IN_OPENERS + vocabulary += self._phrases.barge_in_openers if not vocabulary: return @@ -376,10 +394,29 @@ async def _prepare_listener_behaviors(self) -> None: ) async def _run_checks(self, scheduler: TickScheduler) -> bool: - """Run the listener-reaction checks and act on the verdict. True means hang up.""" + """Run the listener-reaction checks and act on the verdict. True means hang up. + + Gated on there being something new to judge. The checks fire on a timer while + the assistant speaks — ~101 times in one measured call, two concurrent LLM calls + each — but their only input is the transcript, so a tick where the transcript has + not moved re-asks a question already answered. Skipping those is free: identical + input, identical verdict. + """ if self._decisions is None or self._phrase_cache is None: return False + history = self._stt.buffer.current_text() + # Nothing transcribed yet is not a question worth asking, and a transcript that + # has not moved re-asks one already answered. + if not history or history == self._last_checked_text: + return False + self._last_checked_text = history + + # Backchannelling stays available after a barge-in: a listener who cuts in and + # later hums along is ordinary, and the one-barge-in-per-turn cap is about not + # talking over the assistant repeatedly, not about going quiet afterwards. + # `allow_interrupt` alone already skips the interrupt call without reaching the + # model once the cap is spent. verdict = await self._decisions.evaluate( history, allow_interrupt=self._may_interrupt_this_turn, @@ -406,7 +443,7 @@ async def _run_checks(self, scheduler: TickScheduler) -> bool: self._may_interrupt_this_turn = False return await self._play_interruption(scheduler) if verdict.should_backchannel: - phrase = play_backchannel(scheduler, self._phrase_cache, BACKCHANNEL_PHRASES) + phrase = play_backchannel(scheduler, self._phrase_cache, self._phrases.backchannels) # Recorded too, or the saved clean track diverges from what went on the wire. self._record_audio("user_clean", self._phrase_cache.get(phrase)) self.event_logger.log_event("backchannel", {"text": phrase, "tick_index": scheduler.tick}) @@ -428,7 +465,7 @@ async def _play_interruption(self, scheduler: TickScheduler) -> bool: intended_tick = scheduler.tick intended_turn = self._assistant_turn_index started_at = time.monotonic() - opener = self._phrase_cache.choose(BARGE_IN_OPENERS) + opener = self._phrase_cache.choose(self._phrases.barge_in_openers) opener_audio = self._phrase_cache.get(opener) if self._config.speculative_generation and self._candidate_audio: @@ -575,8 +612,14 @@ def _log_tick_state(self, scheduler: TickScheduler, result: TickResult, *, is_ch `rms` is why this exists: `has_assistant_speech` is true for any non-zero bytes, digital silence included, so a transport that pads with silence reads as continuous speech. Recording both lets that be measured rather than inferred. + + `transcript_moved` serves the same purpose for the other gate. A check tick only + reaches the judges when the transcript has changed since the last one judged, so + without this a check tick with no following `listener_check` is ambiguous between + "nothing new was heard" and "the check ran and something went wrong". """ raw = result.assistant_audio[: result.assistant_audio_raw_bytes] + history = self._stt.buffer.current_text() self._decision_log.log( "tick", tick=scheduler.tick, @@ -589,6 +632,7 @@ def _log_tick_state(self, scheduler: TickScheduler, result: TickResult, *, is_ch ticks_since_assistant_started=self._ticks_since_assistant_started, may_interrupt_this_turn=self._may_interrupt_this_turn, is_check_tick=is_check_tick, + transcript_moved=bool(history) and history != self._last_checked_text, has_candidate=bool(self._candidate_audio), ) @@ -659,7 +703,9 @@ async def _prerender_candidate(self, utterance: str) -> None: scripted interruption that lands as a non-sequitur. """ self._candidate_text, self._candidate_audio = "", b"" - if not self._config.speculative_generation: + # A candidate only ever gets spoken by a barge-in, so rendering one with + # interruptions off buys an LLM call and a synthesis per turn for nothing. + if not self._config.speculative_generation or not self._config.enable_interruptions: return started = time.monotonic() prompt = PromptManager().get_prompt("user_simulator.cascade_next_interruption", utterance=utterance) diff --git a/src/eva/user_simulator/cascade/tick_result.py b/src/eva/user_simulator/cascade/tick_result.py index d4e66603..f2343459 100644 --- a/src/eva/user_simulator/cascade/tick_result.py +++ b/src/eva/user_simulator/cascade/tick_result.py @@ -30,6 +30,15 @@ class TickResult: interruption_audio_start_ms: int | None = None """Played position where the caller cut in, in simulated ms.""" + provider_stalled: bool = False + """The assistant has produced nothing for so long that the run is not usable. + + Reported rather than raised. A stall is a bad *record*, not a bad *program*: the + conversation ends with a terminal reason the runner treats as a validation failure + and retries, exactly like any other unfinished record, and the partial audio and + event log survive for diagnosis. + """ + @property def has_assistant_speech(self) -> bool: """Whether any real assistant audio arrived this tick.""" diff --git a/src/eva/user_simulator/cascade/tts.py b/src/eva/user_simulator/cascade/tts.py index c38e82e4..a6c5b496 100644 --- a/src/eva/user_simulator/cascade/tts.py +++ b/src/eva/user_simulator/cascade/tts.py @@ -16,7 +16,7 @@ CARTESIA_URL = "https://api.cartesia.ai/tts/bytes" CARTESIA_VERSION = "2024-06-10" DEFAULT_FEMALE_VOICE = "f786b574-daa5-4673-aa0c-cbe3e8534c02" -DEFAULT_MALE_VOICE = "f786b574-daa5-4673-aa0c-cbe3e8534c02" +DEFAULT_MALE_VOICE = "a0e99841-438c-4a64-b679-ae501e7d6091" _FEMALE_PERSONA_ID = 1 diff --git a/tests/unit/user_simulator/cascade/conftest.py b/tests/unit/user_simulator/cascade/conftest.py new file mode 100644 index 00000000..038040e3 --- /dev/null +++ b/tests/unit/user_simulator/cascade/conftest.py @@ -0,0 +1,17 @@ +import pytest + +from eva.user_simulator.cascade.phrase_cache import PhraseCache + + +@pytest.fixture(autouse=True) +def _isolate_phrase_cache(): + """Clear the process-global phrase audio between tests. + + The cache is shared across conversations on purpose — that is what stops every + record re-rendering the same "mm-hmm" — but tests are conversations too, so + without this one test's renders satisfy the next one's prerender and call + counts come out wrong. + """ + PhraseCache.clear() + yield + PhraseCache.clear() diff --git a/tests/unit/user_simulator/cascade/test_constants.py b/tests/unit/user_simulator/cascade/test_constants.py index 5d7fb0dc..1aeacec0 100644 --- a/tests/unit/user_simulator/cascade/test_constants.py +++ b/tests/unit/user_simulator/cascade/test_constants.py @@ -34,8 +34,10 @@ def test_listener_check_interval_is_two_seconds_in_ticks(): assert ms_to_ticks(LISTENER_CHECK_INTERVAL_MS) == 10 -def test_fixed_vocabularies_are_non_empty(): - from eva.user_simulator.cascade.constants import BACKCHANNEL_PHRASES, BARGE_IN_OPENERS +def test_the_vocabularies_are_no_longer_constants(): + # They moved to configs/caller_phrases.yaml because they are language data, not + # timing. Timing constants staying here is the whole distinction. + from eva.user_simulator.cascade import constants - assert BACKCHANNEL_PHRASES == ["uh-huh", "mm-hmm"] - assert len(BARGE_IN_OPENERS) >= 2 + assert not hasattr(constants, "BACKCHANNEL_PHRASES") + assert not hasattr(constants, "BARGE_IN_OPENERS") diff --git a/tests/unit/user_simulator/cascade/test_phrase_cache.py b/tests/unit/user_simulator/cascade/test_phrase_cache.py index d2ff72f9..b951976e 100644 --- a/tests/unit/user_simulator/cascade/test_phrase_cache.py +++ b/tests/unit/user_simulator/cascade/test_phrase_cache.py @@ -51,3 +51,51 @@ async def test_requesting_an_unrendered_phrase_raises(): with pytest.raises(KeyError, match="not pre-rendered"): cache.get("never-rendered") + + +async def test_a_second_conversation_on_the_same_voice_renders_nothing(): + # This is the point of the global cache: a run of N records against one voice + # renders each phrase once, not N times. + tts = FakeTTS() + first = PhraseCache(tts, voice_id="voice-f") + await first.prerender(["uh-huh", "mm-hmm"]) + + second = PhraseCache(tts, voice_id="voice-f") + await second.prerender(["uh-huh", "mm-hmm"]) + + assert len(tts.calls) == 2 + assert second.get("uh-huh") == b"uh-huh" + + +async def test_only_the_phrases_a_voice_is_missing_are_rendered(): + tts = FakeTTS() + await PhraseCache(tts, voice_id="voice-f").prerender(["uh-huh"]) + tts.calls.clear() + + await PhraseCache(tts, voice_id="voice-f").prerender(["uh-huh", "mm-hmm"]) + + assert tts.calls == ["mm-hmm"] + + +async def test_each_voice_keeps_its_own_audio(): + # Two genders means two voices; one must never be served the other's audio. + tts = FakeTTS() + female = PhraseCache(tts, voice_id="voice-f") + male = PhraseCache(tts, voice_id="voice-m") + await female.prerender(["uh-huh"]) + await male.prerender(["uh-huh"]) + + assert len(tts.calls) == 2 + assert female.get("uh-huh") == b"uh-huh" + assert male.get("uh-huh") == b"uh-huh" + + +async def test_concurrent_conversations_do_not_double_render(): + import asyncio + + tts = FakeTTS() + caches = [PhraseCache(tts, voice_id="voice-f") for _ in range(4)] + + await asyncio.gather(*(c.prerender(["uh-huh", "mm-hmm"]) for c in caches)) + + assert sorted(tts.calls) == ["mm-hmm", "uh-huh"] diff --git a/tests/unit/user_simulator/cascade/test_phrases.py b/tests/unit/user_simulator/cascade/test_phrases.py new file mode 100644 index 00000000..60ca00f3 --- /dev/null +++ b/tests/unit/user_simulator/cascade/test_phrases.py @@ -0,0 +1,88 @@ +import pytest +import yaml + +from eva.user_simulator.cascade import phrases as phrases_module +from eva.user_simulator.cascade.phrases import ( + PHRASES_PATH, + CallerPhrases, + candidate_languages, + load_phrases, +) + + +@pytest.fixture(autouse=True) +def _clear_phrase_cache(): + phrases_module._cache.clear() + yield + phrases_module._cache.clear() + + +@pytest.fixture +def phrase_file(tmp_path, monkeypatch): + """Point the loader at a temporary phrase file.""" + + def write(data): + path = tmp_path / "caller_phrases.yaml" + path.write_text(yaml.safe_dump(data, allow_unicode=True), encoding="utf-8") + monkeypatch.setattr(phrases_module, "PHRASES_PATH", path) + return path + + return write + + +def test_the_shipped_file_has_english(): + # English is the fallback for every language, so it is the one entry that must exist. + data = yaml.safe_load(PHRASES_PATH.read_text(encoding="utf-8")) + + assert data["en"]["backchannels"] + assert data["en"]["barge_in_openers"] + + +def test_a_language_with_its_own_entry_uses_it(phrase_file): + phrase_file( + { + "en": {"backchannels": ["uh-huh"], "barge_in_openers": ["Wait—"]}, + "fr": {"backchannels": ["hmm", "ouais"], "barge_in_openers": ["Attendez—"]}, + } + ) + + assert load_phrases("fr") == CallerPhrases(backchannels=["hmm", "ouais"], barge_in_openers=["Attendez—"]) + + +def test_a_regional_variant_falls_back_to_its_base_language(phrase_file): + # fr-CA shares its continuers with fr; falling all the way to English would be worse. + phrase_file( + { + "en": {"backchannels": ["uh-huh"], "barge_in_openers": ["Wait—"]}, + "fr": {"backchannels": ["hmm"], "barge_in_openers": ["Attendez—"]}, + } + ) + + assert load_phrases("fr-CA").backchannels == ["hmm"] + + +def test_an_unknown_language_falls_back_to_english_rather_than_failing(phrase_file): + # A missing translation should degrade the behavior's realism, not abort the run. + phrase_file({"en": {"backchannels": ["uh-huh"], "barge_in_openers": ["Wait—"]}}) + + assert load_phrases("ja").backchannels == ["uh-huh"] + + +def test_a_missing_file_yields_an_empty_vocabulary(phrase_file, tmp_path, monkeypatch): + monkeypatch.setattr(phrases_module, "PHRASES_PATH", tmp_path / "absent.yaml") + + result = load_phrases("en") + + assert result.vocabulary == [] + + +def test_candidate_order_is_most_specific_first(): + assert candidate_languages("fr-CA") == ["fr-CA", "fr", "en"] + assert candidate_languages("fr") == ["fr", "en"] + assert candidate_languages("en") == ["en"] + + +def test_vocabulary_is_everything_that_needs_rendering(phrase_file): + phrase_file({"en": {"backchannels": ["uh-huh", "mm-hmm"], "barge_in_openers": ["Wait—"]}}) + + assert load_phrases("en").vocabulary == ["uh-huh", "mm-hmm", "Wait—"] diff --git a/tests/unit/user_simulator/cascade/test_simulator.py b/tests/unit/user_simulator/cascade/test_simulator.py index 42ded4ba..a7b824ca 100644 --- a/tests/unit/user_simulator/cascade/test_simulator.py +++ b/tests/unit/user_simulator/cascade/test_simulator.py @@ -1,3 +1,4 @@ +from eva.user_simulator.cascade.phrases import load_phrases from eva.user_simulator.cascade.simulator import CascadeUserSimulator, extract_turn, parse_turn_response @@ -462,6 +463,7 @@ def _interrupting_simulator(message, framework="elevenlabs"): sim._decision_log = _trace_sink() sim._framework = framework sim._config = CascadeSimulatorConfig(enable_interruptions=True) + sim._phrases = load_phrases("en") sim._history = [] sim._voice_id = "voice-f" sim._build_prompt = lambda: "SYSTEM PROMPT" @@ -556,9 +558,16 @@ async def _play(scheduler): sim._decisions = _Decisions() sim._play_interruption = _play - sim._stt = type("_Stt", (), {"buffer": type("_B", (), {"current_text": staticmethod(lambda: "hi")})()})() + # The transcript has to grow between the two checks, or the unchanged-transcript + # gate skips the second one before eligibility is ever consulted. + from eva.user_simulator.cascade.stt import TranscriptBuffer + buffer = TranscriptBuffer() + sim._stt = type("_Stt", (), {"buffer": buffer})() + + buffer.committed = "Let me pull up" await sim._run_checks(_InterruptScheduler()) + buffer.committed = "Let me pull up your account." await sim._run_checks(_InterruptScheduler()) assert offered == [True, False] @@ -706,3 +715,179 @@ async def _complete_then_new_turn(messages, tools=None): assert await sim._play_interruption(scheduler) is False assert scheduler.queued == [] + + +def test_provider_stall_is_a_distinct_terminal_reason_from_inactivity(): + # inactivity_timeout is a legitimate end the metrics treat as definitive when the + # user spoke last; a stalled peer is an invalid record the runner should retry. The + # two must not share a reason, or a dead provider scores as a finished conversation. + from eva.user_simulator.cascade.tick_result import TickResult + + stalled = TickResult( + tick_number=9, + assistant_audio=b"\x00" * 8, + assistant_audio_raw_bytes=0, + wall_clock_ms=0, + provider_stalled=True, + ) + + assert stalled.provider_stalled is True + assert ( + TickResult(tick_number=9, assistant_audio=b"", assistant_audio_raw_bytes=0, wall_clock_ms=0).provider_stalled + is False + ) + + +class _CountingDecisions: + """ListenerDecisions stand-in that records every evaluate() it is asked to run.""" + + def __init__(self) -> None: + self.calls: list[str] = [] + + async def evaluate(self, heard, *, allow_interrupt, allow_backchannel): + from eva.user_simulator.cascade.decisions import ListenerVerdict + + self.calls.append(heard) + self.last_allow_backchannel = allow_backchannel + return ListenerVerdict(should_interrupt=False, should_backchannel=False) + + +def _checking_simulator(): + from eva.models.config import CascadeSimulatorConfig + from eva.user_simulator.cascade.stt import TranscriptBuffer + + sim = CascadeUserSimulator.__new__(CascadeUserSimulator) + sim._decision_log = _trace_sink() + sim._config = CascadeSimulatorConfig(enable_interruptions=True, enable_backchannel=True) + sim._phrases = load_phrases("en") + sim._decisions = _CountingDecisions() + sim._phrase_cache = StubCache() + sim._may_interrupt_this_turn = True + sim._last_checked_text = "" + sim.event_logger = _FakeEventLogger() + sim._record_audio = lambda *a, **k: None + buffer = TranscriptBuffer() + sim._stt = type("_Stt", (), {"buffer": buffer})() + return sim, buffer + + +async def test_an_unchanged_transcript_does_not_re_ask_the_judges(): + # The checks fire on a timer but read only the transcript, so a tick where it has + # not moved re-asks a question already answered — ~101 checks x 2 calls per call. + sim, buffer = _checking_simulator() + buffer.committed = "Let me pull up your account." + + await sim._run_checks(_FakeScheduler()) + await sim._run_checks(_FakeScheduler()) + await sim._run_checks(_FakeScheduler()) + + assert sim._decisions.calls == ["Let me pull up your account."] + + +async def test_a_grown_transcript_is_judged_again(): + sim, buffer = _checking_simulator() + buffer.committed = "Let me pull up" + + await sim._run_checks(_FakeScheduler()) + buffer.committed = "Let me pull up your account." + await sim._run_checks(_FakeScheduler()) + + assert len(sim._decisions.calls) == 2 + + +async def test_an_empty_transcript_is_never_judged(): + sim, _buffer = _checking_simulator() + + await sim._run_checks(_FakeScheduler()) + + assert sim._decisions.calls == [] + + +async def test_backchannelling_survives_a_barge_in_in_the_same_turn(): + # Interrupting and later humming along are not mutually exclusive for a real + # listener; the per-turn cap is about not talking over the assistant twice, not + # about going silent for the rest of the turn. + sim, buffer = _checking_simulator() + sim._may_interrupt_this_turn = False # this turn has already barged in + buffer.committed = "Let me pull up your account." + + await sim._run_checks(_FakeScheduler()) + + assert sim._decisions.last_allow_backchannel is True + + +async def test_backchannelling_is_off_only_when_the_config_disables_it(): + from eva.models.config import CascadeSimulatorConfig + + sim, buffer = _checking_simulator() + sim._config = CascadeSimulatorConfig(enable_interruptions=True, enable_backchannel=False) + buffer.committed = "Let me pull up your account." + + await sim._run_checks(_FakeScheduler()) + + assert sim._decisions.last_allow_backchannel is False + + +async def test_a_transcript_already_judged_in_an_earlier_turn_is_judged_again(): + # take_committed() runs between turns, so current_text() can return a string this + # gate has already seen. Without clearing on a turn boundary that later turn's + # check would be skipped as "unchanged" and the barge-in never considered. + sim, buffer = _checking_simulator() + buffer.committed = "Anything else I can help with?" + await sim._run_checks(_FakeScheduler()) + assert len(sim._decisions.calls) == 1 + + # The ordinary turn path consumes the transcript, then the assistant says the same + # thing again a turn later — a real pattern for closing questions. + buffer.take_committed() + sim._last_checked_text = "" # what the new-turn branch in _run does + buffer.committed = "Anything else I can help with?" + + await sim._run_checks(_FakeScheduler()) + + assert len(sim._decisions.calls) == 2 + + +async def test_a_skipped_check_writes_no_listener_check_row(tmp_path): + # The trace exists to separate "the model said NO" from "the check never ran". A row + # for a check that was gated out before reaching the judges would blur exactly that. + from eva.user_simulator.cascade.decision_log import DecisionLog + + sim, buffer = _checking_simulator() + sim._decision_log = DecisionLog(tmp_path / "trace.jsonl") + buffer.committed = "Let me pull up your account." + + await sim._run_checks(_FakeScheduler()) + await sim._run_checks(_FakeScheduler()) + await sim._run_checks(_FakeScheduler()) + + assert sim._decisions.calls == ["Let me pull up your account."] + assert sim._decision_log._counts.get("listener_check") == 1 + + +async def test_the_tick_trace_records_whether_the_transcript_moved(tmp_path): + # A check tick with no listener_check row is otherwise ambiguous between "nothing + # new was heard" and "the check ran and failed". + from eva.user_simulator.cascade.decision_log import DecisionLog + + sim, buffer = _checking_simulator() + rows = [] + sim._decision_log = DecisionLog(tmp_path / "trace.jsonl") + sim._decision_log.log = lambda kind, **fields: rows.append((kind, fields)) + sim._ticks_assistant_silent = 0 + sim._ticks_since_assistant_started = 3 + sim._candidate_audio = b"" + buffer.committed = "Let me pull up your account." + + sim._log_tick_state(_TickTraceScheduler(), _tick(True), is_check_tick=True) + sim._last_checked_text = buffer.current_text() + sim._log_tick_state(_TickTraceScheduler(), _tick(True), is_check_tick=True) + + assert rows[0][1]["transcript_moved"] is True + assert rows[1][1]["transcript_moved"] is False + + +class _TickTraceScheduler: + tick = 12 + caller_is_speaking = False + caller_spoke_this_tick = False diff --git a/tests/unit/user_simulator/cascade/test_tick_driven_adapter.py b/tests/unit/user_simulator/cascade/test_tick_driven_adapter.py index 96cb96c8..8e9a43ac 100644 --- a/tests/unit/user_simulator/cascade/test_tick_driven_adapter.py +++ b/tests/unit/user_simulator/cascade/test_tick_driven_adapter.py @@ -2,7 +2,11 @@ import json import time -from eva.user_simulator.cascade.adapter.tick_driven import QUIET_TICK_GRACE_S, TickDrivenAdapter +from eva.user_simulator.cascade.adapter.tick_driven import ( + MAX_INACTIVE_SECONDS, + QUIET_TICK_GRACE_S, + TickDrivenAdapter, +) from tests.unit.user_simulator.cascade.test_realtime_ws_adapter import ( BYTES_PER_TICK, FakeWebSocket, @@ -172,3 +176,32 @@ async def test_barge_in_discards_audio_the_caller_never_heard(): assert result.assistant_audio_raw_bytes == 0 assert adapter.played_ms == 0 await adapter.stop() + + +async def test_a_stalled_provider_is_reported_not_raised(): + # Raising aborted the tick loop from under the simulator, so the record ended on the + # generic "error" reason and the terminal state the runner reads was never written. + ws = FakeWebSocket() + adapter = TickDrivenAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await adapter.start() + adapter._last_inbound_monotonic = time.monotonic() - (MAX_INACTIVE_SECONDS + 1) + + result = await adapter.run_tick(0, None) + + assert result.provider_stalled is True + await adapter.stop() + + +async def test_a_live_provider_is_never_reported_as_stalled(): + ws = FakeWebSocket() + adapter = TickDrivenAdapter(websocket=ws, conversation_id="c1", bytes_per_tick=BYTES_PER_TICK) + await adapter.start() + adapter._last_inbound_monotonic = time.monotonic() - (MAX_INACTIVE_SECONDS + 1) + await ws.inbound.put(_media_frame(b"\xff" * 8000)) + await _settle() + + result = await adapter.run_tick(0, None) + + # Audio arrived this tick, so the liveness clock resets rather than firing. + assert result.provider_stalled is False + await adapter.stop() diff --git a/tests/unit/user_simulator/cascade/test_tts.py b/tests/unit/user_simulator/cascade/test_tts.py index e66985ec..242bd595 100644 --- a/tests/unit/user_simulator/cascade/test_tts.py +++ b/tests/unit/user_simulator/cascade/test_tts.py @@ -1,6 +1,6 @@ import pytest -from eva.user_simulator.cascade.tts import CartesiaTTS +from eva.user_simulator.cascade.tts import DEFAULT_FEMALE_VOICE, DEFAULT_MALE_VOICE, CartesiaTTS def test_voice_id_selected_for_female_persona(): @@ -53,3 +53,18 @@ async def fake_stream(text, *, voice_id): monkeypatch.setattr(tts, "stream", fake_stream) assert await tts.synthesize("hello", voice_id="voice-f") == b"abcd" + + +def test_the_two_default_voices_are_actually_different(): + # They were the same id, so voice_for_persona returned one voice for every + # persona and the gender scheme it documents was inert. + assert DEFAULT_FEMALE_VOICE != DEFAULT_MALE_VOICE + + +def test_personas_of_different_genders_get_different_voices(): + tts = CartesiaTTS({"api_key": "k"}) + + female = tts.voice_for_persona({"user_persona_id": 1}) + male = tts.voice_for_persona({"user_persona_id": 2}) + + assert female != male