fix(voice): place recorded audio where it actually played

livekit-agents/livekit/agents/voice/io.py

@@ -123,12 +123,29 @@ class PlaybackStartedEvent:
     """The timestamp (time.time())when the playback started"""
 
 
+@dataclass
+class PlaybackProgressedEvent:
+    """A stretch of the current segment that has played.
+
+    Reported once the audio can no longer be discarded, so it is never revised.
+    """
+
+    started_at: float
+    """The timestamp (time.time()) at which this stretch began to play"""
+    offset: float
+    """Where it starts in the audio captured for the current segment"""
+    duration: float
+    """How much of it played"""
+
+
 @dataclass
 class AudioOutputCapabilities:
     pause: bool
 
 
-class AudioOutput(ABC, rtc.EventEmitter[Literal["playback_finished", "playback_started"]]):
+class AudioOutput(
+    ABC, rtc.EventEmitter[Literal["playback_finished", "playback_started", "playback_progressed"]]
+):
     def __init__(
         self,
         *,
@@ -167,6 +184,7 @@ def __init__(
         if next_in_chain is not None:
             next_in_chain.on("playback_finished", self._forward_next_playback_finished)
             next_in_chain.on("playback_started", self._forward_next_playback_started)
+            next_in_chain.on("playback_progressed", self._forward_next_playback_progressed)
 
     def _forward_next_playback_finished(self, ev: PlaybackFinishedEvent) -> None:
         self.on_playback_finished(
@@ -178,6 +196,11 @@ def _forward_next_playback_finished(self, ev: PlaybackFinishedEvent) -> None:
     def _forward_next_playback_started(self, ev: PlaybackStartedEvent) -> None:
         self.on_playback_started(created_at=ev.created_at)
 
+    def _forward_next_playback_progressed(self, ev: PlaybackProgressedEvent) -> None:
+        self.on_playback_progressed(
+            started_at=ev.started_at, offset=ev.offset, duration=ev.duration
+        )
+
     @property
     def label(self) -> str:
         return self.__label
@@ -189,6 +212,17 @@ def next_in_chain(self) -> AudioOutput | None:
     def on_playback_started(self, *, created_at: float) -> None:
         self.emit("playback_started", PlaybackStartedEvent(created_at=created_at))
 
+    def on_playback_progressed(self, *, started_at: float, offset: float, duration: float) -> None:
+        """Report a stretch of the current segment that has played.
+
+        Sinks that own their playback device report one run at a time; one that reports
+        nothing is described by its segment endpoints instead.
+        """
+        self.emit(
+            "playback_progressed",
+            PlaybackProgressedEvent(started_at=started_at, offset=offset, duration=duration),
+        )
+
     def on_playback_finished(
         self,
         *,
@@ -336,6 +370,7 @@ def set_next_in_chain(self, new: AudioOutput) -> None:
         if old is not None:
             old.off("playback_finished", self._forward_next_playback_finished)
             old.off("playback_started", self._forward_next_playback_started)
+            old.off("playback_progressed", self._forward_next_playback_progressed)
             if self._pending_playback_count > 0:
                 # stop audio still playing on the old sink
                 old.clear_buffer()
@@ -347,6 +382,7 @@ def set_next_in_chain(self, new: AudioOutput) -> None:
 
         new.on("playback_finished", self._forward_next_playback_finished)
         new.on("playback_started", self._forward_next_playback_started)
+        new.on("playback_progressed", self._forward_next_playback_progressed)
         if self._attached:
             new.on_attached()