fix(vad): apply the offline dynamic silence schedule per frame

funasr/models/fsmn_vad_streaming/model.py

@@ -353,6 +353,7 @@ def __init__(
         self.output_data_buf_offset = 0
         self.frame_probs = []
         self.max_end_sil_frame_cnt_thresh = max_end_sil_frame_cnt_thresh
+        self.silence_schedule = None
         self.speech_noise_thres = speech_noise_thres
         self.scores = None
         self.max_time_out = False
@@ -1012,9 +1013,11 @@ def inference(
             "dynamic_silence", kwargs.get("max_end_silence_time") is None
         )
         silence_schedule = kwargs.get("silence_schedule", DEFAULT_SILENCE_SCHEDULE)
-        speech_to_sil_ms = self.vad_opts.speech_to_sil_time_thres
-        accumulated_ms = cache.get("_dynamic_accumulated_ms", 0)
-        in_speech = cache.get("_dynamic_in_speech", False)
+        # The schedule is looked up per frame in DetectOneFrame from the current segment's duration.
+        if "stats" in cache:
+            cache["stats"].silence_schedule = silence_schedule if dynamic_silence else None
+            if dynamic_silence:
+                cache["stats"].speech_noise_thres = 0.5
         frontend_inference_kwargs = {}
         if getattr(frontend, "supports_aligned_waveforms", False):
             frontend_inference_kwargs["return_waveform"] = True
@@ -1023,21 +1026,6 @@ def inference(
             kwargs["is_final"] = _is_final and i == n - 1
             audio_sample_i = audio_sample[i * chunk_stride_samples : (i + 1) * chunk_stride_samples]
 
-            # Apply dynamic silence threshold (only accumulate while in speech)
-            if dynamic_silence and "stats" in cache:
-                vad_state = cache["stats"].vad_state_machine
-                # VadStateMachine: 2=kVadInStateStartPointNotDetected, 3=kVadInStateInSpeechSegment
-                if vad_state.value == 2 or in_speech:  # kVadInStateInSpeechSegment
-                    accumulated_ms += chunk_size
-                    in_speech = True
-                for limit_ms, silence_ms in silence_schedule:
-                    if accumulated_ms <= limit_ms:
-                        cache["stats"].max_end_sil_frame_cnt_thresh = max(silence_ms - speech_to_sil_ms, 0)
-                        cache["stats"].speech_noise_thres = 0.5
-                        break
-                cache["_dynamic_accumulated_ms"] = accumulated_ms
-                cache["_dynamic_in_speech"] = in_speech
-
             # extract fbank feats
             speech, speech_lengths = extract_fbank(
                 [audio_sample_i],
@@ -1074,11 +1062,6 @@ def inference(
             segments_i = self.forward(**batch)
             if len(segments_i) > 0:
                 segments.extend(*segments_i)
-                if dynamic_silence:
-                    accumulated_ms = 0
-                    in_speech = False
-                    cache["_dynamic_accumulated_ms"] = 0
-                    cache["_dynamic_in_speech"] = False
 
         cache["prev_samples"] = audio_sample[-m:] if m > 0 else torch.empty(0)
         if _is_final:
@@ -1264,6 +1247,17 @@ def DetectOneFrame(
                             cur_frm_idx - self.LatencyFrmNumAtStartPoint(cache=cache), cache=cache
                         )
             elif cache["stats"].vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment:
+                silence_schedule = getattr(cache["stats"], "silence_schedule", None)
+                if silence_schedule is not None:
+                    segment_ms = (
+                        cur_frm_idx - cache["stats"].confirmed_start_frame + 1
+                    ) * frm_shift_in_ms
+                    for limit_ms, silence_ms in silence_schedule:
+                        if segment_ms <= limit_ms:
+                            cache["stats"].max_end_sil_frame_cnt_thresh = max(
+                                silence_ms - self.vad_opts.speech_to_sil_time_thres, 0
+                            )
+                            break
                 if (
                     cache["stats"].continous_silence_frame_count * frm_shift_in_ms
                     >= cache["stats"].max_end_sil_frame_cnt_thresh

tests/test_fsmn_vad_dynamic_silence.py

@@ -55,13 +55,83 @@ def test_explicit_max_end_silence_time_keeps_fixed_threshold_by_default(self):
         cache = self._run_inference(max_end_silence_time=300)
 
         self.assertEqual(cache["stats"].max_end_sil_frame_cnt_thresh, 200)
-        self.assertNotIn("_dynamic_accumulated_ms", cache)
+        self.assertIsNone(cache["stats"].silence_schedule)
+        self.assertEqual(cache["stats"].speech_noise_thres, 0.6)
 
     def test_explicit_dynamic_silence_still_enables_schedule(self):
         cache = self._run_inference(max_end_silence_time=300, dynamic_silence=True)
 
-        self.assertEqual(cache["stats"].max_end_sil_frame_cnt_thresh, 1900)
-        self.assertEqual(cache["_dynamic_accumulated_ms"], 1000)
+        self.assertEqual(cache["stats"].silence_schedule, vad_model.DEFAULT_SILENCE_SCHEDULE)
+        self.assertEqual(cache["stats"].speech_noise_thres, 0.5)
+        # The threshold is no longer advanced per chunk; DetectOneFrame applies the schedule.
+        self.assertEqual(cache["stats"].max_end_sil_frame_cnt_thresh, 200)
+        self.assertNotIn("_dynamic_accumulated_ms", cache)
+
+
+class TestFsmnVadDynamicSilencePerFrame(unittest.TestCase):
+    """Drive DetectOneFrame with scripted window-detector transitions."""
+
+    def _end_frames(self, speech_ms, silence_ms, schedule, start_frame=0, fixed_thresh_ms=650):
+        vad = vad_model.FsmnVADStreaming.__new__(vad_model.FsmnVADStreaming)
+        torch.nn.Module.__init__(vad)
+        vad.vad_opts = vad_model.VADXOptions()
+        ends = []
+
+        def on_voice_end(end_frame, fake_result, is_last_frame, cache=None):
+            ends.append(end_frame)
+
+        def reset_detection(cache=None):
+            cache["stats"].confirmed_start_frame = -1
+            cache["stats"].vad_state_machine = vad_model.VadStateMachine.kVadInStateStartPointNotDetected
+
+        vad.OnVoiceEnd = on_voice_end
+        vad.PopDataToOutputBuf = lambda *args, **kwargs: None
+        vad.PopDataBufTillFrame = lambda *args, **kwargs: None
+        vad.LatencyFrmNumAtStartPoint = lambda cache=None: 0
+        vad.ResetDetection = reset_detection
+
+        stats = vad_model.Stats(
+            sil_pdf_ids=[0], max_end_sil_frame_cnt_thresh=fixed_thresh_ms, speech_noise_thres=0.5
+        )
+        stats.silence_schedule = schedule
+        speech_frames, silence_frames = speech_ms // 10, silence_ms // 10
+        changes = (
+            [vad_model.AudioChangeState.kChangeStateSil2Speech]
+            + [vad_model.AudioChangeState.kChangeStateSpeech2Speech] * (speech_frames - 1)
+            + [vad_model.AudioChangeState.kChangeStateSpeech2Sil]
+            + [vad_model.AudioChangeState.kChangeStateSil2Sil] * silence_frames
+        )
+        script = iter(changes)
+        cache = {
+            "stats": stats,
+            "windows_detector": SimpleNamespace(DetectOneFrame=lambda *a, **k: next(script)),
+        }
+        for i in range(len(changes)):
+            vad.DetectOneFrame(vad_model.FrameState.kFrameStateSpeech, start_frame + i, False, cache=cache)
+        return [e - start_frame for e in ends]
+
+    def test_schedule_follows_current_segment_duration(self):
+        schedule = vad_model.DEFAULT_SILENCE_SCHEDULE
+        # Thresholds below are the schedule values minus speech_to_sil_time_thres (150 ms).
+        # 5 s into a segment: 2000 ms -> 1850 ms, so 900 ms of silence does not end it.
+        self.assertEqual(self._end_frames(5000, 900, schedule), [])
+        # 15 s into a segment: 1000 ms -> 850 ms, so 900 ms ends it and 700 ms does not.
+        self.assertEqual(len(self._end_frames(15000, 900, schedule)), 1)
+        self.assertEqual(self._end_frames(15000, 700, schedule), [])
+        # 55 s into a segment: 200 ms -> 50 ms, so 300 ms ends it.
+        self.assertEqual(len(self._end_frames(55000, 300, schedule)), 1)
+
+    def test_schedule_does_not_depend_on_absolute_position(self):
+        schedule = vad_model.DEFAULT_SILENCE_SCHEDULE
+        for speech_ms, silence_ms in [(5000, 900), (15000, 900), (55000, 300)]:
+            at_zero = self._end_frames(speech_ms, silence_ms, schedule)
+            shifted = self._end_frames(speech_ms, silence_ms, schedule, start_frame=6000)
+            self.assertEqual(at_zero, shifted)
+
+    def test_without_schedule_threshold_stays_fixed(self):
+        # 650 ms fixed threshold: 700 ms of silence ends a 5 s segment, 600 ms does not.
+        self.assertEqual(len(self._end_frames(5000, 700, None)), 1)
+        self.assertEqual(self._end_frames(5000, 600, None), [])
 
 
 if __name__ == "__main__":